beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

finalize

Browse source
This commit is contained in:
alteredCoder 2022-04-15 15:53:34 +02:00
parent c7723158a0
commit 7f42f94fe3
11 changed files with 101 additions and 72 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

73
cmd/crowdsec-cli/console.go
View file

@ -263,9 +263,9 @@ Disable given information push to the central API.`,
cmdConsole.AddCommand(cmdConsoleStatus)
cmdLabel := &cobra.Command{
Use: "label [feature-flag]",
Short: "Manage label to send with alerts",
cmdContext := &cobra.Command{
Use: "context [feature-flag]",
Short: "Manage context to send with alerts",
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
printHelp(cmd)
@ -274,40 +274,40 @@ Disable given information push to the central API.`,
var keyToAdd string
var valuesToAdd []string
cmdLabelAdd := &cobra.Command{
cmdContextAdd := &cobra.Command{
Use: "add",
Short: "Add label to send with alerts",
Short: "Add context to send with alerts",
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if _, ok := csConfig.Crowdsec.LabelsToSend[keyToAdd]; !ok {
csConfig.Crowdsec.LabelsToSend[keyToAdd] = make([]string, 0)
if _, ok := csConfig.Crowdsec.ContextToSend[keyToAdd]; !ok {
csConfig.Crowdsec.ContextToSend[keyToAdd] = make([]string, 0)
log.Infof("key '%s' added", keyToAdd)
}
data := csConfig.Crowdsec.LabelsToSend[keyToAdd]
data := csConfig.Crowdsec.ContextToSend[keyToAdd]
for _, val := range valuesToAdd {
if !inSlice(val, data) {
log.Infof("value '%s' added to key '%s'", val, keyToAdd)
data = append(data, val)
}
csConfig.Crowdsec.LabelsToSend[keyToAdd] = data
csConfig.Crowdsec.ContextToSend[keyToAdd] = data
}
if err := csConfig.Crowdsec.DumpLabelConfigFile(); err != nil {
if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
log.Fatalf(err.Error())
}
},
}
cmdLabelAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
cmdLabelAdd.Flags().StringSliceVar(&valuesToAdd, "value", []string{}, "The expr fields to associate with the key")
cmdLabelAdd.MarkFlagRequired("key")
cmdLabelAdd.MarkFlagRequired("value")
cmdLabel.AddCommand(cmdLabelAdd)
cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
cmdContextAdd.Flags().StringSliceVar(&valuesToAdd, "value", []string{}, "The expr fields to associate with the key")
cmdContextAdd.MarkFlagRequired("key")
cmdContextAdd.MarkFlagRequired("value")
cmdContext.AddCommand(cmdContextAdd)
cmdLabelStatus := &cobra.Command{
cmdContextStatus := &cobra.Command{
Use: "status",
Short: "List label to send with alerts",
Short: "List context to send with alerts",
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
dump, err := yaml.Marshal(csConfig.Crowdsec.LabelsToSend)
dump, err := yaml.Marshal(csConfig.Crowdsec.ContextToSend)
if err != nil {
log.Fatalf("unable to show labels status: %s", err)
}
@ -315,16 +315,21 @@ Disable given information push to the central API.`,
},
}
cmdLabel.AddCommand(cmdLabelStatus)
cmdContext.AddCommand(cmdContextStatus)
var detectAll bool
cmdLabelDetect := &cobra.Command{
cmdContextDetect := &cobra.Command{
Use: "detect",
Short: "Detect available fields from the installed parsers",
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
var err error
if !detectAll && len(args) == 0 {
log.Infof("Please provide parsers to detect or --all flag.")
printHelp(cmd)
}
// to avoid all the log.Info from the loaders functions
log.SetLevel(log.ErrorLevel)
@ -392,12 +397,12 @@ Disable given information push to the central API.`,
}
},
}
cmdLabelDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser")
cmdLabel.AddCommand(cmdLabelDetect)
cmdContextDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser")
cmdContext.AddCommand(cmdContextDetect)
var keysToDelete []string
var valuesToDelete []string
cmdLabelDelete := &cobra.Command{
cmdContextDelete := &cobra.Command{
Use: "delete",
Short: "List label to send with alerts",
DisableAutoGenTag: true,
@ -407,8 +412,8 @@ Disable given information push to the central API.`,
}
for _, key := range keysToDelete {
if _, ok := csConfig.Crowdsec.LabelsToSend[key]; ok {
delete(csConfig.Crowdsec.LabelsToSend, key)
if _, ok := csConfig.Crowdsec.ContextToSend[key]; ok {
delete(csConfig.Crowdsec.ContextToSend, key)
log.Infof("key '%s' has been removed", key)
} else {
log.Warningf("key '%s' doesn't exist", key)
@ -417,14 +422,14 @@ Disable given information push to the central API.`,
for _, value := range valuesToDelete {
valueFound := false
for key, labels := range csConfig.Crowdsec.LabelsToSend {
for key, labels := range csConfig.Crowdsec.ContextToSend {
if inSlice(value, labels) {
valueFound = true
csConfig.Crowdsec.LabelsToSend[key] = removeFromSlice(value, labels)
csConfig.Crowdsec.ContextToSend[key] = removeFromSlice(value, labels)
log.Infof("value '%s' has been removed from key '%s'", value, key)
}
if len(csConfig.Crowdsec.LabelsToSend[key]) == 0 {
delete(csConfig.Crowdsec.LabelsToSend, key)
if len(csConfig.Crowdsec.ContextToSend[key]) == 0 {
delete(csConfig.Crowdsec.ContextToSend, key)
}
}
if !valueFound {
@ -432,17 +437,17 @@ Disable given information push to the central API.`,
}
}
if err := csConfig.Crowdsec.DumpLabelConfigFile(); err != nil {
if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
log.Fatalf(err.Error())
}
},
}
cmdLabelDelete.Flags().StringSliceVarP(&keysToDelete, "key", "k", []string{}, "The keys to delete")
cmdLabelDelete.Flags().StringSliceVar(&valuesToDelete, "value", []string{}, "The expr fields to delete")
cmdLabel.AddCommand(cmdLabelDelete)
cmdContextDelete.Flags().StringSliceVarP(&keysToDelete, "key", "k", []string{}, "The keys to delete")
cmdContextDelete.Flags().StringSliceVar(&valuesToDelete, "value", []string{}, "The expr fields to delete")
cmdContext.AddCommand(cmdContextDelete)
cmdConsole.AddCommand(cmdLabel)
cmdConsole.AddCommand(cmdContext)
return cmdConsole
}

2
config/config.yaml
View file

@ -14,7 +14,7 @@ config_paths:
notification_dir: /etc/crowdsec/notifications/
plugin_dir: /usr/local/lib/crowdsec/plugins/
crowdsec_service:
console_labels_path: /etc/crowdsec/console/labels.yaml
console_context_path: /etc/crowdsec/console/context.yaml
acquisition_path: /etc/crowdsec/acquis.yaml
parser_routines: 1
cscli:

0
config/labels.yaml → config/context.yaml
View file

2
debian/rules vendored
View file

@ -46,6 +46,6 @@ override_dh_auto_install:
cp config/simulation.yaml debian/crowdsec/etc/crowdsec/simulation.yaml
cp config/profiles.yaml debian/crowdsec/etc/crowdsec/profiles.yaml
cp config/console.yaml debian/crowdsec/etc/crowdsec/console.yaml
cp config/labels.yaml debian/crowdsec/etc/crowdsec/console/labels.yaml
cp config/context.yaml debian/crowdsec/etc/crowdsec/console/context.yaml
cp -a config/patterns debian/crowdsec/etc/crowdsec

32
pkg/csconfig/crowdsec_service.go
View file

@ -15,7 +15,7 @@ import (
type CrowdsecServiceCfg struct {
AcquisitionFilePath string `yaml:"acquisition_path,omitempty"`
AcquisitionDirPath string `yaml:"acquisition_dir,omitempty"`
ConsoleLabelsPath string `yaml:"console_labels_path"`
ConsoleContextPath string `yaml:"console_context_path"`
AcquisitionFiles []string `yaml:"-"`
ParserRoutinesCount int `yaml:"parser_routines"`
BucketsRoutinesCount int `yaml:"buckets_routines"`
@ -31,10 +31,10 @@ type CrowdsecServiceCfg struct {
ConfigDir string `yaml:"-"`
HubIndexFile string `yaml:"-"`
SimulationFilePath string `yaml:"-"`
LabelsToSend map[string][]string `yaml:"-"`
ContextToSend map[string][]string `yaml:"-"`
}
var DefaultLabelsConfigFilePath = DefaultConfigPath("console", "labels.yaml")
var DefaultContextConfigFilePath = DefaultConfigPath("console", "context.yaml")
func (c *Config) LoadCrowdsec() error {
var err error
@ -94,17 +94,17 @@ func (c *Config) LoadCrowdsec() error {
if c.Crowdsec.OutputRoutinesCount <= 0 {
c.Crowdsec.OutputRoutinesCount = 1
}
if c.Crowdsec.ConsoleLabelsPath == "" {
c.Crowdsec.ConsoleLabelsPath = DefaultLabelsConfigFilePath
if c.Crowdsec.ConsoleContextPath == "" {
c.Crowdsec.ConsoleContextPath = DefaultContextConfigFilePath
}
yamlFile, err := ioutil.ReadFile(c.Crowdsec.ConsoleLabelsPath)
yamlFile, err := ioutil.ReadFile(c.Crowdsec.ConsoleContextPath)
if err != nil {
return fmt.Errorf("reading console label file '%s': %s", c.Crowdsec.ConsoleLabelsPath, err)
return fmt.Errorf("reading console label file '%s': %s", c.Crowdsec.ConsoleContextPath, err)
}
c.Crowdsec.LabelsToSend = make(map[string][]string, 0)
err = yaml.Unmarshal(yamlFile, c.Crowdsec.LabelsToSend)
c.Crowdsec.ContextToSend = make(map[string][]string, 0)
err = yaml.Unmarshal(yamlFile, c.Crowdsec.ContextToSend)
if err != nil {
return fmt.Errorf("unmarshaling labels console config file '%s': %s", DefaultLabelsConfigFilePath, err)
return fmt.Errorf("unmarshaling labels console config file '%s': %s", DefaultContextConfigFilePath, err)
}
var crowdsecCleanup = []*string{
@ -136,19 +136,19 @@ func (c *Config) LoadCrowdsec() error {
return nil
}
func (c *CrowdsecServiceCfg) DumpLabelConfigFile() error {
func (c *CrowdsecServiceCfg) DumpContextConfigFile() error {
var out []byte
var err error
if out, err = yaml.Marshal(c.LabelsToSend); err != nil {
return errors.Wrapf(err, "while marshaling ConsoleConfig (for %s)", DefaultLabelsConfigFilePath)
if out, err = yaml.Marshal(c.ContextToSend); err != nil {
return errors.Wrapf(err, "while marshaling ConsoleConfig (for %s)", DefaultContextConfigFilePath)
}
if err := os.WriteFile(c.ConsoleLabelsPath, out, 0600); err != nil {
return errors.Wrapf(err, "while dumping console config to %s", DefaultLabelsConfigFilePath)
if err := os.WriteFile(c.ConsoleContextPath, out, 0600); err != nil {
return errors.Wrapf(err, "while dumping console config to %s", DefaultContextConfigFilePath)
}
log.Infof("%s file saved", c.ConsoleLabelsPath)
log.Infof("%s file saved", c.ConsoleContextPath)
return nil
}

2
pkg/leakybucket/manager_load.go
View file

@ -218,7 +218,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
bucketFactory.wgDumpState = buckets.wgDumpState
bucketFactory.wgPour = buckets.wgPour
bucketFactory.LabelsToSend = cscfg.LabelsToSend
bucketFactory.LabelsToSend = cscfg.ContextToSend
err = LoadBucket(&bucketFactory, tomb)
if err != nil {
log.Errorf("Failed to load bucket %s : %v", bucketFactory.Name, err)

50
pkg/leakybucket/overflows.go
View file

@ -19,6 +19,10 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
)
const (
maxContextValueLen = 4000
)
//SourceFromEvent extracts and formats a valid models.Source object from an Event
func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, error) {
srcs := make(map[string]models.Source)
@ -234,13 +238,34 @@ func alertFormatSource(leaky *Leaky, queue *Queue) (map[string]models.Source, st
return sources, source_type, nil
}
func EventToLabel(labels map[string][]*vm.Program, queue *Queue) models.Meta {
func truncate(values []string) (string, error) {
var ret string
valueByte, err := json.Marshal(values)
if err != nil {
return "", fmt.Errorf("unable to dump metas: %s", err)
}
ret = string(valueByte)
for {
if len(ret) <= maxContextValueLen {
break
}
values = values[:len(values)-1]
valueByte, err = json.Marshal(values)
if err != nil {
return "", fmt.Errorf("unable to dump metas: %s", err)
}
ret = string(valueByte)
}
return ret, nil
}
func EventToContext(labels map[string][]*vm.Program, queue *Queue) models.Meta {
metas := make([]*models.MetaItems0, 0)
tmpLabels := make(map[string][]string)
tmpContext := make(map[string][]string)
for _, evt := range queue.Queue {
for key, values := range labels {
if _, ok := tmpLabels[key]; !ok {
tmpLabels[key] = make([]string, 0)
if _, ok := tmpContext[key]; !ok {
tmpContext[key] = make([]string, 0)
}
for _, value := range values {
var val string
@ -255,27 +280,26 @@ func EventToLabel(labels map[string][]*vm.Program, queue *Queue) models.Meta {
case int:
val = strconv.Itoa(out)
default:
log.Warningf("unexpected return type for label to send : %T", output)
log.Warningf("unexpected return type for context to send : %T", output)
continue
}
if val != "" && !types.InSlice(val, tmpLabels[key]) {
tmpLabels[key] = append(tmpLabels[key], val)
if val != "" && !types.InSlice(val, tmpContext[key]) {
tmpContext[key] = append(tmpContext[key], val)
}
}
}
}
for key, values := range tmpLabels {
for key, values := range tmpContext {
if len(values) == 0 {
continue
}
valueByte, err := json.Marshal(values)
valueStr, err := truncate(values)
if err != nil {
log.Warningf("unable to dump metas: %s", err)
continue
log.Warningf(err.Error())
}
meta := models.MetaItems0{
Key: key,
Value: string(valueByte),
Value: valueStr,
}
metas = append(metas, &meta)
}
@ -345,7 +369,7 @@ func NewAlert(leaky *Leaky, queue *Queue) (types.RuntimeAlert, error) {
*apiAlert.Message = fmt.Sprintf("%s %s performed '%s' (%d events over %s) at %s", source_scope, sourceStr, leaky.Name, leaky.Total_count, leaky.Ovflw_ts.Sub(leaky.First_ts), leaky.Last_ts)
//Get the events from Leaky/Queue
apiAlert.Events = EventsFromQueue(queue)
apiAlert.Meta = EventToLabel(leaky.LabelsToSend, leaky.Queue)
apiAlert.Meta = EventToContext(leaky.LabelsToSend, leaky.Queue)
//Loop over the Sources and generate appropriate number of ApiAlerts
for _, srcValue := range sources {

4
rpm/SPECS/crowdsec.spec
View file

@ -63,7 +63,7 @@ install -m 644 -D config/config.yaml %{buildroot}%{_sysconfdir}/crowdsec
install -m 644 -D config/simulation.yaml %{buildroot}%{_sysconfdir}/crowdsec
install -m 644 -D config/profiles.yaml %{buildroot}%{_sysconfdir}/crowdsec
install -m 644 -D config/console.yaml %{buildroot}%{_sysconfdir}/crowdsec
install -m 644 -D config/console.yaml %{buildroot}%{_sysconfdir}/crowdsec/console/
install -m 644 -D config/context.yaml %{buildroot}%{_sysconfdir}/crowdsec/console/
install -m 644 -D %{SOURCE1} %{buildroot}%{_presetdir}
install -m 551 plugins/notifications/slack/notification-slack %{buildroot}%{_libdir}/%{name}/plugins/
@ -116,7 +116,7 @@ rm -rf %{buildroot}
%config(noreplace) %{_sysconfdir}/%{name}/simulation.yaml
%config(noreplace) %{_sysconfdir}/%{name}/profiles.yaml
%config(noreplace) %{_sysconfdir}/%{name}/console.yaml
%config(noreplace) %{_sysconfdir}/%{name}/console/labels.yaml
%config(noreplace) %{_sysconfdir}/%{name}/console/context.yaml
%config(noreplace) %{_presetdir}/80-%{name}.preset
%config(noreplace) %{_sysconfdir}/%{name}/notifications/http.yaml
%config(noreplace) %{_sysconfdir}/%{name}/notifications/slack.yaml

2
tests/bats/81_alerts-context.bats
View file

@ -35,7 +35,7 @@ teardown() {
ACQUIS_YAML=$(config_yq '.crowdsec_service.acquisition_path')
echo -e "---\nfilename: $tmpfile\nlabels:\n type: syslog\n" >>"${ACQUIS_YAML}"
CONTEXT_YAML=$(config_yq '.crowdsec_service.console_labels_path')
CONTEXT_YAML=$(config_yq '.crowdsec_service.console_context_path')
echo -e "---\ntarget_user:\n- evt.Parsed.sshd_invalid_user\nsource_ip:\n- evt.Parsed.sshd_client_ip\nsource_host:\n- evt.Meta.machine\n" >>"${CONTEXT_YAML}"
./instance-crowdsec start

4
tests/lib/config/config-local
View file

@ -53,7 +53,7 @@ config_generate() {
../config/online_api_credentials.yaml \
"${CONFIG_DIR}/"
cp ../config/labels.yaml "${CONFIG_DIR}/console/"
cp ../config/context.yaml "${CONFIG_DIR}/console/"
cp ../plugins/notifications/*/{http,email,slack,splunk,dummy}.yaml \
"${CONFIG_DIR}/notifications/"
@ -75,7 +75,7 @@ config_generate() {
.api.client.credentials_path=strenv(CONFIG_DIR)+"/local_api_credentials.yaml" |
.api.server.profiles_path=strenv(CONFIG_DIR)+"/profiles.yaml" |
.api.server.console_path=strenv(CONFIG_DIR)+"/console.yaml" |
.crowdsec_service.console_labels_path=strenv(CONFIG_DIR) + "/console/labels.yaml" |
.crowdsec_service.console_context_path=strenv(CONFIG_DIR) + "/console/context.yaml" |
.api.server.online_client.credentials_path=strenv(CONFIG_DIR)+"/online_api_credentials.yaml"
' <../config/config.yaml >"${CONFIG_DIR}/config.yaml"
}

2
wizard.sh
View file

@ -412,7 +412,7 @@ install_crowdsec() {
install -v -m 644 -D ./config/profiles.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit
install -v -m 644 -D ./config/simulation.yaml "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit
install -v -m 644 -D ./config/"${CONSOLE_FILE}" "${CROWDSEC_CONFIG_PATH}" 1> /dev/null || exit
install -v -m 644 -D ./config/labels.yaml "${CROWDSEC_CONSOLE_DIR}" 1> /dev/null || exit
install -v -m 644 -D ./config/context.yaml "${CROWDSEC_CONSOLE_DIR}" 1> /dev/null || exit
DATA=${CROWDSEC_DATA_DIR} CFG=${CROWDSEC_CONFIG_PATH} envsubst '$CFG $DATA' < ./config/user.yaml > ${CROWDSEC_CONFIG_PATH}"/user.yaml" || log_fatal "unable to generate user configuration file"
if [[ ${DOCKER_MODE} == "false" ]]; then