beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

cwhub: context type (#2631)

Browse source 
* add hub type "context"
* cscli lapi: log.Fatal -> fmt.Errorf; lint
* tests for context.yaml
* load console context from hub
* original & compiled context
* deprecate "cscli lapi context delete"
$ cscli lapi context delete
Command "delete" is deprecated, please manually edit the context file.
* cscli completion: add appsec-rules, appsec-configs, explain, hubtest
This commit is contained in:
mmetc 2023-12-07 16:20:13 +01:00 committed by GitHub
parent 3e86f52250
commit 4acb4f8df3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 427 additions and 151 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
cmd/crowdsec-cli/hubcontext.go Normal file
View file

@ -0,0 +1,40 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewContextCLI() *itemCLI {
return &itemCLI{
name: cwhub.CONTEXTS,
singular: "context",
oneOrMore: "context(s)",
help: cliHelp{
example: `cscli contexts list -a
cscli contexts install crowdsecurity/yyy crowdsecurity/zzz
cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz
cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz
cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz
`,
},
installHelp: cliHelp{
example: `cscli contexts install crowdsecurity/yyy crowdsecurity/zzz`,
},
removeHelp: cliHelp{
example: `cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz`,
},
upgradeHelp: cliHelp{
example: `cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz`,
},
inspectHelp: cliHelp{
example: `cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz`,
},
listHelp: cliHelp{
example: `cscli contexts list
cscli contexts list -a
cscli contexts list crowdsecurity/yyy crowdsecurity/zzz
List only enabled contexts unless "-a" or names are specified.`,
},
}
}

151
cmd/crowdsec-cli/lapi.go
View file

@ -2,10 +2,10 @@ package main
import (
"context"
"errors"
"fmt"
"net/url"
"os"
"slices"
"sort"
"strings"
@ -13,6 +13,7 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"slices"
"github.com/crowdsecurity/go-cs-lib/version"
@ -26,26 +27,24 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/parser"
)
var LAPIURLPrefix string = "v1"
const LAPIURLPrefix = "v1"
func runLapiStatus(cmd *cobra.Command, args []string) error {
var err error
password := strfmt.Password(csConfig.API.Client.Credentials.Password)
apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL)
login := csConfig.API.Client.Credentials.Login
if err != nil {
log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
return fmt.Errorf("parsing api url: %w", err)
}
hub, err := require.Hub(csConfig, nil)
if err != nil {
log.Fatal(err)
return err
}
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
log.Fatalf("failed to get scenarios : %s", err)
return fmt.Errorf("failed to get scenarios: %w", err)
}
Client, err = apiclient.NewDefaultClient(apiurl,
@ -53,28 +52,27 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
fmt.Sprintf("crowdsec/%s", version.String()),
nil)
if err != nil {
log.Fatalf("init default client: %s", err)
return fmt.Errorf("init default client: %w", err)
}
t := models.WatcherAuthRequest{
MachineID: &login,
Password: &password,
Scenarios: scenarios,
}
log.Infof("Loaded credentials from %s", csConfig.API.Client.CredentialsFilePath)
log.Infof("Trying to authenticate with username %s on %s", login, apiurl)
_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
if err != nil {
log.Fatalf("Failed to authenticate to Local API (LAPI) : %s", err)
} else {
log.Infof("You can successfully interact with Local API (LAPI)")
return fmt.Errorf("failed to authenticate to Local API (LAPI): %w", err)
}
log.Infof("You can successfully interact with Local API (LAPI)")
return nil
}
func runLapiRegister(cmd *cobra.Command, args []string) error {
var err error
flags := cmd.Flags()
apiURL, err := flags.GetString("url")
@ -95,16 +93,15 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
if lapiUser == "" {
lapiUser, err = generateID("")
if err != nil {
log.Fatalf("unable to generate machine id: %s", err)
return fmt.Errorf("unable to generate machine id: %w", err)
}
}
password := strfmt.Password(generatePassword(passwordLength))
if apiURL == "" {
if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" {
apiURL = csConfig.API.Client.Credentials.URL
} else {
log.Fatalf("No Local API URL. Please provide it in your configuration or with the -u parameter")
if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil || csConfig.API.Client.Credentials.URL == "" {
return fmt.Errorf("no Local API URL. Please provide it in your configuration or with the -u parameter")
}
apiURL = csConfig.API.Client.Credentials.URL
}
/*URL needs to end with /, but user doesn't care*/
if !strings.HasSuffix(apiURL, "/") {
@ -116,7 +113,7 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
}
apiurl, err := url.Parse(apiURL)
if err != nil {
log.Fatalf("parsing api url: %s", err)
return fmt.Errorf("parsing api url: %w", err)
}
_, err = apiclient.RegisterClient(&apiclient.Config{
MachineID: lapiUser,
@ -127,7 +124,7 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
}, nil)
if err != nil {
log.Fatalf("api client register: %s", err)
return fmt.Errorf("api client register: %w", err)
}
log.Printf("Successfully registered to Local API (LAPI)")
@ -147,12 +144,12 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
}
apiConfigDump, err := yaml.Marshal(apiCfg)
if err != nil {
log.Fatalf("unable to marshal api credentials: %s", err)
return fmt.Errorf("unable to marshal api credentials: %w", err)
}
if dumpFile != "" {
err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
if err != nil {
log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
return fmt.Errorf("write api credentials to '%s' failed: %w", dumpFile, err)
}
log.Printf("Local API credentials written to '%s'", dumpFile)
} else {
@ -195,7 +192,7 @@ Keep in mind the machine needs to be validated by an administrator on LAPI side
}
func NewLapiCmd() *cobra.Command {
var cmdLapi = &cobra.Command{
cmdLapi := &cobra.Command{
Use: "lapi [action]",
Short: "Manage interaction with Local API (LAPI)",
Args: cobra.MinimumNArgs(1),
@ -221,6 +218,7 @@ func AddContext(key string, values []string) error {
}
if _, ok := csConfig.Crowdsec.ContextToSend[key]; !ok {
csConfig.Crowdsec.ContextToSend[key] = make([]string, 0)
log.Infof("key '%s' added", key)
}
data := csConfig.Crowdsec.ContextToSend[key]
@ -247,11 +245,11 @@ func NewLapiContextCmd() *cobra.Command {
if err := csConfig.LoadCrowdsec(); err != nil {
fileNotFoundMessage := fmt.Sprintf("failed to open context file: open %s: no such file or directory", csConfig.Crowdsec.ConsoleContextPath)
if err.Error() != fileNotFoundMessage {
log.Fatalf("Unable to load CrowdSec Agent: %s", err)
return fmt.Errorf("unable to start CrowdSec agent: %w", err)
}
}
if csConfig.DisableAgent {
log.Fatalf("Agent is disabled and lapi context can only be used on the agent")
return errors.New("agent is disabled and lapi context can only be used on the agent")
}
return nil
@ -271,12 +269,21 @@ cscli lapi context add --key file_source --value evt.Line.Src
cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil {
return fmt.Errorf("while loading context: %w", err)
}
if keyToAdd != "" {
if err := AddContext(keyToAdd, valuesToAdd); err != nil {
log.Fatalf(err.Error())
return err
}
return
return nil
}
for _, v := range valuesToAdd {
@ -284,9 +291,11 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
key := keySlice[len(keySlice)-1]
value := []string{v}
if err := AddContext(key, value); err != nil {
log.Fatalf(err.Error())
return err
}
}
return nil
},
}
cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
@ -298,19 +307,29 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
Use: "status",
Short: "List context to send with alerts",
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil {
return fmt.Errorf("while loading context: %w", err)
}
if len(csConfig.Crowdsec.ContextToSend) == 0 {
fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.")
return
return nil
}
dump, err := yaml.Marshal(csConfig.Crowdsec.ContextToSend)
if err != nil {
log.Fatalf("unable to show context status: %s", err)
return fmt.Errorf("unable to show context status: %w", err)
}
fmt.Println(string(dump))
fmt.Print(string(dump))
return nil
},
}
cmdContext.AddCommand(cmdContextStatus)
@ -323,9 +342,7 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
cscli lapi context detect crowdsecurity/sshd-logs
`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
var err error
RunE: func(cmd *cobra.Command, args []string) error {
if !detectAll && len(args) == 0 {
log.Infof("Please provide parsers to detect or --all flag.")
printHelp(cmd)
@ -334,19 +351,18 @@ cscli lapi context detect crowdsecurity/sshd-logs
// to avoid all the log.Info from the loaders functions
log.SetLevel(log.WarnLevel)
err = exprhelpers.Init(nil)
if err != nil {
log.Fatalf("Failed to init expr helpers : %s", err)
if err := exprhelpers.Init(nil); err != nil {
return fmt.Errorf("failed to init expr helpers: %w", err)
}
hub, err := require.Hub(csConfig, nil)
if err != nil {
log.Fatal(err)
return err
}
csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil {
log.Fatalf("unable to load parsers: %s", err)
return fmt.Errorf("unable to load parsers: %w", err)
}
fieldByParsers := make(map[string][]string)
@ -366,7 +382,6 @@ cscli lapi context detect crowdsecurity/sshd-logs
fieldByParsers[node.Name] = append(fieldByParsers[node.Name], field)
}
}
}
fmt.Printf("Acquisition :\n\n")
@ -399,59 +414,17 @@ cscli lapi context detect crowdsecurity/sshd-logs
log.Errorf("parser '%s' not found, can't detect fields", parserNotFound)
}
}
return nil
},
}
cmdContextDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser")
cmdContext.AddCommand(cmdContextDetect)
var keysToDelete []string
var valuesToDelete []string
cmdContextDelete := &cobra.Command{
Use: "delete",
Short: "Delete context to send with alerts",
Example: `cscli lapi context delete --key source_ip
cscli lapi context delete --value evt.Line.Src
`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if len(keysToDelete) == 0 && len(valuesToDelete) == 0 {
log.Fatalf("please provide at least a key or a value to delete")
}
for _, key := range keysToDelete {
if _, ok := csConfig.Crowdsec.ContextToSend[key]; ok {
delete(csConfig.Crowdsec.ContextToSend, key)
log.Infof("key '%s' has been removed", key)
} else {
log.Warningf("key '%s' doesn't exist", key)
}
}
for _, value := range valuesToDelete {
valueFound := false
for key, context := range csConfig.Crowdsec.ContextToSend {
if slices.Contains(context, value) {
valueFound = true
csConfig.Crowdsec.ContextToSend[key] = removeFromSlice(value, context)
log.Infof("value '%s' has been removed from key '%s'", value, key)
}
if len(csConfig.Crowdsec.ContextToSend[key]) == 0 {
delete(csConfig.Crowdsec.ContextToSend, key)
}
}
if !valueFound {
log.Warningf("value '%s' not found", value)
}
}
if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
log.Fatalf(err.Error())
}
},
Deprecated: "please manually edit the context file.",
}
cmdContextDelete.Flags().StringSliceVarP(&keysToDelete, "key", "k", []string{}, "The keys to delete")
cmdContextDelete.Flags().StringSliceVar(&valuesToDelete, "value", []string{}, "The expr fields to delete")
cmdContext.AddCommand(cmdContextDelete)
return cmdContext
@ -459,6 +432,7 @@ cscli lapi context delete --value evt.Line.Src
func detectStaticField(GrokStatics []parser.ExtraField) []string {
ret := make([]string, 0)
for _, static := range GrokStatics {
if static.Parsed != "" {
fieldName := fmt.Sprintf("evt.Parsed.%s", static.Parsed)
@ -487,7 +461,8 @@ func detectStaticField(GrokStatics []parser.ExtraField) []string {
}
func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
var ret = make([]string, 0)
ret := make([]string, 0)
if node.Grok.RunTimeRegexp != nil {
for _, capturedField := range node.Grok.RunTimeRegexp.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)

8
cmd/crowdsec-cli/main.go
View file

@ -93,9 +93,10 @@ func initConfig() {
}
var validArgs = []string{
"scenarios", "parsers", "collections", "capi", "lapi", "postoverflows", "machines",
"metrics", "bouncers", "alerts", "decisions", "simulation", "hub", "dashboard",
"config", "completion", "version", "console", "notifications", "support",
"alerts", "appsec-configs", "appsec-rules", "bouncers", "capi", "collections",
"completion", "config", "console", "contexts", "dashboard", "decisions", "explain",
"hub", "hubtest", "lapi", "machines", "metrics", "notifications", "parsers",
"postoverflows", "scenarios", "simulation", "support", "version",
}
func prepender(filename string) string {
@ -246,6 +247,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
rootCmd.AddCommand(NewParserCLI().NewCommand())
rootCmd.AddCommand(NewScenarioCLI().NewCommand())
rootCmd.AddCommand(NewPostOverflowCLI().NewCommand())
rootCmd.AddCommand(NewContextCLI().NewCommand())
rootCmd.AddCommand(NewAppsecConfigCLI().NewCommand())
rootCmd.AddCommand(NewAppsecRuleCLI().NewCommand())

4
cmd/crowdsec-cli/support.go
View file

@ -37,6 +37,7 @@ const (
SUPPORT_OS_INFO_PATH = "osinfo.txt"
SUPPORT_PARSERS_PATH = "hub/parsers.txt"
SUPPORT_SCENARIOS_PATH = "hub/scenarios.txt"
SUPPORT_CONTEXTS_PATH = "hub/scenarios.txt"
SUPPORT_COLLECTIONS_PATH = "hub/collections.txt"
SUPPORT_POSTOVERFLOWS_PATH = "hub/postoverflows.txt"
SUPPORT_BOUNCERS_PATH = "lapi/bouncers.txt"
@ -272,6 +273,7 @@ func (cli cliSupport) NewDumpCmd() *cobra.Command {
- Installed parsers list
- Installed scenarios list
- Installed postoverflows list
- Installed context list
- Bouncers list
- Machines list
- CAPI status
@ -321,6 +323,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())
infos[SUPPORT_SCENARIOS_PATH] = []byte(err.Error())
infos[SUPPORT_POSTOVERFLOWS_PATH] = []byte(err.Error())
infos[SUPPORT_CONTEXTS_PATH] = []byte(err.Error())
infos[SUPPORT_COLLECTIONS_PATH] = []byte(err.Error())
}
@ -356,6 +359,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS)
infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS)
infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
infos[SUPPORT_CONTEXTS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS)
}

6
cmd/crowdsec/crowdsec.go
View file

@ -14,6 +14,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/acquisition"
"github.com/crowdsecurity/crowdsec/pkg/appsec"
"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
@ -24,6 +25,10 @@ import (
func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) {
var err error
if err = alertcontext.LoadConsoleContext(cConfig, hub); err != nil {
return nil, fmt.Errorf("while loading context: %w", err)
}
// Start loading configs
csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
@ -41,6 +46,7 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, er
if err := LoadAcquisition(cConfig); err != nil {
return nil, fmt.Errorf("while loading acquisition config: %w", err)
}
return csParsers, nil
}

2
docker/README.md
View file

@ -320,10 +320,12 @@ config.yaml) each time the container is run.
| `PARSERS` | | Parsers to install, separated by space |
| `SCENARIOS` | | Scenarios to install, separated by space |
| `POSTOVERFLOWS` | | Postoverflows to install, separated by space |
| `CONTEXTS` | | Context files to install, separated by space |
| `DISABLE_COLLECTIONS` | | Collections to remove, separated by space: `-e DISABLE_COLLECTIONS="crowdsecurity/linux crowdsecurity/nginx"` |
| `DISABLE_PARSERS` | | Parsers to remove, separated by space |
| `DISABLE_SCENARIOS` | | Scenarios to remove, separated by space |
| `DISABLE_POSTOVERFLOWS` | | Postoverflows to remove, separated by space |
| `DISABLE_POSTOVERFLOWS` | | Context files to remove, separated by space |
| | | |
| __Log verbosity__ | | |
| `LEVEL_INFO` | false | Force INFO level for the container log |

12
docker/docker_start.sh
View file

@ -300,7 +300,7 @@ fi
conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)'
## Install collections, parsers, scenarios & postoverflows
## Install hub items
cscli hub update
cscli_if_clean collections upgrade crowdsecurity/linux
@ -328,6 +328,11 @@ if [ "$POSTOVERFLOWS" != "" ]; then
cscli_if_clean postoverflows install "$(difference "$POSTOVERFLOWS" "$DISABLE_POSTOVERFLOWS")"
fi
if [ "$CONTEXTS" != "" ]; then
# shellcheck disable=SC2086
cscli_if_clean contexts install "$(difference "$CONTEXTS" "$DISABLE_CONTEXTS")"
fi
## Remove collections, parsers, scenarios & postoverflows
if [ "$DISABLE_COLLECTIONS" != "" ]; then
# shellcheck disable=SC2086
@ -349,6 +354,11 @@ if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" --force
fi
if [ "$DISABLE_CONTEXTS" != "" ]; then
# shellcheck disable=SC2086
cscli_if_clean contexts remove "$DISABLE_CONTEXTS" --force
fi
## Register bouncers via env
for BOUNCER in $(compgen -A variable | grep -i BOUNCER_KEY); do
KEY=$(printf '%s' "${!BOUNCER}")

10
pkg/alertcontext/alertcontext.go
View file

@ -63,13 +63,21 @@ func NewAlertContext(contextToSend map[string][]string, valueLength int) error {
}
for key, values := range contextToSend {
alertContext.ContextToSendCompiled[key] = make([]*vm.Program, 0)
if _, ok := alertContext.ContextToSend[key]; !ok {
alertContext.ContextToSend[key] = make([]string, 0)
}
if _, ok := alertContext.ContextToSendCompiled[key]; !ok {
alertContext.ContextToSendCompiled[key] = make([]*vm.Program, 0)
}
for _, value := range values {
valueCompiled, err := expr.Compile(value, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
if err != nil {
return fmt.Errorf("compilation of '%s' context value failed: %v", value, err)
}
alertContext.ContextToSendCompiled[key] = append(alertContext.ContextToSendCompiled[key], valueCompiled)
alertContext.ContextToSend[key] = append(alertContext.ContextToSend[key], value)
}
}

125
pkg/alertcontext/config.go Normal file
View file

@ -0,0 +1,125 @@
package alertcontext
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
"slices"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
// this file is here to avoid circular dependencies between the configuration and the hub
// HubItemWrapper is a wrapper around a hub item to unmarshal only the context part
// because there are other fields like name etc.
type HubItemWrapper struct {
Context map[string][]string `yaml:"context"`
}
// mergeContext adds the context from src to dest.
func mergeContext(dest map[string][]string, src map[string][]string) {
for k, v := range src {
if _, ok := dest[k]; !ok {
dest[k] = make([]string, 0)
}
for _, s := range v {
if !slices.Contains(dest[k], s) {
dest[k] = append(dest[k], s)
}
}
}
}
// addContextFromItem merges the context from an item into the context to send to the console.
func addContextFromItem(toSend map[string][]string, item *cwhub.Item) error {
filePath := item.State.LocalPath
log.Tracef("loading console context from %s", filePath)
content, err := os.ReadFile(filePath)
if err != nil {
return err
}
wrapper := &HubItemWrapper{}
err = yaml.Unmarshal(content, wrapper)
if err != nil {
return fmt.Errorf("%s: %w", filePath, err)
}
mergeContext(toSend, wrapper.Context)
return nil
}
// addContextFromFile merges the context from a file into the context to send to the console.
func addContextFromFile(toSend map[string][]string, filePath string) error {
log.Tracef("loading console context from %s", filePath)
content, err := os.ReadFile(filePath)
if err != nil {
return err
}
newContext := make(map[string][]string, 0)
err = yaml.Unmarshal(content, newContext)
if err != nil {
return fmt.Errorf("%s: %w", filePath, err)
}
mergeContext(toSend, newContext)
return nil
}
// LoadConsoleContext loads the context from the hub (if provided) and the file console_context_path.
func LoadConsoleContext(c *csconfig.Config, hub *cwhub.Hub) error {
c.Crowdsec.ContextToSend = make(map[string][]string, 0)
if hub != nil {
items, err := hub.GetInstalledItems(cwhub.CONTEXTS)
if err != nil {
return err
}
for _, item := range items {
// context in item files goes under the key 'context'
if err = addContextFromItem(c.Crowdsec.ContextToSend, item); err != nil {
return err
}
}
}
ignoreMissing := false
if c.Crowdsec.ConsoleContextPath != "" {
// if it's provided, it must exist
if _, err := os.Stat(c.Crowdsec.ConsoleContextPath); err != nil {
return fmt.Errorf("while checking console_context_path: %w", err)
}
} else {
c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml")
ignoreMissing = true
}
if err := addContextFromFile(c.Crowdsec.ContextToSend, c.Crowdsec.ConsoleContextPath); err != nil {
if !ignoreMissing || !os.IsNotExist(err) {
return err
}
}
feedback, err := json.Marshal(c.Crowdsec.ContextToSend)
if err != nil {
return fmt.Errorf("marshaling console context: %s", err)
}
log.Debugf("console context to send: %s", feedback)
return nil
}

39
pkg/csconfig/crowdsec_service.go
View file

@ -108,8 +108,9 @@ func (c *Config) LoadCrowdsec() error {
c.Crowdsec.OutputRoutinesCount = 1
}
var crowdsecCleanup = []*string{
crowdsecCleanup := []*string{
&c.Crowdsec.AcquisitionFilePath,
&c.Crowdsec.ConsoleContextPath,
}
for _, k := range crowdsecCleanup {
@ -131,38 +132,10 @@ func (c *Config) LoadCrowdsec() error {
c.Crowdsec.AcquisitionFiles[i] = f
}
if err := c.LoadAPIClient(); err != nil {
if err = c.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %s", err)
}
c.Crowdsec.ContextToSend = make(map[string][]string, 0)
fallback := false
if c.Crowdsec.ConsoleContextPath == "" {
// fallback to default config file
c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml")
fallback = true
}
f, err := filepath.Abs(c.Crowdsec.ConsoleContextPath)
if err != nil {
return fmt.Errorf("fail to get absolute path of %s: %s", c.Crowdsec.ConsoleContextPath, err)
}
c.Crowdsec.ConsoleContextPath = f
yamlFile, err := os.ReadFile(c.Crowdsec.ConsoleContextPath)
if err != nil {
if fallback {
log.Debugf("Default context config file doesn't exist, will not use it")
} else {
return fmt.Errorf("failed to open context file: %s", err)
}
} else {
err = yaml.Unmarshal(yamlFile, c.Crowdsec.ContextToSend)
if err != nil {
return fmt.Errorf("unmarshaling labels console config file '%s': %s", c.Crowdsec.ConsoleContextPath, err)
}
}
return nil
}
@ -170,10 +143,16 @@ func (c *CrowdsecServiceCfg) DumpContextConfigFile() error {
var out []byte
var err error
// XXX: MakeDirs
if out, err = yaml.Marshal(c.ContextToSend); err != nil {
return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", c.ConsoleContextPath, err)
}
if err = os.MkdirAll(filepath.Dir(c.ConsoleContextPath), 0700); err != nil {
return fmt.Errorf("while creating directories for %s: %w", c.ConsoleContextPath, err)
}
if err := os.WriteFile(c.ConsoleContextPath, out, 0600); err != nil {
return fmt.Errorf("while dumping console config to %s: %w", c.ConsoleContextPath, err)
}

21
pkg/csconfig/crowdsec_service_test.go
View file

@ -60,9 +60,10 @@ func TestLoadCrowdsec(t *testing.T) {
ConsoleContextValueLength: 2500,
AcquisitionFiles: []string{acquisFullPath},
SimulationFilePath: "./testdata/simulation.yaml",
ContextToSend: map[string][]string{
"source_ip": {"evt.Parsed.source_ip"},
},
// context is loaded in pkg/alertcontext
// ContextToSend: map[string][]string{
// "source_ip": {"evt.Parsed.source_ip"},
// },
SimulationConfig: &SimulationConfig{
Simulation: ptr.Of(false),
},
@ -98,9 +99,10 @@ func TestLoadCrowdsec(t *testing.T) {
OutputRoutinesCount: 1,
ConsoleContextValueLength: 0,
AcquisitionFiles: []string{acquisFullPath, acquisInDirFullPath},
ContextToSend: map[string][]string{
"source_ip": {"evt.Parsed.source_ip"},
},
// context is loaded in pkg/alertcontext
// ContextToSend: map[string][]string{
// "source_ip": {"evt.Parsed.source_ip"},
// },
SimulationFilePath: "./testdata/simulation.yaml",
SimulationConfig: &SimulationConfig{
Simulation: ptr.Of(false),
@ -136,9 +138,10 @@ func TestLoadCrowdsec(t *testing.T) {
ConsoleContextValueLength: 10,
AcquisitionFiles: []string{},
SimulationFilePath: "",
ContextToSend: map[string][]string{
"source_ip": {"evt.Parsed.source_ip"},
},
// context is loaded in pkg/alertcontext
// ContextToSend: map[string][]string{
// "source_ip": {"evt.Parsed.source_ip"},
// },
SimulationConfig: &SimulationConfig{
Simulation: ptr.Of(false),
},

19
pkg/cwhub/item.go
View file

@ -16,6 +16,7 @@ const (
PARSERS = "parsers"
POSTOVERFLOWS = "postoverflows"
SCENARIOS = "scenarios"
CONTEXTS = "contexts"
APPSEC_CONFIGS = "appsec-configs"
APPSEC_RULES = "appsec-rules"
)
@ -29,7 +30,7 @@ const (
var (
// The order is important, as it is used to range over sub-items in collections.
ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, APPSEC_CONFIGS, APPSEC_RULES, COLLECTIONS}
ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, CONTEXTS, APPSEC_CONFIGS, APPSEC_RULES, COLLECTIONS}
)
type HubItems map[string]map[string]*Item
@ -120,6 +121,7 @@ type Item struct {
PostOverflows []string `json:"postoverflows,omitempty" yaml:"postoverflows,omitempty"`
Scenarios []string `json:"scenarios,omitempty" yaml:"scenarios,omitempty"`
Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"`
Contexts []string `json:"contexts,omitempty" yaml:"contexts,omitempty"`
AppsecConfigs []string `json:"appsec-configs,omitempty" yaml:"appsec-configs,omitempty"`
AppsecRules []string `json:"appsec-rules,omitempty" yaml:"appsec-rules,omitempty"`
}
@ -231,6 +233,15 @@ func (i *Item) SubItems() []*Item {
sub = append(sub, s)
}
for _, name := range i.Contexts {
s := i.hub.GetItem(CONTEXTS, name)
if s == nil {
continue
}
sub = append(sub, s)
}
for _, name := range i.AppsecConfigs {
s := i.hub.GetItem(APPSEC_CONFIGS, name)
if s == nil {
@ -284,6 +295,12 @@ func (i *Item) logMissingSubItems() {
}
}
for _, subName := range i.Contexts {
if i.hub.GetItem(CONTEXTS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, CONTEXTS, i.Name)
}
}
for _, subName := range i.AppsecConfigs {
if i.hub.GetItem(APPSEC_CONFIGS, subName) == nil {
log.Errorf("can't find %s in %s, required by %s", subName, APPSEC_CONFIGS, i.Name)

2
pkg/cwhub/sync.go
View file

@ -346,7 +346,7 @@ func (i *Item) checkSubItemVersions() error {
// syncDir scans a directory for items, and updates the Hub state accordingly.
func (h *Hub) syncDir(dir string) error {
// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
// For each, scan PARSERS, POSTOVERFLOWS... and COLLECTIONS last
for _, scan := range ItemTypes {
// cpath: top-level item directory, either downloaded or installed items.
// i.e. /etc/crowdsec/parsers, /etc/crowdsec/hub/parsers, ...

2
test/bats.mk
View file

@ -83,7 +83,7 @@ bats-build: bats-environment
# Create a reusable package with initial configuration + data
bats-fixture: bats-check-requirements bats-update-tools
@echo "Creating functional test fixture..."
@echo "Creating functional test fixture."
@$(TEST_DIR)/instance-data make
# Remove the local crowdsec installation and the fixture config + data

21
test/bats/01_cscli.bats
View file

@ -252,19 +252,20 @@ teardown() {
@test "cscli - malformed LAPI url" {
LOCAL_API_CREDENTIALS=$(config_get '.api.client.credentials_path')
config_set "${LOCAL_API_CREDENTIALS}" '.url="https://127.0.0.1:-80"'
config_set "${LOCAL_API_CREDENTIALS}" '.url="http://127.0.0.1:-80"'
rune -1 cscli lapi status
assert_stderr --partial 'parsing api url'
assert_stderr --partial 'invalid port \":-80\" after host'
rune -1 cscli lapi status -o json
rune -0 jq -r '.msg' <(stderr)
assert_output 'parsing api url: parse "http://127.0.0.1:-80/": invalid port ":-80" after host'
}
rune -1 cscli alerts list
assert_stderr --partial 'parsing api url'
assert_stderr --partial 'invalid port \":-80\" after host'
@test "cscli - bad LAPI password" {
LOCAL_API_CREDENTIALS=$(config_get '.api.client.credentials_path')
config_set "${LOCAL_API_CREDENTIALS}" '.password="meh"'
rune -1 cscli decisions list
assert_stderr --partial 'parsing api url'
assert_stderr --partial 'invalid port \":-80\" after host'
rune -1 cscli lapi status -o json
rune -0 jq -r '.msg' <(stderr)
assert_output 'failed to authenticate to Local API (LAPI): API error: incorrect Username or Password'
}
@test "cscli metrics" {

95
test/bats/09_context.bats Normal file
View file

@ -0,0 +1,95 @@
#!/usr/bin/env bats
# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
set -u
setup_file() {
load "../lib/setup_file.sh"
CONFIG_DIR=$(config_get '.config_paths.config_dir')
export CONFIG_DIR
CONTEXT_YAML="$CONFIG_DIR/console/context.yaml"
export CONTEXT_YAML
}
teardown_file() {
load "../lib/teardown_file.sh"
}
setup() {
load "../lib/setup.sh"
load "../lib/bats-file/load.bash"
./instance-data load
config_set '.common.log_media="stdout"'
mkdir -p "$CONFIG_DIR/console"
}
teardown() {
./instance-crowdsec stop
}
#----------
@test "detect available context" {
rune -0 cscli lapi context detect -a
rune -0 yq -o json <(output)
assert_json '{"Acquisition":["evt.Line.Module","evt.Line.Raw","evt.Line.Src"]}'
rune -0 cscli parsers install crowdsecurity/dateparse-enrich
rune -0 cscli lapi context detect crowdsecurity/dateparse-enrich
rune -0 yq -o json '.crowdsecurity/dateparse-enrich' <(output)
assert_json '["evt.MarshaledTime","evt.Meta.timestamp"]'
}
@test "attempt to load from default context file, ignore if missing" {
rune -0 rm -f "$CONTEXT_YAML"
rune -0 "$CROWDSEC" -t --trace
assert_stderr --partial "loading console context from $CONTEXT_YAML"
}
@test "error if context file is explicitly set but does not exist" {
config_set ".crowdsec_service.console_context_path=strenv(CONTEXT_YAML)"
rune -0 rm -f "$CONTEXT_YAML"
rune -1 "$CROWDSEC" -t
assert_stderr --partial "while checking console_context_path: stat $CONTEXT_YAML: no such file or directory"
}
@test "context file is bad" {
echo "bad yaml" > "$CONTEXT_YAML"
rune -1 "$CROWDSEC" -t
assert_stderr --partial "while loading context: $CONTEXT_YAML: yaml: unmarshal errors"
}
@test "context file is good" {
echo '{"source_ip":["evt.Parsed.source_ip"]}' > "$CONTEXT_YAML"
rune -0 "$CROWDSEC" -t --debug
assert_stderr --partial 'console context to send: {"source_ip":["evt.Parsed.source_ip"]}'
}
@test "context file is from hub (local item)" {
mkdir -p "$CONFIG_DIR/contexts"
config_set "del(.crowdsec_service.console_context_path)"
echo '{"context":{"source_ip":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/foobar.yaml"
rune -0 "$CROWDSEC" -t --trace
assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/foobar.yaml"
assert_stderr --partial 'console context to send: {"source_ip":["evt.Parsed.source_ip"]}'
}
@test "merge multiple contexts" {
mkdir -p "$CONFIG_DIR/contexts"
echo '{"context":{"one":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/one.yaml"
echo '{"context":{"two":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/two.yaml"
rune -0 "$CROWDSEC" -t --trace
assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/one.yaml"
assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/two.yaml"
assert_stderr --partial 'console context to send: {"one":["evt.Parsed.source_ip"],"two":["evt.Parsed.source_ip"]}'
}
@test "merge contexts from hub and context.yaml file" {
mkdir -p "$CONFIG_DIR/contexts"
echo '{"context":{"one":["evt.Parsed.source_ip"]}}' > "$CONFIG_DIR/contexts/one.yaml"
echo '{"one":["evt.Parsed.source_ip_2"]}' > "$CONFIG_DIR/console/context.yaml"
rune -0 "$CROWDSEC" -t --trace
assert_stderr --partial "loading console context from $CONFIG_DIR/contexts/one.yaml"
assert_stderr --partial "loading console context from $CONFIG_DIR/console/context.yaml"
assert_stderr --partial 'console context to send: {"one":["evt.Parsed.source_ip","evt.Parsed.source_ip_2"]}'
}

15
test/bats/20_hub.bats
View file

@ -36,7 +36,7 @@ teardown() {
rune -0 cscli hub list
assert_output "No items to display"
rune -0 cscli hub list -o json
assert_json '{"appsec-configs":[],"appsec-rules":[],parsers:[],scenarios:[],collections:[],postoverflows:[]}'
assert_json '{"appsec-configs":[],"appsec-rules":[],parsers:[],scenarios:[],collections:[],contexts:[],postoverflows:[]}'
rune -0 cscli hub list -o raw
assert_output 'name,status,version,description,type'
@ -47,6 +47,7 @@ teardown() {
assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*SCENARIOS.*crowdsecurity/telnet-bf.*"
refute_output --partial 'POSTOVERFLOWS'
refute_output --partial 'COLLECTIONS'
rune -0 cscli hub list -o json
rune -0 jq -e '(.parsers | length == 1) and (.scenarios | length == 1)' <(output)
rune -0 cscli hub list -o raw
@ -55,8 +56,11 @@ teardown() {
refute_output --partial 'crowdsecurity/iptables'
# all items
mkdir -p "$CONFIG_DIR/contexts"
# there are no contexts yet, so we create a local one
touch "$CONFIG_DIR/contexts/mycontext.yaml"
rune -0 cscli hub list -a
assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*COLLECTIONS.*crowdsecurity/iptables.*"
assert_output --regexp ".*PARSERS.*crowdsecurity/whitelists.*POSTOVERFLOWS.*SCENARIOS.*crowdsecurity/telnet-bf.*CONTEXTS.*mycontext.yaml.*COLLECTIONS.*crowdsecurity/iptables.*"
rune -0 cscli hub list -a -o json
rune -0 jq -e '(.parsers | length > 1) and (.scenarios | length > 1)' <(output)
rune -0 cscli hub list -a -o raw
@ -107,6 +111,8 @@ teardown() {
assert_stderr --partial "Upgraded 0 postoverflows"
assert_stderr --partial "Upgrading scenarios"
assert_stderr --partial "Upgraded 0 scenarios"
assert_stderr --partial "Upgrading contexts"
assert_stderr --partial "Upgraded 0 contexts"
assert_stderr --partial "Upgrading collections"
assert_stderr --partial "Upgraded 0 collections"
@ -134,10 +140,11 @@ teardown() {
assert_line "parsers"
assert_line "postoverflows"
assert_line "scenarios"
assert_line "contexts"
assert_line "collections"
rune -0 cscli hub types -o human
rune -0 yq -o json <(output)
assert_json '["parsers","postoverflows","scenarios","appsec-configs","appsec-rules","collections"]'
assert_json '["parsers","postoverflows","scenarios","contexts","appsec-configs","appsec-rules","collections"]'
rune -0 cscli hub types -o json
assert_json '["parsers","postoverflows","scenarios","appsec-configs","appsec-rules","collections"]'
assert_json '["parsers","postoverflows","scenarios","contexts","appsec-configs","appsec-rules","collections"]'
}

6
test/lib/setup_file.sh
View file

@ -247,12 +247,14 @@ hub_purge_all() {
"$CONFIG_DIR"/collections/* \
"$CONFIG_DIR"/parsers/*/* \
"$CONFIG_DIR"/scenarios/* \
"$CONFIG_DIR"/postoverflows/*
"$CONFIG_DIR"/postoverflows/* \
"$CONFIG_DIR"/contexts/*
rm -rf \
"$CONFIG_DIR"/hub/collections/* \
"$CONFIG_DIR"/hub/parsers/*/* \
"$CONFIG_DIR"/hub/scenarios/* \
"$CONFIG_DIR"/hub/postoverflows/*
"$CONFIG_DIR"/hub/postoverflows/* \
"$CONFIG_DIR"/hub/contexts/*
local DATA_DIR
DATA_DIR=$(config_get .config_paths.data_dir)
# should remove everything except the db (find $DATA_DIR -not -name "crowdsec.db*" -delete),