From b6be18ca657197da0c6be6465c4909fd60cca0ed Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 6 Feb 2023 07:33:04 +0100 Subject: [PATCH] cscli setup (#1923) Detect running services and generate acquisition configuration --- cmd/crowdsec-cli/main.go | 4 + cmd/crowdsec-cli/setup.go | 312 +++++++ config/detect.yaml | 482 ++++++++++ go.mod | 17 +- go.sum | 37 +- pkg/setup/README.md | 340 ++++++++ pkg/setup/detect.go | 581 ++++++++++++ pkg/setup/detect_test.go | 1017 ++++++++++++++++++++++ pkg/setup/export_test.go | 9 + pkg/setup/install.go | 255 ++++++ pkg/setup/units.go | 59 ++ pkg/setup/units_test.go | 32 + tests/bats/07_setup.bats | 816 +++++++++++++++++ tests/bats/testdata/07_setup/detect.yaml | 88 ++ tests/lib/config/config-local | 3 + 15 files changed, 4040 insertions(+), 12 deletions(-) create mode 100644 cmd/crowdsec-cli/setup.go create mode 100644 config/detect.yaml create mode 100644 pkg/setup/README.md create mode 100644 pkg/setup/detect.go create mode 100644 pkg/setup/detect_test.go create mode 100644 pkg/setup/export_test.go create mode 100644 pkg/setup/install.go create mode 100644 pkg/setup/units.go create mode 100644 pkg/setup/units_test.go create mode 100644 tests/bats/07_setup.bats create mode 100644 tests/bats/testdata/07_setup/detect.yaml diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index e0955617a..c1ce6adf3 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -242,6 +242,10 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.AddCommand(NewNotificationsCmd()) rootCmd.AddCommand(NewSupportCmd()) + if fflag.CscliSetup.IsEnabled() { + rootCmd.AddCommand(NewSetupCmd()) + } + if err := rootCmd.Execute(); err != nil { if bincoverTesting != "" { log.Debug("coverage report is enabled") diff --git a/cmd/crowdsec-cli/setup.go b/cmd/crowdsec-cli/setup.go new file mode 100644 index 000000000..7f1da4c44 --- /dev/null +++ b/cmd/crowdsec-cli/setup.go @@ -0,0 +1,312 @@ +package main + +import ( + "bytes" + "fmt" + "os" + "os/exec" + + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "gopkg.in/yaml.v3" + goccyyaml "github.com/goccy/go-yaml" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/setup" +) + +// NewSetupCmd defines the "cscli setup" command. +func NewSetupCmd() *cobra.Command { + cmdSetup := &cobra.Command{ + Use: "setup", + Short: "Tools to configure crowdsec", + Long: "Manage hub configuration and service detection", + Args: cobra.MinimumNArgs(0), + DisableAutoGenTag: true, + } + + // + // cscli setup detect + // + { + cmdSetupDetect := &cobra.Command{ + Use: "detect", + Short: "detect running services, generate a setup file", + DisableAutoGenTag: true, + RunE: runSetupDetect, + } + + defaultServiceDetect := csconfig.DefaultConfigPath("hub", "detect.yaml") + + flags := cmdSetupDetect.Flags() + flags.String("detect-config", defaultServiceDetect, "path to service detection configuration") + flags.Bool("list-supported-services", false, "do not detect; only print supported services") + flags.StringSlice("force-unit", nil, "force detection of a systemd unit (can be repeated)") + flags.StringSlice("force-process", nil, "force detection of a running process (can be repeated)") + flags.StringSlice("skip-service", nil, "ignore a service, don't recommend hub/datasources (can be repeated)") + flags.String("force-os-family", "", "override OS.Family: one of linux, freebsd, windows or darwin") + flags.String("force-os-id", "", "override OS.ID=[debian | ubuntu | , redhat...]") + flags.String("force-os-version", "", "override OS.RawVersion (of OS or Linux distribution)") + flags.Bool("snub-systemd", false, "don't use systemd, even if available") + flags.Bool("yaml", false, "output yaml, not json") + cmdSetup.AddCommand(cmdSetupDetect) + } + + // + // cscli setup install-hub + // + { + cmdSetupInstallHub := &cobra.Command{ + Use: "install-hub [setup_file] [flags]", + Short: "install items from a setup file", + Args: cobra.ExactArgs(1), + DisableAutoGenTag: true, + RunE: runSetupInstallHub, + } + + flags := cmdSetupInstallHub.Flags() + flags.Bool("dry-run", false, "don't install anything; print out what would have been") + cmdSetup.AddCommand(cmdSetupInstallHub) + } + + // + // cscli setup datasources + // + { + cmdSetupDataSources := &cobra.Command{ + Use: "datasources [setup_file] [flags]", + Short: "generate datasource (acquisition) configuration from a setup file", + Args: cobra.ExactArgs(1), + DisableAutoGenTag: true, + RunE: runSetupDataSources, + } + + flags := cmdSetupDataSources.Flags() + flags.String("to-dir", "", "write the configuration to a directory, in multiple files") + cmdSetup.AddCommand(cmdSetupDataSources) + } + + // + // cscli setup validate + // + { + cmdSetupValidate := &cobra.Command{ + Use: "validate [setup_file]", + Short: "validate a setup file", + Args: cobra.ExactArgs(1), + DisableAutoGenTag: true, + RunE: runSetupValidate, + } + + cmdSetup.AddCommand(cmdSetupValidate) + } + + return cmdSetup +} + +func runSetupDetect(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + detectConfigFile, err := flags.GetString("detect-config") + if err != nil { + return err + } + + listSupportedServices, err := flags.GetBool("list-supported-services") + if err != nil { + return err + } + + forcedUnits, err := flags.GetStringSlice("force-unit") + if err != nil { + return err + } + + forcedProcesses, err := flags.GetStringSlice("force-process") + if err != nil { + return err + } + + forcedOSFamily, err := flags.GetString("force-os-family") + if err != nil { + return err + } + + forcedOSID, err := flags.GetString("force-os-id") + if err != nil { + return err + } + + forcedOSVersion, err := flags.GetString("force-os-version") + if err != nil { + return err + } + + skipServices, err := flags.GetStringSlice("skip-service") + if err != nil { + return err + } + + snubSystemd, err := flags.GetBool("snub-systemd") + if err != nil { + return err + } + + if !snubSystemd { + _, err := exec.LookPath("systemctl") + if err != nil { + log.Debug("systemctl not available: snubbing systemd") + snubSystemd = true + } + } + + outYaml, err := flags.GetBool("yaml") + if err != nil { + return err + } + + if forcedOSFamily == "" && forcedOSID != "" { + log.Debug("force-os-id is set: force-os-family defaults to 'linux'") + forcedOSFamily = "linux" + } + + if listSupportedServices { + supported, err := setup.ListSupported(detectConfigFile) + if err != nil { + return err + } + + for _, svc := range supported { + fmt.Println(svc) + } + + return nil + } + + opts := setup.DetectOptions{ + ForcedUnits: forcedUnits, + ForcedProcesses: forcedProcesses, + ForcedOS: setup.ExprOS{ + Family: forcedOSFamily, + ID: forcedOSID, + RawVersion: forcedOSVersion, + }, + SkipServices: skipServices, + SnubSystemd: snubSystemd, + } + + hubSetup, err := setup.Detect(detectConfigFile, opts) + if err != nil { + return fmt.Errorf("detecting services: %w", err) + } + + setup, err := setupAsString(hubSetup, outYaml) + if err != nil { + return err + } + fmt.Println(setup) + + return nil +} + +func setupAsString(cs setup.Setup, outYaml bool) (string, error) { + var ( + ret []byte + err error + ) + + wrap := func(err error) error { + return fmt.Errorf("while marshaling setup: %w", err) + } + + indentLevel := 2 + buf := &bytes.Buffer{} + enc := yaml.NewEncoder(buf) + enc.SetIndent(indentLevel) + + if err = enc.Encode(cs); err != nil { + return "", wrap(err) + } + + if err = enc.Close(); err != nil { + return "", wrap(err) + } + + ret = buf.Bytes() + + if !outYaml { + // take a general approach to output json, so we avoid the + // double tags in the structures and can use go-yaml features + // missing from the json package + ret, err = goccyyaml.YAMLToJSON(ret) + if err != nil { + return "", wrap(err) + } + } + + return string(ret), nil +} + +func runSetupDataSources(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + fromFile := args[0] + + toDir, err := flags.GetString("to-dir") + if err != nil { + return err + } + + input, err := os.ReadFile(fromFile) + if err != nil { + return fmt.Errorf("while reading setup file: %w", err) + } + + output, err := setup.DataSources(input, toDir) + if err != nil { + return err + } + + if toDir == "" { + fmt.Println(output) + } + + return nil +} + +func runSetupInstallHub(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + fromFile := args[0] + + dryRun, err := flags.GetBool("dry-run") + if err != nil { + return err + } + + input, err := os.ReadFile(fromFile) + if err != nil { + return fmt.Errorf("while reading file %s: %w", fromFile, err) + } + + if err = setup.InstallHubItems(csConfig, input, dryRun); err != nil { + return err + } + + return nil +} + +func runSetupValidate(cmd *cobra.Command, args []string) error { + fromFile := args[0] + input, err := os.ReadFile(fromFile) + if err != nil { + return fmt.Errorf("while reading stdin: %w", err) + } + + if err = setup.Validate(input); err != nil { + fmt.Printf("%v\n", err) + return fmt.Errorf("invalid setup file") + } + + return nil +} diff --git a/config/detect.yaml b/config/detect.yaml new file mode 100644 index 000000000..46a851b81 --- /dev/null +++ b/config/detect.yaml @@ -0,0 +1,482 @@ +--- +version: 1.0 + +# TODO: This file must be reviewed before the `cscli setup` command becomes GA + +detect: + + # + # crowdsecurity/apache2 + # + + # XXX some distro is using this path? + # - /var/log/*http*/*.log + + apache2-systemd-deb: + when: + - UnitFound("apache2.service") + - PathExists("/etc/debian_version") + install: + collections: + - crowdsecurity/apache2 + datasource: + source: file + filenames: + - /var/log/apache2/*.log + labels: + type: apache2 + + apache2-systemd-rpm: + when: + - UnitFound("httpd.service") + - PathExists("/etc/redhat-release") + install: + collections: + - crowdsecurity/apache2 + datasource: + source: file + filenames: + - /var/log/httpd/*.log + # XXX /var/log/*http*/*.log + labels: + type: apache2 + + # + # crowdsecurity/asterisk + # + + asterisk-systemd: + when: + - UnitFound("asterisk.service") + install: + collections: + - crowdsecurity/asterisk + datasource: + source: file + labels: + type: asterisk + filenames: + - /var/log/asterisk/*.log + + # + # crowdsecurity/caddy + # + + caddy-systemd: + when: + - UnitFound("caddy.service") + install: + collections: + - crowdsecurity/caddy + datasource: + source: file + labels: + type: caddy + filenames: + - /var/log/caddy/*.log + + # + # crowdsecurity/dovecot + # + + dovecot-systemd: + when: + - UnitFound("dovecot.service") + install: + collections: + - crowdsecurity/dovecot + datasource: + source: file + labels: + type: syslog + filenames: + - /var/log/mail.log + + # + # LePresidente/emby + # + + emby-systemd: + when: + - UnitFound("emby-server.service") + install: + collections: + - LePresidente/emby + datasource: + source: file + labels: + type: emby + filenames: + - /var/log/embyserver.txt + + # + # crowdsecurity/endlessh + # + + endlessh-systemd: + when: + - UnitFound("endlessh.service") + install: + collections: + - crowdsecurity/endlessh + datasource: + source: journalctl + labels: + type: syslog + # XXX this? or /var/log/syslog? + journalctl_filter: + - "_SYSTEMD_UNIT=endlessh.service" + + # + # crowdsecurity/gitea + # + + # XXX untested + + gitea-systemd: + when: + - UnitFound("gitea.service") + install: + collections: + - crowdsecurity/gitea + datasource: + source: file + labels: + type: gitea + filenames: + - /var/log/gitea.log + + # + # crowdsecurity/haproxy + # + + haproxy-systemd: + when: + - UnitFound("haproxy.service") + install: + collections: + - crowdsecurity/haproxy + datasource: + source: file + labels: + type: haproxy + filenames: + - /var/log/haproxy/*.log + + # + # firewallservices/lemonldap-ng + # + + lemonldap-ng-systemd: + when: + - UnitFound("lemonldap-ng-fastcgi-server.service") + install: + collections: + - firewallservices/lemonldap-ng + #datasource: + # # XXX todo where are the logs? + # labels: + # type: syslog + + # + # crowdsecurity/mariadb + # + + mariadb-systemd: + when: + - UnitFound("mariadb.service") + install: + collections: + - crowdsecurity/mariadb + datasource: + source: file + labels: + type: mysql + filenames: + - /var/log/mysql/error.log + + # + # crowdsecurity/mysql + # + + mysql-systemd: + when: + - UnitFound("mysql.service") + install: + collections: + - crowdsecurity/mysql + datasource: + source: file + labels: + type: mysql + filenames: + - /var/log/mysql/error.log + + # + # crowdsecurity/nginx + # + + nginx-systemd: + when: + - UnitFound("nginx.service") + install: + collections: + - crowdsecurity/nginx + datasource: + source: file + labels: + type: nginx + filenames: + - /var/log/nginx/*.log + + openresty-systemd: + when: + - UnitFound("openresty.service") + install: + collections: + - crowdsecurity/nginx + datasource: + source: file + labels: + type: nginx + filenames: + - /usr/local/openresty/nginx/logs/*.log + + # + # crowdsecurity/odoo + # + + odoo-systemd: + when: + - UnitFound("odoo.service") + install: + collections: + - crowdsecurity/odoo + datasource: + source: file + labels: + type: odoo + filenames: + - /var/log/odoo/*.log + + # + # LePresidente/ombi + # + + # This only works on deb-based systems. On other distributions, the + # application is run from the release tarball and the log location depends on + # the location it's run from. + + ombi-systemd: + when: + - UnitFound("ombi.service") + - PathExists("/etc/debian_version") + install: + collections: + - LePresidente/ombi + datasource: + source: file + labels: + type: ombi + filenames: + - /var/log/ombi/log-*.txt + + # + # crowdsecurity/pgsql + # + + pgsql-systemd-deb: + when: + - UnitFound("postgresql.service") + - PathExists("/etc/debian_version") + install: + collections: + - crowdsecurity/pgsql + datasource: + source: file + labels: + type: postgres + filenames: + - /var/log/postgresql/*.log + + pgsql-systemd-rpm: + when: + - UnitFound("postgresql.service") + - PathExists("/etc/redhat-release") + install: + collections: + - crowdsecurity/pgsql + datasource: + source: file + labels: + type: postgres + filenames: + - /var/lib/pgsql/data/log/*.log + + # + # crowdsecurity/postfix + # + + postfix-systemd: + when: + - UnitFound("postfix.service") + install: + collections: + - crowdsecurity/postfix + datasource: + source: file + labels: + type: syslog + filenames: + - /var/log/mail.log + + # + # crowdsecurity/proftpd + # + + proftpd-systemd: + when: + - UnitFound("proftpd.service") + install: + collections: + - crowdsecurity/proftpd + datasource: + source: file + labels: + type: proftpd + filenames: + - /var/log/proftpd/*.log + + # + # fulljackz/pureftpd + # + + pureftpd-systemd: + when: + - UnitFound("pure-ftpd.service") + install: + collections: + - fulljackz/pureftpd + # XXX ? + datasource: + source: file + labels: + type: syslog + filenames: + - /var/log/pure-ftpd/*.log + + # + # crowdsecurity/smb + # + + smb-systemd: + when: + # deb -> smbd.service + # rpm -> smb.service + - UnitFound("smbd.service") or UnitFound("smb.service") + install: + collections: + - crowdsecurity/smb + datasource: + source: file + labels: + type: smb + filenames: + - /var/log/samba*.log + + # + # crowdsecurity/sshd + # + + sshd-systemd: + when: + # deb -> ssh.service + # rpm -> sshd.service + - UnitFound("ssh.service") or UnitFound("sshd.service") or UnitFound("ssh.socket") or UnitFound("sshd.socket") + install: + collections: + - crowdsecurity/sshd + datasource: + source: file + labels: + type: syslog + filenames: + - /var/log/auth.log + - /var/log/sshd.log + - /var/log/secure + + # + # crowdsecurity/suricata + # + + suricata-systemd: + when: + - UnitFound("suricata.service") + install: + collections: + - crowdsecurity/suricata + datasource: + source: file + labels: + type: suricata-evelogs + filenames: + - /var/log/suricata/eve.json + + # + # crowdsecurity/vsftpd + # + + vsftpd-systemd: + when: + - UnitFound("vsftpd.service") + install: + collections: + - crowdsecurity/vsftpd + datasource: + source: file + labels: + type: vsftpd + filenames: + - /var/log/vsftpd/*.log + + # + # Operating Systems + # + + linux: + when: + - OS.Family == "linux" + install: + collections: + - crowdsecurity/linux + datasource: + source: file + labels: + type: syslog + filenames: + - /var/log/syslog + - /var/log/kern.log + - /var/log/messages + + freebsd: + when: + - OS.Family == "freebsd" + install: + collections: + - crowdsecurity/freebsd + + windows: + when: + - OS.Family == "windows" + install: + collections: + - crowdsecurity/windows + + # + # anti-lockout + # + + whitelists: + install: + parsers: + - crowdsecurity/whitelists diff --git a/go.mod b/go.mod index 202e5fcce..c18fee27d 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,7 @@ require ( github.com/r3labs/diff/v2 v2.14.1 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.5.0 - github.com/stretchr/testify v1.8.0 + github.com/stretchr/testify v1.8.1 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 google.golang.org/grpc v1.47.0 @@ -65,6 +65,7 @@ require ( ) require ( + github.com/Masterminds/semver v1.5.0 github.com/Masterminds/sprig/v3 v3.2.2 github.com/aquasecurity/table v1.8.0 github.com/beevik/etree v1.1.0 @@ -75,11 +76,14 @@ require ( github.com/golang-jwt/jwt/v4 v4.2.0 github.com/google/winops v0.0.0-20211216095627-f0e86eb1453b github.com/ivanpirog/coloredcobra v1.0.1 + github.com/lithammer/dedent v1.1.0 github.com/mattn/go-isatty v0.0.14 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/segmentio/kafka-go v0.4.34 + github.com/shirou/gopsutil/v3 v3.22.12 github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c - golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f + golang.org/x/sys v0.3.0 + gopkg.in/yaml.v3 v3.0.1 k8s.io/apiserver v0.22.5 ) @@ -101,6 +105,7 @@ require ( github.com/docker/go-units v0.4.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-logr/logr v1.2.3 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/analysis v0.19.16 // indirect github.com/go-openapi/inflect v0.19.0 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -115,7 +120,7 @@ require ( github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v0.0.0-20210429001901-424d2337a529 // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/google/go-cmp v0.5.8 // indirect + github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/gorilla/mux v1.7.3 // indirect github.com/hashicorp/hcl/v2 v2.13.0 // indirect @@ -136,6 +141,7 @@ require ( github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.15.7 // indirect github.com/leodido/go-urn v1.2.1 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-colorable v0.1.12 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect @@ -154,6 +160,7 @@ require ( github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect github.com/rivo/uniseg v0.2.0 // indirect @@ -163,9 +170,12 @@ require ( github.com/spf13/cast v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tidwall/gjson v1.13.0 // indirect + github.com/tklauser/go-sysconf v0.3.11 // indirect + github.com/tklauser/numcpus v0.6.0 // indirect github.com/ugorji/go/codec v1.2.6 // indirect github.com/vjeantet/grok v1.0.1 // indirect github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect + github.com/yusufpapurcu/wmi v1.2.2 // indirect github.com/zclconf/go-cty v1.8.0 // indirect go.mongodb.org/mongo-driver v1.9.0 // indirect golang.org/x/net v0.0.0-20220722155237-a158d28d115b // indirect @@ -177,7 +187,6 @@ require ( google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/api v0.25.2 // indirect k8s.io/apimachinery v0.25.2 // indirect k8s.io/klog/v2 v2.70.1 // indirect diff --git a/go.sum b/go.sum index 3fdb79b98..bdacfc4b8 100644 --- a/go.sum +++ b/go.sum @@ -57,6 +57,8 @@ github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= +github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= +github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= @@ -120,8 +122,6 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84= github.com/blackfireio/osinfo v1.0.3 h1:Yk2t2GTPjBcESv6nDSWZKO87bGMQgO+Hi9OoXPpxX8c= github.com/blackfireio/osinfo v1.0.3/go.mod h1:Pd987poVNmd5Wsx6PRPw4+w7kLlf9iJxoRKPtPAjOrA= -github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= -github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bluele/gcache v0.0.2 h1:WcbfdXICg7G/DGBh1PFfcirkWOQV+v077yF1pSy3DGw= github.com/bluele/gcache v0.0.2/go.mod h1:m15KV+ECjptwSPxKhOhQoAFQVtUFjTVkc3H8o0t/fp0= @@ -251,6 +251,8 @@ github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTg github.com/go-logr/logr v1.2.0/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-openapi/analysis v0.0.0-20180825180245-b006789cd277/go.mod h1:k70tL6pCuVxPJOHXQ+wIac1FUrvNkHolPie/cLEU6hI= github.com/go-openapi/analysis v0.17.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= github.com/go-openapi/analysis v0.18.0/go.mod h1:IowGgpVeD0vNm45So8nr+IcQ3pxVtpRoBWb8PVZO0ik= @@ -438,8 +440,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg= -github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -641,8 +643,12 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY= +github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc= github.com/lucasb-eyer/go-colorful v1.0.2/go.mod h1:0MS4r+7BZKSJ5mw4/S5MPN+qHFF1fYclkSPilDOKW0s= github.com/lucasb-eyer/go-colorful v1.0.3/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -770,6 +776,8 @@ github.com/pmezard/go-difflib v0.0.0-20151028094244-d8ed2627bdf0/go.mod h1:iKH77 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -836,6 +844,8 @@ github.com/segmentio/kafka-go v0.4.34 h1:Dm6YlLMiVSiwwav20KY0AoY63s661FXevwJ3CVH github.com/segmentio/kafka-go v0.4.34/go.mod h1:GAjxBQJdQMB5zfNA21AhpaqOB2Mu+w3De4ni3Gbm8y0= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs= +github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -874,8 +884,9 @@ github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= -github.com/stretchr/objx v0.4.0 h1:M2gUjqZET1qApGOWNSnZ49BAIMX4F/1plDv3+l31EJ4= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v0.0.0-20161117074351-18a02ba4a312/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -885,8 +896,9 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c h1:HelZ2kAFadG0La9d+4htN4HzQ68Bm2iM9qKMSMES6xg= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c/go.mod h1:JlzghshsemAMDGZLytTFY8C1JQxQPhnatWqNwUXjggo= @@ -898,6 +910,10 @@ github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JT github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= +github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= +github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= +github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20201229170055-e5319fda7802/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= @@ -928,6 +944,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= +github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zclconf/go-cty v1.8.0 h1:s4AvqaeQzJIu3ndv4gVIhplVD0krU+bgrcLSVUnaWuA= github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= @@ -1136,6 +1154,7 @@ golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1167,6 +1186,7 @@ golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1186,8 +1206,9 @@ golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f h1:v4INt8xihDGvnrfjMDVXGxw9wrfxYyCjk0KbXjhR55s= -golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0 h1:w8ZOecv6NaNa/zC8944JTU3vz4u6Lagfk4RPQxv92NQ= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/pkg/setup/README.md b/pkg/setup/README.md new file mode 100644 index 000000000..3585ee8b1 --- /dev/null +++ b/pkg/setup/README.md @@ -0,0 +1,340 @@ + +> **_NOTE_**: The following document describes an experimental, work-in-progress feature. To enable the `cscli setup` command, set the environment variable `CROWDSEC_FEATURE_CSCLI_SETUP=true` or add the line " - cscli_setup" to `/etc/crowdsec/feature.yaml`. Any feedback is welcome. + +--- + +# cscli setup + +The "cscli setup" command can configure a crowdsec instance based on the services that are installed or running on the server. + +There are three main subcommands: + +- `cscli setup detect`: *detect* the services, the OS family, version or the Linux distribution +- `cscli setup install-hub`: *install* the recommended collections, parsers, etc. based on the detection result +- `cscli setup datasources`: *generate* the appropriate acquisition rules + +The setup command is used in the `wizard.sh` script, but can also be invoked by hand or customized via a configuration file +by adding new services, log locations and detection rules. + +Detection and installation are performed as separate steps, as you can see in the following diagram: + +``` + +-------------+ + | | + | detect.yaml | + | | + +-------------+ + | + v + setup detect + | + v + +--------------+ + | +---> setup install-hub +-----------------------+ + | setup.yaml | | | + | +---> setup datasources --->| etc/crowdsec/acquis.d | + +--------------+ | | + +-----------------------+ +``` + +You can inspect and customize the intermediary file (`setup.yaml`), which is useful +in case of many instances, deployment automation or unusual setups. + +A subcommand can be used to check your changes in this case: + +- `cscli setup validate`: *validate* or report errors on a setup file + +## Basic usage + +Identify the existing services and write out what was detected: + +```console +# cscli setup detect > setup.yaml +``` + +See what was found. + +```console +# cscli setup install-hub setup.yaml --dry-run +dry-run: would install collection crowdsecurity/apache2 +dry-run: would install collection crowdsecurity/linux +dry-run: would install collection crowdsecurity/pgsql +dry-run: would install parser crowdsecurity/whitelists +``` + +Install the objects (parsers, scenarios...) required to support the detected services: + +```console +# cscli setup install-hub setup.yaml +INFO[29-06-2022 03:16:14 PM] crowdsecurity/apache2-logs : OK +INFO[29-06-2022 03:16:14 PM] Enabled parsers : crowdsecurity/apache2-logs +INFO[29-06-2022 03:16:14 PM] crowdsecurity/http-logs : OK +[...] +INFO[29-06-2022 03:16:18 PM] Enabled crowdsecurity/linux +``` + +Generate the datasource configuration: + +```console +# cscli setup datasources setup.yaml --to-dir /etc/crowdsec/acquis.d +``` + +With the above command, each detected service gets a corresponding file in the +`acquis.d` directory. Running `cscli setup` again may add more services as they +are detected, but datasource files or hub items are never removed +automatically. + + +## The detect.yaml file + +A detect.yaml file is downloaded when you first install crowdsec, and is updated by the `cscli hub update` +command. + +> **_NOTE_**: XXX XXX - this is currently not the case, the file is distributed in the crowdsec repository, but it should change. + +You can see the default location with `cscli setup detect --help | grep detect-config` + +The YAML file contains a version number (always 1.0) and a list of sections, one per supported service. + +Each service defines its detection rules, the recommended hub items and +recommended datasources. The same software can be defined in multiple service +sections: for example, apache on debian and fedora have different detection +rules and different datasources so it requires two sections to support both platforms. + +The following are minimal `detect.yaml` examples just to show a few concepts. + +```yaml +version: 1.0 + +services: + + apache2: + when: + - ProcessRunning("apache2") + install: + collections: + - crowdsecurity/apache2 + datasources: + source: file + labels: + type: apache2 + filenames: + - /var/log/apache2/*.log + - /var/log/httpd/*.log +``` + + +- `ProcessRunning()` matches the process name of a running application. The +`when:` clause can contain any number of expressions, they are all evaluated +and must all return true for a service to be detected (implied *and* clause, no +short-circuit). A missing or empty `when:` section is evaluated as true. +The [expression +engine](https://github.com/antonmedv/expr/blob/master/docs/Language-Definition.md) +is the same one used by CrowdSec parser filters. You can force the detection of +a process by using the `cscli setup detect... --force-process ` +flag. It will always behave as if `` was running. + +The `install:` section can contain any number of collections, parsers, scenarios +and postoverflows. In practices, it's most often a single collection. + +The `datasource:` section is copied as-is in the acquisition file. + +> **_NOTE_**: XXX TODO - the current version does not validate the `datasource:` mapping. Bad content is written to acquis.d until crowdsec chokes on it. + +Detecting a running process may seem a good idea, but if a process manager like +systemd is available it's better to ask it for the information we want. + + +```yaml +version: 1.0 + +services: + + apache2-systemd: + when: + - UnitFound("apache2.service") + - OS.ID != "centos" + install: + collections: + - crowdsecurity/apache2 + datasource: + source: file + labels: + type: syslog + filenames: + - /var/log/apache2/*.log + + apache2-systemd-centos: + when: + - UnitFound("httpd.service") + - OS.ID == "centos" + install: + collections: + - crowdsecurity/apache2 + datasource: + source: file + labels: + type: syslog + filenames: + - /var/log/httpd/*.log +``` + +Here we see two more detection methods: + +- `UnitFound()` matches the name of systemd units, if the are in state enabled, + generated or static. You can see here that CentOS is using a different unit + name for Apache so it must have its own service section. You can force the + detection of a unit by using the `cscli setup detect... --force-unit ` flag. + +- OS.Family, OS.ID and OS.RawVersion are read from /etc/os-release in case of + Linux, and detected by other methods for FreeBSD and Windows. Under FreeBSD + and Windows, the value of OS.ID is the same as OS.Family. If OS detection + fails, it can be overridden with the flags `--force-os-family`, `--force-os-id` + and `--force-os-version`. + +If you want to ignore one or more services (i.e. not install anything and not +generate acquisition rules) you can specify it with `cscli setup detect... +--skip-service `. For example, `--skip-service apache2-systemd`. +If you want to disable systemd unit detection, use `cscli setup detect... --snub-systemd`. + +If you used the `--force-process` or `--force-unit` flags, but none of the +defined services is looking for them, you'll have an error like "detecting +services: process(es) forced but not supported". + +> **_NOTE_**: XXX XXX - having an error for this is maybe too much, but can tell that a configuration is outdated. Could this be a warning with optional flag to make it an error? + +We used the `OS.ID` value to check for the linux distribution, but since the same configuration +is required for CentOS and the other RedHat derivatives, it's better to check for the existence +of a file that is known to exist in all of them: + +```yaml +version: 1.0 + +services: + + apache2-systemd-deb: + when: + - UnitFound("apache2.service") + - PathExists("/etc/debian_version") + install: + # [...] + + apache2-systemd-rpm: + when: + - UnitFound("httpd.service") + - PathExists("/etc/redhat-release") + install: + # [...] +``` + +- `PathExists()` evaluates to true if a file, directory or link exists at the + given path. It does not check for broken links. + + + +Rules can be used to detect operating systems and environments: + +```yaml +version: 1.0 + +services: + + linux: + when: + - OS.Family == "linux" + install: + collections: + - crowdsecurity/linux + datasource: + type: file + labels: + type: syslog + log_files: + - /var/log/syslog + - /var/log/kern.log + - /var/log/messages + + freebsd: + when: + - OS.Family == "freebsd" + install: + collections: + - crowdsecurity/freebsd + + windows: + when: + - OS.Family == "windows" + install: + collections: + - crowdsecurity/windows +``` + +The OS object contains a methods to check for version numbers: +`OS.VersionCheck("")`. It uses the +[Masterminds/semver](https://github.com/Masterminds/semver) package and accepts +a variety of operators. + +Instead of: OS.RawVersion == "1.2.3" you should use `OS.VersionCheck("~1")`, +`OS.VersionCheck("~1.2")` depending if you want to match the major or the minor +version. It's unlikely that you need to match the exact patch level. + +Leading zeroes are permitted, to allow comparison of Ubuntu versions: strict semver rules would treat "22.04" as invalid. + + +# The `setup.yaml` file + +This file does not actually have a specific name, as it's usually written to standard output. + +For example, on a Debian system running Apache under systemd you can execute: + +```console +$ cscli setup detect --yaml +setup: + - detected_service: apache2-systemd-deb + install: + collections: + - crowdsecurity/apache2 + datasource: + filenames: + - /var/log/apache2/*.log + labels: + type: apache2 + - detected_service: linux + install: + collections: + - crowdsecurity/linux + datasource: + filenames: + - /var/log/syslog + - /var/log/kern.log + - /var/log/messages + labels: + type: syslog + - detected_service: whitelists + install: + parsers: + - crowdsecurity/whitelists +``` + +The default output format is JSON, which is compatible with YAML but less readable to humans. + + - `detected_service`: used to generate a name for the files written to `acquis.d` + - `install`: can contain collections, parsers, scenarios, postoverflows + - `datasource`: copied to `acquis.d` + + +```console +$ cscli setup datasources --help +generate datasource (acquisition) configuration from a setup file + +Usage: + cscli setup datasources [setup_file] [flags] + +Flags: + -h, --help help for datasources + --to-dir string write the configuration to a directory, in multiple files +[...] +``` + +If the `--to-dir` option is not specified, a single monolithic `acquis.yaml` is printed to the standard output. + diff --git a/pkg/setup/detect.go b/pkg/setup/detect.go new file mode 100644 index 000000000..957f70a25 --- /dev/null +++ b/pkg/setup/detect.go @@ -0,0 +1,581 @@ +package setup + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "sort" + + "github.com/Masterminds/semver" + "github.com/antonmedv/expr" + "github.com/blackfireio/osinfo" + "github.com/shirou/gopsutil/v3/process" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v3" + // goccyyaml "github.com/goccy/go-yaml" + + // "github.com/k0kubun/pp" + + "github.com/crowdsecurity/crowdsec/pkg/acquisition" + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" +) + +// ExecCommand can be replaced with a mock during tests. +var ExecCommand = exec.Command + +// HubItems contains the objects that are recommended to support a service. +type HubItems struct { + Collections []string `yaml:"collections,omitempty"` + Parsers []string `yaml:"parsers,omitempty"` + Scenarios []string `yaml:"scenarios,omitempty"` + PostOverflows []string `yaml:"postoverflows,omitempty"` +} + +type DataSourceItem map[string]interface{} + +// ServiceSetup describes the recommendations (hub objects and datasources) for a detected service. +type ServiceSetup struct { + DetectedService string `yaml:"detected_service"` + Install *HubItems `yaml:"install,omitempty"` + DataSource DataSourceItem `yaml:"datasource,omitempty"` +} + +// Setup is a container for a list of ServiceSetup objects, allowing for future extensions. +type Setup struct { + Setup []ServiceSetup `yaml:"setup"` +} + +func validateDataSource(opaqueDS DataSourceItem) error { + if len(opaqueDS) == 0 { + // empty datasource is valid + return nil + } + + + // formally validate YAML + + commonDS := configuration.DataSourceCommonCfg{} + body, err := yaml.Marshal(opaqueDS) + if err != nil { + return err + } + + err = yaml.Unmarshal(body, &commonDS) + if err != nil { + return err + } + + // source is mandatory // XXX unless it's not? + + if commonDS.Source == "" { + return fmt.Errorf("source is empty") + } + + + // source must be known + + ds := acquisition.GetDataSourceIface(commonDS.Source) + if ds == nil { + return fmt.Errorf("unknown source '%s'", commonDS.Source) + } + + // unmarshal and validate the rest with the specific implementation + + err = ds.UnmarshalConfig(body) + if err != nil { + return err + } + + // pp.Println(ds) + return nil +} + +func readDetectConfig(file string) (DetectConfig, error) { + var dc DetectConfig + + yamlBytes, err := os.ReadFile(file) + if err != nil { + return DetectConfig{}, fmt.Errorf("while reading file: %w", err) + } + + dec := yaml.NewDecoder(bytes.NewBuffer(yamlBytes)) + dec.KnownFields(true) + + if err = dec.Decode(&dc); err != nil { + return DetectConfig{}, fmt.Errorf("while parsing %s: %w", file, err) + } + + switch dc.Version { + case "": + return DetectConfig{}, fmt.Errorf("missing version tag (must be 1.0)") + case "1.0": + // all is well + default: + return DetectConfig{}, fmt.Errorf("unsupported version tag '%s' (must be 1.0)", dc.Version) + } + + for name, svc := range dc.Detect { + err = validateDataSource(svc.DataSource) + if err != nil { + return DetectConfig{}, fmt.Errorf("invalid datasource for %s: %w", name, err) + } + } + + return dc, nil +} + +// Service describes the rules for detecting a service and its recommended items. +type Service struct { + When []string `yaml:"when"` + Install *HubItems `yaml:"install,omitempty"` + DataSource DataSourceItem `yaml:"datasource,omitempty"` + // AcquisYAML []byte +} + +// DetectConfig is the container of all detection rules (detect.yaml). +type DetectConfig struct { + Version string `yaml:"version"` + Detect map[string]Service `yaml:"detect"` +} + +// ExprState keeps a global state for the duration of the service detection (cache etc.) +type ExprState struct { + unitsSearched map[string]bool + detectOptions DetectOptions + + // cache + installedUnits map[string]bool + // true if the list of running processes has already been retrieved, we can + // avoid getting it a second time. + processesSearched map[string]bool + // cache + runningProcesses map[string]bool +} + +// ExprServiceState keep a local state during the detection of a single service. It is reset before each service rules' evaluation. +type ExprServiceState struct { + detectedUnits []string +} + +// ExprOS contains the detected (or forced) OS fields available to the rule engine. +type ExprOS struct { + Family string + ID string + RawVersion string +} + +// This is not required with Masterminds/semver +/* +// normalizeVersion strips leading zeroes from each part, to allow comparison of ubuntu-like versions. +func normalizeVersion(version string) string { + // if it doesn't match a version string, return unchanged + if ok := regexp.MustCompile(`^(\d+)(\.\d+)?(\.\d+)?$`).MatchString(version); !ok { + // definitely not an ubuntu-like version, return unchanged + return version + } + + ret := []rune{} + + var cur rune + + trim := true + for _, next := range version + "." { + if trim && cur == '0' && next != '.' { + cur = next + + continue + } + + if cur != 0 { + ret = append(ret, cur) + } + + trim = (cur == '.' || cur == 0) + cur = next + } + + return string(ret) +} +*/ + +// VersionCheck returns true if the version of the OS matches the given constraint +func (os ExprOS) VersionCheck(constraint string) (bool, error) { + v, err := semver.NewVersion(os.RawVersion) + if err != nil { + return false, err + } + + c, err := semver.NewConstraint(constraint) + if err != nil { + return false, err + } + + return c.Check(v), nil +} + +// VersionAtLeast returns true if the version of the OS is at least the given version. +func (os ExprOS) VersionAtLeast(constraint string) (bool, error) { + return os.VersionCheck(">=" + constraint) +} + +// VersionIsLower returns true if the version of the OS is lower than the given version. +func (os ExprOS) VersionIsLower(version string) (bool, error) { + result, err := os.VersionAtLeast(version) + if err != nil { + return false, err + } + + return !result, nil +} + +// ExprEnvironment is used to expose functions and values to the rule engine. +// It can cache the results of service detection commands, like systemctl etc. +type ExprEnvironment struct { + OS ExprOS + + _serviceState *ExprServiceState + _state *ExprState +} + +// NewExprEnvironment creates an environment object for the rule engine. +func NewExprEnvironment(opts DetectOptions, os ExprOS) ExprEnvironment { + return ExprEnvironment{ + _state: &ExprState{ + detectOptions: opts, + + unitsSearched: make(map[string]bool), + installedUnits: make(map[string]bool), + + processesSearched: make(map[string]bool), + runningProcesses: make(map[string]bool), + }, + _serviceState: &ExprServiceState{}, + OS: os, + } +} + +// PathExists returns true if the given path exists. +func (e ExprEnvironment) PathExists(path string) bool { + _, err := os.Stat(path) + + return err == nil +} + +// UnitFound returns true if the unit is listed in the systemctl output. +// Whether a disabled or failed unit is considered found or not, depends on the +// systemctl parameters used. +func (e ExprEnvironment) UnitFound(unitName string) (bool, error) { + // fill initial caches + if len(e._state.unitsSearched) == 0 { + if !e._state.detectOptions.SnubSystemd { + units, err := systemdUnitList() + if err != nil { + return false, err + } + + for _, name := range units { + e._state.installedUnits[name] = true + } + } + + for _, name := range e._state.detectOptions.ForcedUnits { + e._state.installedUnits[name] = true + } + } + + e._state.unitsSearched[unitName] = true + if e._state.installedUnits[unitName] { + e._serviceState.detectedUnits = append(e._serviceState.detectedUnits, unitName) + + return true, nil + } + + return false, nil +} + +// ProcessRunning returns true if there is a running process with the given name. +func (e ExprEnvironment) ProcessRunning(processName string) (bool, error) { + if len(e._state.processesSearched) == 0 { + procs, err := process.Processes() + if err != nil { + return false, fmt.Errorf("while looking up running processes: %w", err) + } + + for _, p := range procs { + name, err := p.Name() + if err != nil { + return false, fmt.Errorf("while looking up running processes: %w", err) + } + + e._state.runningProcesses[name] = true + } + + for _, name := range e._state.detectOptions.ForcedProcesses { + e._state.runningProcesses[name] = true + } + } + + e._state.processesSearched[processName] = true + + return e._state.runningProcesses[processName], nil +} + +// applyRules checks if the 'when' expressions are true and returns a Service struct, +// augmented with default values and anything that might be useful later on +// +// All expressions are evaluated (no short-circuit) because we want to know if there are errors. +func applyRules(svc Service, env ExprEnvironment) (Service, bool, error) { + newsvc := svc + svcok := true + env._serviceState = &ExprServiceState{} + + for _, rule := range svc.When { + out, err := expr.Eval(rule, env) + log.Tracef(" Rule '%s' -> %t, %v", rule, out, err) + + if err != nil { + return Service{}, false, fmt.Errorf("rule '%s': %w", rule, err) + } + + outbool, ok := out.(bool) + if !ok { + return Service{}, false, fmt.Errorf("rule '%s': type must be a boolean", rule) + } + + svcok = svcok && outbool + } + + // if newsvc.Acquis == nil || (newsvc.Acquis.LogFiles == nil && newsvc.Acquis.JournalCTLFilter == nil) { + // for _, unitName := range env._serviceState.detectedUnits { + // if newsvc.Acquis == nil { + // newsvc.Acquis = &AcquisItem{} + // } + // // if there is reference to more than one unit in the rules, we use the first one + // newsvc.Acquis.JournalCTLFilter = []string{fmt.Sprintf(`_SYSTEMD_UNIT=%s`, unitName)} + // break //nolint // we want to exit after one iteration + // } + // } + + return newsvc, svcok, nil +} + +// filterWithRules decorates a DetectConfig map by filtering according to the when: clauses, +// and applying default values or whatever useful to the Service items. +func filterWithRules(dc DetectConfig, env ExprEnvironment) (map[string]Service, error) { + ret := make(map[string]Service) + + for name := range dc.Detect { + // + // an empty list of when: clauses defaults to true, if we want + // to change this behavior, the place is here. + // if len(svc.When) == 0 { + // log.Warningf("empty 'when' clause: %+v", svc) + // } + // + log.Trace("Evaluating rules for: ", name) + + svc, ok, err := applyRules(dc.Detect[name], env) + if err != nil { + return nil, fmt.Errorf("while looking for service %s: %w", name, err) + } + + if !ok { + log.Tracef(" Skipping %s", name) + + continue + } + + log.Tracef(" Detected %s", name) + + ret[name] = svc + } + + return ret, nil +} + +// return units that have been forced but not searched yet. +func (e ExprEnvironment) unsearchedUnits() []string { + ret := []string{} + + for _, unit := range e._state.detectOptions.ForcedUnits { + if !e._state.unitsSearched[unit] { + ret = append(ret, unit) + } + } + + return ret +} + +// return processes that have been forced but not searched yet. +func (e ExprEnvironment) unsearchedProcesses() []string { + ret := []string{} + + for _, proc := range e._state.detectOptions.ForcedProcesses { + if !e._state.processesSearched[proc] { + ret = append(ret, proc) + } + } + + return ret +} + +// checkConsumedForcedItems checks if all the "forced" options (units or processes) have been evaluated during the service detection. +func checkConsumedForcedItems(e ExprEnvironment) error { + unconsumed := e.unsearchedUnits() + + unitMsg := "" + if len(unconsumed) > 0 { + unitMsg = fmt.Sprintf("unit(s) forced but not supported: %v", unconsumed) + } + + unconsumed = e.unsearchedProcesses() + + procsMsg := "" + if len(unconsumed) > 0 { + procsMsg = fmt.Sprintf("process(es) forced but not supported: %v", unconsumed) + } + + join := "" + if unitMsg != "" && procsMsg != "" { + join = "; " + } + + if unitMsg != "" || procsMsg != "" { + return fmt.Errorf("%s%s%s", unitMsg, join, procsMsg) + } + + return nil +} + +// DetectOptions contains parameters for the Detect function. +type DetectOptions struct { + // slice of unit names that we want to force-detect + ForcedUnits []string + // slice of process names that we want to force-detect + ForcedProcesses []string + ForcedOS ExprOS + SkipServices []string + SnubSystemd bool +} + +// Detect performs the service detection from a given configuration. +// It outputs a setup file that can be used as input to "cscli setup install-hub" +// or "cscli setup datasources". +func Detect(serviceDetectionFile string, opts DetectOptions) (Setup, error) { + ret := Setup{} + + // explicitly initialize to avoid json mashaling an empty slice as "null" + ret.Setup = make([]ServiceSetup, 0) + + log.Tracef("Reading detection rules: %s", serviceDetectionFile) + + sc, err := readDetectConfig(serviceDetectionFile) + if err != nil { + return ret, err + } + + // // generate acquis.yaml snippet for this service + // for key := range sc.Detect { + // svc := sc.Detect[key] + // if svc.Acquis != nil { + // svc.AcquisYAML, err = yaml.Marshal(svc.Acquis) + // if err != nil { + // return ret, err + // } + // sc.Detect[key] = svc + // } + // } + + var osfull *osinfo.OSInfo + + os := opts.ForcedOS + if os == (ExprOS{}) { + osfull, err = osinfo.GetOSInfo() + if err != nil { + return ret, fmt.Errorf("detecting OS: %w", err) + } + + log.Tracef("Detected OS - %+v", *osfull) + + os = ExprOS{ + Family: osfull.Family, + ID: osfull.ID, + RawVersion: osfull.Version, + } + } else { + log.Tracef("Forced OS - %+v", os) + } + + if len(opts.ForcedUnits) > 0 { + log.Tracef("Forced units - %v", opts.ForcedUnits) + } + + if len(opts.ForcedProcesses) > 0 { + log.Tracef("Forced processes - %v", opts.ForcedProcesses) + } + + env := NewExprEnvironment(opts, os) + + detected, err := filterWithRules(sc, env) + if err != nil { + return ret, err + } + + if err = checkConsumedForcedItems(env); err != nil { + return ret, err + } + + // remove services the user asked to ignore + for _, name := range opts.SkipServices { + delete(detected, name) + } + + // sort the keys (service names) to have them in a predictable + // order in the final output + + keys := make([]string, 0) + for k := range detected { + keys = append(keys, k) + } + + sort.Strings(keys) + + for _, name := range keys { + svc := detected[name] + // if svc.DataSource != nil { + // if svc.DataSource.Labels["type"] == "" { + // return Setup{}, fmt.Errorf("missing type label for service %s", name) + // } + // err = yaml.Unmarshal(svc.AcquisYAML, svc.DataSource) + // if err != nil { + // return Setup{}, fmt.Errorf("while unmarshaling datasource for service %s: %w", name, err) + // } + // } + + ret.Setup = append(ret.Setup, ServiceSetup{ + DetectedService: name, + Install: svc.Install, + DataSource: svc.DataSource, + }) + } + + return ret, nil +} + +// ListSupported parses the configuration file and outputs a list of the supported services. +func ListSupported(serviceDetectionFile string) ([]string, error) { + dc, err := readDetectConfig(serviceDetectionFile) + if err != nil { + return nil, err + } + + keys := make([]string, 0) + for k := range dc.Detect { + keys = append(keys, k) + } + + sort.Strings(keys) + + return keys, nil +} diff --git a/pkg/setup/detect_test.go b/pkg/setup/detect_test.go new file mode 100644 index 000000000..86e738299 --- /dev/null +++ b/pkg/setup/detect_test.go @@ -0,0 +1,1017 @@ +package setup_test + +import ( + "fmt" + "os" + "os/exec" + "runtime" + "testing" + + "github.com/lithammer/dedent" + "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/crowdsec/pkg/cstest" + "github.com/crowdsecurity/crowdsec/pkg/setup" +) + +//nolint:dupword +var fakeSystemctlOutput = `UNIT FILE STATE VENDOR PRESET +crowdsec-setup-detect.service enabled enabled +apache2.service enabled enabled +apparmor.service enabled enabled +apport.service enabled enabled +atop.service enabled enabled +atopacct.service enabled enabled +finalrd.service enabled enabled +fwupd-refresh.service enabled enabled +fwupd.service enabled enabled + +9 unit files listed.` + +func fakeExecCommandNotFound(command string, args ...string) *exec.Cmd { + cs := []string{"-test.run=TestSetupHelperProcess", "--", command} + cs = append(cs, args...) + cmd := exec.Command("this-command-does-not-exist", cs...) + cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"} + + return cmd +} + +func fakeExecCommand(command string, args ...string) *exec.Cmd { + cs := []string{"-test.run=TestSetupHelperProcess", "--", command} + cs = append(cs, args...) + //nolint:gosec + cmd := exec.Command(os.Args[0], cs...) + cmd.Env = []string{"GO_WANT_HELPER_PROCESS=1"} + + return cmd +} + +func TestSetupHelperProcess(t *testing.T) { + if os.Getenv("GO_WANT_HELPER_PROCESS") != "1" { + return + } + + fmt.Fprint(os.Stdout, fakeSystemctlOutput) + os.Exit(0) +} + +func tempYAML(t *testing.T, content string) string { + t.Helper() + require := require.New(t) + file, err := os.CreateTemp("", "") + require.NoError(err) + + _, err = file.WriteString(dedent.Dedent(content)) + require.NoError(err) + + err = file.Close() + require.NoError(err) + + return file.Name() +} + +func TestPathExists(t *testing.T) { + t.Parallel() + + type test struct { + path string + expected bool + } + + tests := []test{ + {"/this-should-not-exist", false}, + } + + if runtime.GOOS == "windows" { + tests = append(tests, test{`C:\`, true}) + } else { + tests = append(tests, test{"/tmp", true}) + } + + for _, tc := range tests { + tc := tc + env := setup.NewExprEnvironment(setup.DetectOptions{}, setup.ExprOS{}) + + t.Run(tc.path, func(t *testing.T) { + t.Parallel() + actual := env.PathExists(tc.path) + require.Equal(t, tc.expected, actual) + }) + } +} + +func TestVersionCheck(t *testing.T) { + t.Parallel() + + tests := []struct { + version string + constraint string + expected bool + expectedErr string + }{ + {"1", "=1", true, ""}, + {"1", "!=1", false, ""}, + {"1", "<=1", true, ""}, + {"1", ">1", false, ""}, + {"1", ">=1", true, ""}, + {"1.0", "<1.0", false, ""}, + {"1", "<1", true, ""}, // XXX why? + {"1.3.5", "1.3", false, ""}, // XXX ok? + {"1.0", "<1.0", false, ""}, + {"1.0", "<=1.0", true, ""}, + {"2", ">1, <3", true, ""}, + {"2", "<=2, >=2.2", false, ""}, + {"2.3", "~2", true, ""}, + {"2.3", "=2", true, ""}, + {"1.1.1", "=1.1", false, ""}, + {"1.1.1", "1.1", false, ""}, + {"1.1", "!=1.1.1", true, ""}, + {"1.1", "~1.1.1", false, ""}, + {"1.1.1", "~1.1", true, ""}, + {"1.1.3", "~1.1", true, ""}, + {"19.04", "<19.10", true, ""}, + {"19.04", ">=19.10", false, ""}, + {"19.04", "=19.4", true, ""}, + {"19.04", "~19.4", true, ""}, + {"1.2.3", "~1.2", true, ""}, + {"1.2.3", "!=1.2", true, ""}, + {"1.2.3", "1.1.1 - 1.3.4", true, ""}, + {"1.3.5", "1.1.1 - 1.3.4", false, ""}, + {"1.3.5", "=1", true, ""}, + {"1.3.5", "1", true, ""}, + } + + for _, tc := range tests { + tc := tc + e := setup.ExprOS{RawVersion: tc.version} + + t.Run(fmt.Sprintf("Check(%s,%s)", tc.version, tc.constraint), func(t *testing.T) { + t.Parallel() + actual, err := e.VersionCheck(tc.constraint) + cstest.RequireErrorContains(t, err, tc.expectedErr) + require.Equal(t, tc.expected, actual) + }) + } +} + +// This is not required for Masterminds/semver +/* +func TestNormalizeVersion(t *testing.T) { + t.Parallel() + + tests := []struct { + version string + expected string + }{ + {"0", "0"}, + {"2", "2"}, + {"3.14", "3.14"}, + {"1.0", "1.0"}, + {"18.04", "18.4"}, + {"0.0.0", "0.0.0"}, + {"18.04.0", "18.4.0"}, + {"18.0004.0", "18.4.0"}, + {"21.04.2", "21.4.2"}, + {"050", "50"}, + {"trololo", "trololo"}, + {"0001.002.03", "1.2.3"}, + {"0001.002.03-trololo", "0001.002.03-trololo"}, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.version, func(t *testing.T) { + t.Parallel() + actual := setup.NormalizeVersion(tc.version) + require.Equal(t, tc.expected, actual) + }) + } +} +*/ + +func TestListSupported(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + yml string + expected []string + expectedErr string + }{ + { + "list configured services", + ` + version: 1.0 + detect: + foo: + bar: + baz: + `, + []string{"foo", "bar", "baz"}, + "", + }, + { + "invalid yaml: blahblah", + "blahblah", + nil, + "yaml: unmarshal errors:", + }, + { + "invalid yaml: tabs are not allowed", + ` + version: 1.0 + detect: + foos: + `, + nil, + "yaml: line 4: found character that cannot start any token", + }, + { + "invalid yaml: no version", + "{}", + nil, + "missing version tag (must be 1.0)", + }, + { + "invalid yaml: bad version", + "version: 2.0", + nil, + "unsupported version tag '2.0' (must be 1.0)", + }, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + f := tempYAML(t, tc.yml) + defer os.Remove(f) + supported, err := setup.ListSupported(f) + cstest.RequireErrorContains(t, err, tc.expectedErr) + require.ElementsMatch(t, tc.expected, supported) + }) + } +} + +func TestApplyRules(t *testing.T) { + t.Parallel() + require := require.New(t) + + tests := []struct { + name string + rules []string + expectedOk bool + expectedErr string + }{ + { + "empty list is always true", // XXX or false? + []string{}, + true, + "", + }, + { + "simple true expression", + []string{"1+1==2"}, + true, + "", + }, + { + "simple false expression", + []string{"2+2==5"}, + false, + "", + }, + { + "all expressions are true", + []string{"1+2==3", "1!=2"}, + true, + "", + }, + { + "all expressions must be true", + []string{"true", "1==3", "1!=2"}, + false, + "", + }, + { + "each expression must be a boolan", + []string{"true", "\"notabool\""}, + false, + "rule '\"notabool\"': type must be a boolean", + }, + { + // we keep evaluating expressions to ensure that the + // file is formally correct, even if it can some time. + "each expression must be a boolan (no short circuit)", + []string{"false", "3"}, + false, + "rule '3': type must be a boolean", + }, + { + "unknown variable", + []string{"false", "doesnotexist"}, + false, + "rule 'doesnotexist': cannot fetch doesnotexist from", + }, + { + "unknown expression", + []string{"false", "doesnotexist()"}, + false, + "rule 'doesnotexist()': cannot get \"doesnotexist\" from", + }, + } + + env := setup.ExprEnvironment{} + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + svc := setup.Service{When: tc.rules} + _, actualOk, err := setup.ApplyRules(svc, env) //nolint:typecheck,nolintlint // exported only for tests + cstest.RequireErrorContains(t, err, tc.expectedErr) + require.Equal(tc.expectedOk, actualOk) + }) + } +} + +// XXX TODO: TestApplyRules with journalctl default + +func TestUnitFound(t *testing.T) { + require := require.New(t) + setup.ExecCommand = fakeExecCommand + + defer func() { setup.ExecCommand = exec.Command }() + + env := setup.NewExprEnvironment(setup.DetectOptions{}, setup.ExprOS{}) + + installed, err := env.UnitFound("crowdsec-setup-detect.service") + require.NoError(err) + + require.Equal(true, installed) +} + +// TODO apply rules to filter a list of Service structs +// func testFilterWithRules(t *testing.T) { +// } + +func TestDetectSimpleRule(t *testing.T) { + require := require.New(t) + setup.ExecCommand = fakeExecCommand + + f := tempYAML(t, ` + version: 1.0 + detect: + good: + when: + - true + bad: + when: + - false + ugly: + `) + defer os.Remove(f) + + detected, err := setup.Detect(f, setup.DetectOptions{}) + require.NoError(err) + + expected := []setup.ServiceSetup{ + {DetectedService: "good"}, + {DetectedService: "ugly"}, + } + + require.ElementsMatch(expected, detected.Setup) +} + +func TestDetectUnitError(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping on windows") + } + + require := require.New(t) + setup.ExecCommand = fakeExecCommandNotFound + + defer func() { setup.ExecCommand = exec.Command }() + + tests := []struct { + name string + config string + expected setup.Setup + expectedErr string + }{ + { + "error is reported if systemctl does not exist", + ` +version: 1.0 +detect: + wizard: + when: + - UnitFound("crowdsec-setup-detect.service")`, + setup.Setup{[]setup.ServiceSetup{}}, + `while looking for service wizard: rule 'UnitFound("crowdsec-setup-detect.service")': ` + + `running systemctl: exec: "this-command-does-not-exist": executable file not found in $PATH`, + }, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + f := tempYAML(t, tc.config) + defer os.Remove(f) + + detected, err := setup.Detect(f, setup.DetectOptions{}) + cstest.RequireErrorContains(t, err, tc.expectedErr) + require.Equal(tc.expected, detected) + }) + } +} + +func TestDetectUnit(t *testing.T) { + require := require.New(t) + setup.ExecCommand = fakeExecCommand + + defer func() { setup.ExecCommand = exec.Command }() + + tests := []struct { + name string + config string + expected setup.Setup + expectedErr string + }{ + // { + // "detect a single unit, with default log filter", + // ` + // version: 1.0 + // detect: + // wizard: + // when: + // - UnitFound("crowdsec-setup-detect.service") + // datasource: + // labels: + // type: syslog + // sorcerer: + // when: + // - UnitFound("sorcerer.service")`, + // setup.Setup{ + // Setup: []setup.ServiceSetup{ + // { + // DetectedService: "wizard", + // DataSource: setup.DataSourceItem{ + // "Labels": map[string]string{"type": "syslog"}, + // "JournalCTLFilter": []string{"_SYSTEMD_UNIT=crowdsec-setup-detect.service"}, + // }, + // }, + // }, + // }, + // "", + // }, + // { + // "detect a single unit, but type label is missing", + // ` + // version: 1.0 + // detect: + // wizard: + // when: + // - UnitFound("crowdsec-setup-detect.service")`, + // setup.Setup{}, + // "missing type label for service wizard", + // }, + { + "detect unit and pick up acquisistion filter", + ` +version: 1.0 +detect: + wizard: + when: + - UnitFound("crowdsec-setup-detect.service") + datasource: + source: journalctl + labels: + type: syslog + journalctl_filter: + - _MY_CUSTOM_FILTER=something`, + setup.Setup{ + Setup: []setup.ServiceSetup{ + { + DetectedService: "wizard", + DataSource: setup.DataSourceItem{ + // XXX this should not be DataSourceItem ?? + "source": "journalctl", + "labels": setup.DataSourceItem{"type": "syslog"}, + "journalctl_filter": []interface{}{"_MY_CUSTOM_FILTER=something"}, + }, + }, + }, + }, + "", + }, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + f := tempYAML(t, tc.config) + defer os.Remove(f) + + detected, err := setup.Detect(f, setup.DetectOptions{}) + cstest.RequireErrorContains(t, err, tc.expectedErr) + require.Equal(tc.expected, detected) + }) + } +} + +func TestDetectForcedUnit(t *testing.T) { + require := require.New(t) + setup.ExecCommand = fakeExecCommand + + defer func() { setup.ExecCommand = exec.Command }() + + f := tempYAML(t, ` + version: 1.0 + detect: + wizard: + when: + - UnitFound("crowdsec-setup-forced.service") + datasource: + source: journalctl + labels: + type: syslog + journalctl_filter: + - _SYSTEMD_UNIT=crowdsec-setup-forced.service + `) + defer os.Remove(f) + + detected, err := setup.Detect(f, setup.DetectOptions{ForcedUnits: []string{"crowdsec-setup-forced.service"}}) + require.NoError(err) + + expected := setup.Setup{ + Setup: []setup.ServiceSetup{ + { + DetectedService: "wizard", + DataSource: setup.DataSourceItem{ + "source": "journalctl", + "labels": setup.DataSourceItem{"type": "syslog"}, + "journalctl_filter": []interface{}{"_SYSTEMD_UNIT=crowdsec-setup-forced.service"}, + }, + }, + }, + } + require.Equal(expected, detected) +} + +func TestDetectForcedProcess(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping on windows") + // while looking for service wizard: rule 'ProcessRunning("foobar")': while looking up running processes: could not get Name: A device attached to the system is not functioning. + } + + require := require.New(t) + setup.ExecCommand = fakeExecCommand + + defer func() { setup.ExecCommand = exec.Command }() + + f := tempYAML(t, ` + version: 1.0 + detect: + wizard: + when: + - ProcessRunning("foobar") + `) + defer os.Remove(f) + + detected, err := setup.Detect(f, setup.DetectOptions{ForcedProcesses: []string{"foobar"}}) + require.NoError(err) + + expected := setup.Setup{ + Setup: []setup.ServiceSetup{ + {DetectedService: "wizard"}, + }, + } + require.Equal(expected, detected) +} + +func TestDetectSkipService(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("skipping on windows") + } + + require := require.New(t) + setup.ExecCommand = fakeExecCommand + + defer func() { setup.ExecCommand = exec.Command }() + + f := tempYAML(t, ` + version: 1.0 + detect: + wizard: + when: + - ProcessRunning("foobar") + `) + defer os.Remove(f) + + detected, err := setup.Detect(f, setup.DetectOptions{ForcedProcesses: []string{"foobar"}, SkipServices: []string{"wizard"}}) + require.NoError(err) + + expected := setup.Setup{[]setup.ServiceSetup{}} + require.Equal(expected, detected) +} + +func TestDetectForcedOS(t *testing.T) { + require := require.New(t) + setup.ExecCommand = fakeExecCommand + + defer func() { setup.ExecCommand = exec.Command }() + + type test struct { + name string + config string + forced setup.ExprOS + expected setup.Setup + expectedErr string + } + + tests := []test{ + { + "detect OS - force linux", + ` + version: 1.0 + detect: + linux: + when: + - OS.Family == "linux"`, + setup.ExprOS{Family: "linux"}, + setup.Setup{ + Setup: []setup.ServiceSetup{ + {DetectedService: "linux"}, + }, + }, + "", + }, + { + "detect OS - force windows", + ` + version: 1.0 + detect: + windows: + when: + - OS.Family == "windows"`, + setup.ExprOS{Family: "windows"}, + setup.Setup{ + Setup: []setup.ServiceSetup{ + {DetectedService: "windows"}, + }, + }, + "", + }, + { + "detect OS - ubuntu (no match)", + ` + version: 1.0 + detect: + linux: + when: + - OS.Family == "linux" && OS.ID == "ubuntu"`, + setup.ExprOS{Family: "linux"}, + setup.Setup{[]setup.ServiceSetup{}}, + "", + }, + { + "detect OS - ubuntu (match)", + ` + version: 1.0 + detect: + linux: + when: + - OS.Family == "linux" && OS.ID == "ubuntu"`, + setup.ExprOS{Family: "linux", ID: "ubuntu"}, + setup.Setup{ + Setup: []setup.ServiceSetup{ + {DetectedService: "linux"}, + }, + }, + "", + }, + { + "detect OS - ubuntu (match with version)", + ` + version: 1.0 + detect: + linux: + when: + - OS.Family == "linux" && OS.ID == "ubuntu" && OS.VersionCheck("19.04")`, + setup.ExprOS{Family: "linux", ID: "ubuntu", RawVersion: "19.04"}, + setup.Setup{ + Setup: []setup.ServiceSetup{ + {DetectedService: "linux"}, + }, + }, + "", + }, + { + "detect OS - ubuntu >= 20.04 (no match: no version detected)", + ` + version: 1.0 + detect: + linux: + when: + - OS.ID == "ubuntu" && OS.VersionCheck(">=20.04")`, + setup.ExprOS{Family: "linux"}, + setup.Setup{[]setup.ServiceSetup{}}, + "", + }, + { + "detect OS - ubuntu >= 20.04 (no match: version is lower)", + ` + version: 1.0 + detect: + linux: + when: + - OS.ID == "ubuntu" && OS.VersionCheck(">=20.04")`, + setup.ExprOS{Family: "linux", ID: "ubuntu", RawVersion: "19.10"}, + setup.Setup{[]setup.ServiceSetup{}}, + "", + }, + { + "detect OS - ubuntu >= 20.04 (match: same version)", + ` + version: 1.0 + detect: + linux: + when: + - OS.ID == "ubuntu" && OS.VersionCheck(">=20.04")`, + setup.ExprOS{Family: "linux", ID: "ubuntu", RawVersion: "20.04"}, + setup.Setup{ + Setup: []setup.ServiceSetup{ + {DetectedService: "linux"}, + }, + }, + "", + }, + { + "detect OS - ubuntu >= 20.04 (match: version is higher)", + ` + version: 1.0 + detect: + linux: + when: + - OS.ID == "ubuntu" && OS.VersionCheck(">=20.04")`, + setup.ExprOS{Family: "linux", ID: "ubuntu", RawVersion: "22.04"}, + setup.Setup{ + Setup: []setup.ServiceSetup{ + {DetectedService: "linux"}, + }, + }, + "", + }, + + { + "detect OS - ubuntu < 20.04 (no match: no version detected)", + ` + version: 1.0 + detect: + linux: + when: + - OS.ID == "ubuntu" && OS.VersionCheck("<20.04")`, + setup.ExprOS{Family: "linux"}, + setup.Setup{[]setup.ServiceSetup{}}, + "", + }, + { + "detect OS - ubuntu < 20.04 (no match: version is higher)", + ` + version: 1.0 + detect: + linux: + when: + - OS.ID == "ubuntu" && OS.VersionCheck("<20.04")`, + setup.ExprOS{Family: "linux", ID: "ubuntu", RawVersion: "20.10"}, + setup.Setup{[]setup.ServiceSetup{}}, + "", + }, + { + "detect OS - ubuntu < 20.04 (no match: same version)", + ` + version: 1.0 + detect: + linux: + when: + - OS.ID == "ubuntu" && OS.VersionCheck("<20.04")`, + setup.ExprOS{Family: "linux", ID: "ubuntu", RawVersion: "20.04"}, + setup.Setup{[]setup.ServiceSetup{}}, + "", + }, + { + "detect OS - ubuntu < 20.04 (match: version is lower)", + ` + version: 1.0 + detect: + linux: + when: + - OS.ID == "ubuntu" + - OS.VersionCheck("<20.04")`, + setup.ExprOS{Family: "linux", ID: "ubuntu", RawVersion: "19.10"}, + setup.Setup{ + Setup: []setup.ServiceSetup{ + {DetectedService: "linux"}, + }, + }, + "", + }, + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + f := tempYAML(t, tc.config) + defer os.Remove(f) + + detected, err := setup.Detect(f, setup.DetectOptions{ForcedOS: tc.forced}) + cstest.RequireErrorContains(t, err, tc.expectedErr) + require.Equal(tc.expected, detected) + }) + } +} + +func TestDetectDatasourceValidation(t *testing.T) { + // It could be a good idea to test UnmarshalConfig() separately in addition + // to Configure(), in each datasource. For now, we test these here. + + require := require.New(t) + setup.ExecCommand = fakeExecCommand + + defer func() { setup.ExecCommand = exec.Command }() + + type test struct { + name string + config string + expected setup.Setup + expectedErr string + } + + tests := []test{ + { + name: "source is empty", + config: ` + version: 1.0 + detect: + wizard: + datasource: + labels: + type: something`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for wizard: source is empty", + }, { + name: "source is unknown", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: wombat`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for foobar: unknown source 'wombat'", + }, { + name: "source is misplaced", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: file`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "while parsing {{.DetectYaml}}: yaml: unmarshal errors:\n line 6: field source not found in type setup.Service", + }, { + name: "source is mismatched", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: journalctl + filename: /path/to/file.log`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for foobar: cannot parse JournalCtlSource configuration: yaml: unmarshal errors:\n line 1: field filename not found in type journalctlacquisition.JournalCtlConfiguration", + }, { + name: "source file: required fields", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: file`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for foobar: no filename or filenames configuration provided", + }, { + name: "source journalctl: required fields", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: journalctl`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for foobar: journalctl_filter is required", + }, { + name: "source cloudwatch: required fields", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: cloudwatch`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for foobar: group_name is mandatory for CloudwatchSource", + }, { + name: "source syslog: all fields are optional", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: syslog`, + expected: setup.Setup{ + Setup: []setup.ServiceSetup{ + { + DetectedService:"foobar", + DataSource: setup.DataSourceItem{"source":"syslog"}, + }, + }, + }, + }, { + name: "source docker: required fields", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: docker`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for foobar: no containers names or containers ID configuration provided", + }, { + name: "source kinesis: required fields (enhanced fanout=false)", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: kinesis`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for foobar: stream_name is mandatory when use_enhanced_fanout is false", + }, { + name: "source kinesis: required fields (enhanced fanout=true)", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: kinesis + use_enhanced_fanout: true`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for foobar: stream_arn is mandatory when use_enhanced_fanout is true", + }, { + name: "source kafka: required fields", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: kafka`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for foobar: cannot create a kafka reader with an empty list of broker addresses", + }, + } + + if runtime.GOOS == "windows" { + tests = append(tests, test{ + name: "source wineventlog: required fields", + config: ` + version: 1.0 + detect: + foobar: + datasource: + source: wineventlog`, + expected: setup.Setup{Setup:[]setup.ServiceSetup{}}, + expectedErr: "invalid datasource for foobar: event_channel or xpath_query must be set", + }) + } + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + detectYaml := tempYAML(t, tc.config) + defer os.Remove(detectYaml) + + data := map[string]string{ + "DetectYaml": detectYaml, + } + + expectedErr, err := cstest.Interpolate(tc.expectedErr, data) + require.NoError(err) + + detected, err := setup.Detect(detectYaml, setup.DetectOptions{}) + cstest.RequireErrorContains(t, err, expectedErr) + require.Equal(tc.expected, detected) + }) + } +} diff --git a/pkg/setup/export_test.go b/pkg/setup/export_test.go new file mode 100644 index 000000000..56ca02945 --- /dev/null +++ b/pkg/setup/export_test.go @@ -0,0 +1,9 @@ +package setup + +var ( + SystemdUnitList = systemdUnitList + FilterWithRules = filterWithRules + ApplyRules = applyRules + +// NormalizeVersion = normalizeVersion +) diff --git a/pkg/setup/install.go b/pkg/setup/install.go new file mode 100644 index 000000000..5d3bfdbc9 --- /dev/null +++ b/pkg/setup/install.go @@ -0,0 +1,255 @@ +package setup + +import ( + "bytes" + "fmt" + "os" + "path/filepath" + "strings" + + goccyyaml "github.com/goccy/go-yaml" + "gopkg.in/yaml.v3" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" +) + +// AcquisDocument is created from a SetupItem. It represents a single YAML document, and can be part of a multi-document file. +type AcquisDocument struct { + AcquisFilename string + DataSource map[string]interface{} +} + +func decodeSetup(input []byte, fancyErrors bool) (Setup, error) { + ret := Setup{} + + // parse with goccy to have better error messages in many cases + dec := goccyyaml.NewDecoder(bytes.NewBuffer(input), goccyyaml.Strict()) + + if err := dec.Decode(&ret); err != nil { + if fancyErrors { + return ret, fmt.Errorf("%v", goccyyaml.FormatError(err, true, true)) + } + // XXX errors here are multiline, should we just print them to stderr instead of logging? + return ret, fmt.Errorf("%v", err) + } + + // parse again because goccy is not strict enough anyway + dec2 := yaml.NewDecoder(bytes.NewBuffer(input)) + dec2.KnownFields(true) + + if err := dec2.Decode(&ret); err != nil { + return ret, fmt.Errorf("while unmarshaling setup file: %w", err) + } + + return ret, nil +} + +// InstallHubItems installs the objects recommended in a setup file. +func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error { + setupEnvelope, err := decodeSetup(input, false) + if err != nil { + return err + } + + if err := csConfig.LoadHub(); err != nil { + return fmt.Errorf("loading hub: %w", err) + } + + if err := cwhub.SetHubBranch(); err != nil { + return fmt.Errorf("setting hub branch: %w", err) + } + + if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { + return fmt.Errorf("getting hub index: %w", err) + } + + for _, setupItem := range setupEnvelope.Setup { + forceAction := false + downloadOnly := false + install := setupItem.Install + + if install == nil { + continue + } + + if len(install.Collections) > 0 { + for _, collection := range setupItem.Install.Collections { + if dryRun { + fmt.Println("dry-run: would install collection", collection) + + continue + } + + if err := cwhub.InstallItem(csConfig, collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil { + return fmt.Errorf("while installing collection %s: %w", collection, err) + } + } + } + + if len(install.Parsers) > 0 { + for _, parser := range setupItem.Install.Parsers { + if dryRun { + fmt.Println("dry-run: would install parser", parser) + + continue + } + + if err := cwhub.InstallItem(csConfig, parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil { + return fmt.Errorf("while installing parser %s: %w", parser, err) + } + } + } + + if len(install.Scenarios) > 0 { + for _, scenario := range setupItem.Install.Scenarios { + if dryRun { + fmt.Println("dry-run: would install scenario", scenario) + + continue + } + + if err := cwhub.InstallItem(csConfig, scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil { + return fmt.Errorf("while installing scenario %s: %w", scenario, err) + } + } + } + + if len(install.PostOverflows) > 0 { + for _, postoverflow := range setupItem.Install.PostOverflows { + if dryRun { + fmt.Println("dry-run: would install postoverflow", postoverflow) + + continue + } + + if err := cwhub.InstallItem(csConfig, postoverflow, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil { + return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err) + } + } + } + } + + return nil +} + +// marshalAcquisDocuments creates the monolithic file, or itemized files (if a directory is provided) with the acquisition documents. +func marshalAcquisDocuments(ads []AcquisDocument, toDir string) (string, error) { + var sb strings.Builder + + dashTerminator := false + + disclaimer := ` +# +# This file was automatically generated by "cscli setup datasources". +# You can modify it by hand, but will be responsible for its maintenance. +# To add datasources or logfiles, you can instead write a new configuration +# in the directory defined by acquisition_dir. +# + +` + + if toDir == "" { + sb.WriteString(disclaimer) + } else { + _, err := os.Stat(toDir) + if os.IsNotExist(err) { + return "", fmt.Errorf("directory %s does not exist", toDir) + } + } + + for _, ad := range ads { + out, err := goccyyaml.MarshalWithOptions(ad.DataSource, goccyyaml.IndentSequence(true)) + if err != nil { + return "", fmt.Errorf("while encoding datasource: %w", err) + } + + if toDir != "" { + if ad.AcquisFilename == "" { + return "", fmt.Errorf("empty acquis filename") + } + + fname := filepath.Join(toDir, ad.AcquisFilename) + fmt.Println("creating", fname) + + f, err := os.Create(fname) + if err != nil { + return "", fmt.Errorf("creating acquisition file: %w", err) + } + defer f.Close() + + _, err = f.WriteString(disclaimer) + if err != nil { + return "", fmt.Errorf("while writing to %s: %w", ad.AcquisFilename, err) + } + + _, err = f.Write(out) + if err != nil { + return "", fmt.Errorf("while writing to %s: %w", ad.AcquisFilename, err) + } + + f.Sync() + + continue + } + + if dashTerminator { + sb.WriteString("---\n") + } + + sb.Write(out) + + dashTerminator = true + } + + return sb.String(), nil +} + +// Validate checks the validity of a setup file. +func Validate(input []byte) error { + _, err := decodeSetup(input, true) + if err != nil { + return err + } + + return nil +} + +// DataSources generates the acquisition documents from a setup file. +func DataSources(input []byte, toDir string) (string, error) { + setupEnvelope, err := decodeSetup(input, false) + if err != nil { + return "", err + } + + ads := make([]AcquisDocument, 0) + + filename := func(basename string, ext string) string { + if basename == "" { + return basename + } + + return basename + ext + } + + for _, setupItem := range setupEnvelope.Setup { + datasource := setupItem.DataSource + + basename := "" + if toDir != "" { + basename = "setup." + setupItem.DetectedService + } + + if datasource == nil { + continue + } + + ad := AcquisDocument{ + AcquisFilename: filename(basename, ".yaml"), + DataSource: datasource, + } + ads = append(ads, ad) + } + + return marshalAcquisDocuments(ads, toDir) +} diff --git a/pkg/setup/units.go b/pkg/setup/units.go new file mode 100644 index 000000000..a0bccba4a --- /dev/null +++ b/pkg/setup/units.go @@ -0,0 +1,59 @@ +package setup + +import ( + "bufio" + "fmt" + "strings" + + log "github.com/sirupsen/logrus" +) + +// systemdUnitList returns all enabled systemd units. +// It needs to parse the table because -o json does not work everywhere. +func systemdUnitList() ([]string, error) { + wrap := func(err error) error { + return fmt.Errorf("running systemctl: %w", err) + } + + ret := make([]string, 0) + cmd := ExecCommand("systemctl", "list-unit-files", "--state=enabled,generated,static") + + stdout, err := cmd.StdoutPipe() + if err != nil { + return ret, wrap(err) + } + + log.Debugf("Running systemctl...") + + if err := cmd.Start(); err != nil { + return ret, wrap(err) + } + + scanner := bufio.NewScanner(stdout) + header := true // skip the first line + + for scanner.Scan() { + line := scanner.Text() + if len(line) == 0 { + break // the rest of the output is footer + } + + if !header { + spaceIdx := strings.IndexRune(line, ' ') + if spaceIdx == -1 { + return ret, fmt.Errorf("can't parse systemctl output") + } + + line = line[:spaceIdx] + ret = append(ret, line) + } + + header = false + } + + if err := cmd.Wait(); err != nil { + return ret, wrap(err) + } + + return ret, nil +} diff --git a/pkg/setup/units_test.go b/pkg/setup/units_test.go new file mode 100644 index 000000000..b1bfd8816 --- /dev/null +++ b/pkg/setup/units_test.go @@ -0,0 +1,32 @@ +package setup_test + +import ( + "os/exec" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/crowdsec/pkg/setup" +) + +func TestSystemdUnitList(t *testing.T) { + require := require.New(t) + setup.ExecCommand = fakeExecCommand + + defer func() { setup.ExecCommand = exec.Command }() + + units, err := setup.SystemdUnitList() //nolint:typecheck,nolintlint // exported only for tests + require.NoError(err) + + require.Equal([]string{ + "crowdsec-setup-detect.service", + "apache2.service", + "apparmor.service", + "apport.service", + "atop.service", + "atopacct.service", + "finalrd.service", + "fwupd-refresh.service", + "fwupd.service", + }, units) +} diff --git a/tests/bats/07_setup.bats b/tests/bats/07_setup.bats new file mode 100644 index 000000000..714b730c0 --- /dev/null +++ b/tests/bats/07_setup.bats @@ -0,0 +1,816 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" + ./instance-data load + HUB_DIR=$(config_get '.config_paths.hub_dir') + export HUB_DIR + DETECT_YAML="${HUB_DIR}/detect.yaml" + export DETECT_YAML + # shellcheck disable=SC2154 + TESTDATA="${BATS_TEST_DIRNAME}/testdata/07_setup" + export TESTDATA + + export CROWDSEC_FEATURE_CSCLI_SETUP="true" +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + load "../lib/bats-file/load.bash" + load "../lib/bats-mock/load.bash" + ./instance-data load +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +#shellcheck disable=SC2154 +@test "cscli setup" { + rune -0 cscli help + assert_line --regexp '^ +setup +Tools to configure crowdsec$' + + rune -0 cscli setup --help + assert_line 'Usage:' + assert_line ' cscli setup [command]' + assert_line 'Manage hub configuration and service detection' + assert_line --partial "detect detect running services, generate a setup file" + assert_line --partial "datasources generate datasource (acquisition) configuration from a setup file" + assert_line --partial "install-hub install items from a setup file" + assert_line --partial "validate validate a setup file" + + # cobra should return error for non-existing sub-subcommands, but doesn't + rune -0 cscli setup blahblah + assert_line 'Usage:' +} + +@test "cscli setup detect --help; --detect-config" { + rune -0 cscli setup detect --help + assert_line --regexp "detect running services, generate a setup file" + assert_line 'Usage:' + assert_line ' cscli setup detect [flags]' + assert_line --partial "--detect-config string path to service detection configuration (default \"${HUB_DIR}/detect.yaml\")" + assert_line --partial "--force-process strings force detection of a running process (can be repeated)" + assert_line --partial "--force-unit strings force detection of a systemd unit (can be repeated)" + assert_line --partial "--list-supported-services do not detect; only print supported services" + assert_line --partial "--force-os-family string override OS.Family: one of linux, freebsd, windows or darwin" + assert_line --partial "--force-os-id string override OS.ID=[debian | ubuntu | , redhat...]" + assert_line --partial "--force-os-version string override OS.RawVersion (of OS or Linux distribution)" + assert_line --partial "--skip-service strings ignore a service, don't recommend hub/datasources (can be repeated)" + + rune -1 --separate-stderr cscli setup detect --detect-config /path/does/not/exist + assert_stderr --partial "detecting services: while reading file: open /path/does/not/exist: no such file or directory" + + # rm -f "${HUB_DIR}/detect.yaml" +} + +@test "cscli setup detect (linux), --skip-service" { + [[ ${OSTYPE} =~ linux.* ]] || skip + tempfile=$(TMPDIR="$BATS_TEST_TMPDIR" mktemp) + cat <<-EOT >"${tempfile}" + version: 1.0 + detect: + linux: + when: + - OS.Family == "linux" + install: + collections: + - crowdsecurity/linux + thewiz: + when: + - OS.Family != "linux" + foobarbaz: + EOT + + rune -0 --separate-stderr cscli setup detect --detect-config "$tempfile" + assert_json '{setup:[{detected_service:"foobarbaz"},{detected_service:"linux",install:{collections:["crowdsecurity/linux"]}}]}' + + rune -0 --separate-stderr cscli setup detect --detect-config "$tempfile" --skip-service linux + assert_json '{setup:[{detected_service:"foobarbaz"}]}' +} + +@test "cscli setup detect --force-os-*" { + rune -0 --separate-stderr cscli setup detect --force-os-family linux --detect-config "${TESTDATA}/detect.yaml" + rune -0 jq -cS '.setup[] | select(.detected_service=="linux")' <(output) + assert_json '{detected_service:"linux",install:{collections:["crowdsecurity/linux"]},datasource:{source:"file",labels:{type:"syslog"},filenames:["/var/log/syslog","/var/log/kern.log","/var/log/messages"]}}' + + rune -0 --separate-stderr cscli setup detect --force-os-family freebsd --detect-config "${TESTDATA}/detect.yaml" + rune -0 jq -cS '.setup[] | select(.detected_service=="freebsd")' <(output) + assert_json '{detected_service:"freebsd",install:{collections:["crowdsecurity/freebsd"]}}' + + rune -0 --separate-stderr cscli setup detect --force-os-family windows --detect-config "${TESTDATA}/detect.yaml" + rune -0 jq -cS '.setup[] | select(.detected_service=="windows")' <(output) + assert_json '{detected_service:"windows",install:{collections:["crowdsecurity/windows"]}}' + + rune -0 --separate-stderr cscli setup detect --force-os-family darwin --detect-config "${TESTDATA}/detect.yaml" + + # XXX do we want do disallow unknown family? + # assert_stderr --partial "detecting services: OS 'darwin' not supported" + + # XXX TODO force-os-id, force-os-version +} + +@test "cscli setup detect --list-supported-services" { + tempfile=$(TMPDIR="$BATS_TEST_TMPDIR" mktemp) + cat <<-EOT >"${tempfile}" + version: 1.0 + detect: + thewiz: + foobarbaz: + apache2: + EOT + + rune -0 --separate-stderr cscli setup detect --list-supported-services --detect-config "$tempfile" + # the service list is sorted + assert_output - <<-EOT + apache2 + foobarbaz + thewiz + EOT + + cat <<-EOT >"${tempfile}" + thisisajoke + EOT + + rune -1 --separate-stderr cscli setup detect --list-supported-services --detect-config "$tempfile" + assert_stderr --partial "while parsing ${tempfile}: yaml: unmarshal errors:" + + rm -f "$tempfile" +} + +@test "cscli setup detect (systemctl)" { + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + apache2: + when: + - UnitFound("mock-apache2.service") + datasource: + source: file + filename: dummy.log + labels: + type: apache2 + EOT + + # transparently mock systemctl. It's easier if you can tell the application + # under test which executable to call (in which case just call $mock) but + # here we do the symlink and $PATH dance as an example + mocked_command="systemctl" + + # mock setup + mock="$(mock_create)" + mock_path="${mock%/*}" + mock_file="${mock##*/}" + ln -sf "${mock_path}/${mock_file}" "${mock_path}/${mocked_command}" + + #shellcheck disable=SC2030 + PATH="${mock_path}:${PATH}" + + mock_set_output "$mock" \ +'UNIT FILE STATE VENDOR PRESET +snap-bare-5.mount enabled enabled +snap-core-13308.mount enabled enabled +snap-firefox-1635.mount enabled enabled +snap-fx-158.mount enabled enabled +snap-gimp-393.mount enabled enabled +snap-gtk\x2dcommon\x2dthemes-1535.mount enabled enabled +snap-kubectl-2537.mount enabled enabled +snap-rustup-1027.mount enabled enabled +cups.path enabled enabled +console-setup.service enabled enabled +dmesg.service enabled enabled +getty@.service enabled enabled +grub-initrd-fallback.service enabled enabled +irqbalance.service enabled enabled +keyboard-setup.service enabled enabled +mock-apache2.service enabled enabled +networkd-dispatcher.service enabled enabled +ua-timer.timer enabled enabled +update-notifier-download.timer enabled enabled +update-notifier-motd.timer enabled enabled + +20 unit files listed.' + mock_set_status "$mock" 1 2 + + rune -0 --separate-stderr cscli setup detect + rune -0 jq -c '.setup' <(output) + + # If a call to UnitFoundwas part of the expression and it returned true, + # there is a default journalctl_filter derived from the unit's name. + assert_json '[{datasource:{source:"file",filename:"dummy.log",labels:{type:"apache2"}},detected_service:"apache2"}]' + + # the command was called exactly once + [[ $(mock_get_call_num "$mock") -eq 1 ]] + + # the command was called with the expected parameters + [[ $(mock_get_call_args "$mock" 1) == "list-unit-files --state=enabled,generated,static" ]] + + rune -1 systemctl + + # mock teardown + unlink "${mock_path}/${mocked_command}" + PATH="${PATH/${mock_path}:/}" +} + +# XXX this is the same boilerplate as the previous test, can be simplified +@test "cscli setup detect (snub systemd)" { + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + apache2: + when: + - UnitFound("mock-apache2.service") + datasource: + source: file + filename: dummy.log + labels: + type: apache2 + EOT + + # transparently mock systemctl. It's easier if you can tell the application + # under test which executable to call (in which case just call $mock) but + # here we do the symlink and $PATH dance as an example + mocked_command="systemctl" + + # mock setup + mock="$(mock_create)" + mock_path="${mock%/*}" + mock_file="${mock##*/}" + ln -sf "${mock_path}/${mock_file}" "${mock_path}/${mocked_command}" + + #shellcheck disable=SC2031 + PATH="${mock_path}:${PATH}" + + # we don't really care about the output, it's not used anyway + mock_set_output "$mock" "" + mock_set_status "$mock" 1 2 + + rune -0 --separate-stderr cscli setup detect --snub-systemd + + # setup must not be 'null', but an empty list + assert_json '{setup:[]}' + + # the command was never called + [[ $(mock_get_call_num "$mock") -eq 0 ]] + + rune -0 systemctl + + # mock teardown + unlink "${mock_path}/${mocked_command}" + PATH="${PATH/${mock_path}:/}" +} + +@test "cscli setup detect --force-unit" { + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + apache2: + when: + - UnitFound("force-apache2") + datasource: + source: file + filename: dummy.log + labels: + type: apache2 + apache3: + when: + - UnitFound("force-apache3") + datasource: + source: file + filename: dummy.log + labels: + type: apache3 + EOT + + rune -0 --separate-stderr cscli setup detect --force-unit force-apache2 + rune -0 jq -cS '.setup' <(output) + assert_json '[{datasource:{source:"file",filename:"dummy.log",labels:{"type":"apache2"}},detected_service:"apache2"}]' + + rune -0 --separate-stderr cscli setup detect --force-unit force-apache2,force-apache3 + rune -0 jq -cS '.setup' <(output) + assert_json '[{datasource:{source:"file",filename:"dummy.log",labels:{type:"apache2"}},detected_service:"apache2"},{datasource:{source:"file",filename:"dummy.log",labels:{"type":"apache3"}},detected_service:"apache3"}]' + + # force-unit can be specified multiple times, the order does not matter + rune -0 --separate-stderr cscli setup detect --force-unit force-apache3 --force-unit force-apache2 + rune -0 jq -cS '.setup' <(output) + assert_json '[{datasource:{source:"file",filename:"dummy.log",labels:{type:"apache2"}},detected_service:"apache2"},{datasource:{source:"file",filename:"dummy.log",labels:{type:"apache3"}},detected_service:"apache3"}]' + + rune -1 --separate-stderr cscli setup detect --force-unit mock-doesnotexist + assert_stderr --partial "detecting services: unit(s) forced but not supported: [mock-doesnotexist]" +} + +@test "cscli setup detect (process)" { + # This is harder to mock, because gopsutil requires proc/ to be a mount + # point. So we pick a process that exists for sure. + expected_process=$(basename "$SHELL") + + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + apache2: + when: + - ProcessRunning("${expected_process}") + apache3: + when: + - ProcessRunning("this-does-not-exist") + EOT + + rune -0 --separate-stderr cscli setup detect + rune -0 jq -cS '.setup' <(output) + assert_json '[{detected_service:"apache2"}]' +} + +@test "cscli setup detect --force-process" { + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + apache2: + when: + - ProcessRunning("force-apache2") + apache3: + when: + - ProcessRunning("this-does-not-exist") + EOT + + rune -0 --separate-stderr cscli setup detect --force-process force-apache2 + rune -0 jq -cS '.setup' <(output) + assert_json '[{detected_service:"apache2"}]' +} + +@test "cscli setup detect (acquisition only, no hub items)" { + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + apache2: + when: + - UnitFound("force-apache2") + datasource: + source: file + filename: dummy.log + labels: + type: apache2 + EOT + + rune -0 --separate-stderr cscli setup detect --force-unit force-apache2 + rune -0 jq -cS '.setup' <(output) + assert_json '[{datasource:{source:"file",filename:"dummy.log",labels:{type:"apache2"}},detected_service:"apache2"}]' + + rune -0 --separate-stderr cscli setup detect --force-unit force-apache2 --yaml + assert_output - <<-EOT + setup: + - detected_service: apache2 + datasource: + filename: dummy.log + labels: + type: apache2 + source: file + EOT +} + +@test "cscli setup detect (full acquisition section)" { + skip "not supported yet" + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + foobar: + datasource: + filenames: + - /path/to/log/*.log + exclude_regexps: + - ^/path/to/log/excludeme\.log$ + force_inotify: true + mode: tail + labels: + type: foolog + EOT + + rune -0 cscli setup detect --yaml + assert_output - <<-EOT + setup: + - detected_service: foobar + datasource: + filenames: + - /path/to/log/*.log + exclude_regexps: + - ^/path/to/log/excludeme.log$ + force_inotify: true + mode: tail + labels: + type: foolog + EOT +} + +@test "cscli setup detect + acquis + install (no acquisition, no hub items)" { + # no-op edge case, to make sure we don't crash + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + always: + EOT + + rune -0 --separate-stderr cscli setup detect + assert_json '{setup:[{detected_service:"always"}]}' + setup=$output + rune -0 cscli setup datasources /dev/stdin <<<"$setup" + rune -0 cscli setup install-hub /dev/stdin <<<"$setup" +} + +@test "cscli setup detect (with collections)" { + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + foobar: + when: + - ProcessRunning("force-foobar") + install: + collections: + - crowdsecurity/foobar + qox: + when: + - ProcessRunning("test-qox") + install: + collections: + - crowdsecurity/foobar + apache2: + when: + - ProcessRunning("force-apache2") + install: + collections: + - crowdsecurity/apache2 + EOT + + rune -0 --separate-stderr cscli setup detect --force-process force-apache2,force-foobar + rune -0 jq -Sc '.setup | sort' <(output) + assert_json '[{install:{collections:["crowdsecurity/apache2"]},detected_service:"apache2"},{install:{collections:["crowdsecurity/foobar"]},detected_service:"foobar"}]' +} + +@test "cscli setup detect (with acquisition)" { + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + foobar: + when: + - ProcessRunning("force-foobar") + datasource: + source: file + labels: + type: foobar + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + EOT + + rune -0 --separate-stderr cscli setup detect --force-process force-foobar + rune -0 yq -op '.setup | sort_keys(..)' <(output) + assert_output - <<-EOT + 0.datasource.filenames.0 = /var/log/apache2/*.log + 0.datasource.filenames.1 = /var/log/*http*/*.log + 0.datasource.labels.type = foobar + 0.datasource.source = file + 0.detected_service = foobar + EOT + + rune -1 --separate-stderr cscli setup detect --force-process mock-doesnotexist + assert_stderr --partial "detecting services: process(es) forced but not supported: [mock-doesnotexist]" +} + +@test "cscli setup detect (datasource validation)" { + cat <<-EOT >"${DETECT_YAML}" + version: 1.0 + detect: + foobar: + datasource: + labels: + type: something + EOT + + rune -1 --separate-stderr cscli setup detect + assert_stderr --partial "detecting services: invalid datasource for foobar: source is empty" + + # more datasource-specific tests are in detect_test.go +} + +@test "cscli setup install-hub (dry run)" { + # it's not installed + rune -0 --separate-stderr cscli collections list -o json + rune -0 jq -r '.collections[].name' <(output) + refute_line "crowdsecurity/apache2" + + # we install it + rune -0 --separate-stderr cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/apache2"]}}]}' + assert_output 'dry-run: would install collection crowdsecurity/apache2' + + # still not installed + rune -0 --separate-stderr cscli collections list -o json + rune -0 jq -r '.collections[].name' <(output) + refute_line "crowdsecurity/apache2" +} + +@test "cscli setup install-hub (dry run: install multiple collections)" { + # it's not installed + rune -0 --separate-stderr cscli collections list -o json + rune -0 jq -r '.collections[].name' <(output) + refute_line "crowdsecurity/apache2" + + # we install it + rune -0 --separate-stderr cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/apache2"]}}]}' + assert_output 'dry-run: would install collection crowdsecurity/apache2' + + # still not installed + rune -0 --separate-stderr cscli collections list -o json + rune -0 jq -r '.collections[].name' <(output) + refute_line "crowdsecurity/apache2" +} + +@test "cscli setup install-hub (dry run: install multiple collections, parsers, scenarios, postoverflows)" { + rune -0 --separate-stderr cscli setup install-hub /dev/stdin --dry-run <<< '{"setup":[{"install":{"collections":["crowdsecurity/foo","johndoe/bar"],"parsers":["crowdsecurity/fooparser","johndoe/barparser"],"scenarios":["crowdsecurity/fooscenario","johndoe/barscenario"],"postoverflows":["crowdsecurity/foopo","johndoe/barpo"]}}]}' + assert_line 'dry-run: would install collection crowdsecurity/foo' + assert_line 'dry-run: would install collection johndoe/bar' + assert_line 'dry-run: would install parser crowdsecurity/fooparser' + assert_line 'dry-run: would install parser johndoe/barparser' + assert_line 'dry-run: would install scenario crowdsecurity/fooscenario' + assert_line 'dry-run: would install scenario johndoe/barscenario' + assert_line 'dry-run: would install postoverflow crowdsecurity/foopo' + assert_line 'dry-run: would install postoverflow johndoe/barpo' +} + +@test "cscli setup datasources" { + rune -0 --separate-stderr cscli setup datasources --help + assert_line --partial "--to-dir string write the configuration to a directory, in multiple files" + + # single item + + rune -0 --separate-stderr cscli setup datasources /dev/stdin <<-EOT + setup: + - datasource: + source: file + labels: + type: syslog + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + - /var/log/httpd/*.log + EOT + + # remove diclaimer + rune -0 yq '. head_comment=""' <(output) + assert_output - <<-EOT + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + - /var/log/httpd/*.log + labels: + type: syslog + source: file + EOT + + # multiple items + + rune -0 --separate-stderr cscli setup datasources /dev/stdin <<-EOT + setup: + - datasource: + labels: + type: syslog + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + - /var/log/httpd/*.log + - datasource: + labels: + type: foobar + filenames: + - /var/log/foobar/*.log + - datasource: + labels: + type: barbaz + filenames: + - /path/to/barbaz.log + EOT + + rune -0 yq '. head_comment=""' <(output) + assert_output - <<-EOT + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + - /var/log/httpd/*.log + labels: + type: syslog + --- + filenames: + - /var/log/foobar/*.log + labels: + type: foobar + --- + filenames: + - /path/to/barbaz.log + labels: + type: barbaz + EOT + + # multiple items, to a directory + + # avoid the BATS_TEST_TMPDIR variable, it can have a double // + acquisdir=$(TMPDIR="$BATS_FILE_TMPDIR" mktemp -u) + mkdir "$acquisdir" + + rune -0 cscli setup datasources /dev/stdin --to-dir "$acquisdir" <<-EOT + setup: + - detected_service: apache2 + datasource: + labels: + type: syslog + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + - /var/log/httpd/*.log + - detected_service: foobar + datasource: + labels: + type: foobar + filenames: + - /var/log/foobar/*.log + - detected_service: barbaz + datasource: + labels: + type: barbaz + filenames: + - /path/to/barbaz.log + EOT + + # XXX what if detected_service is missing? + + rune -0 cat "${acquisdir}/setup.apache2.yaml" + rune -0 yq '. head_comment=""' <(output) + assert_output - <<-EOT + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + - /var/log/httpd/*.log + labels: + type: syslog + EOT + + rune -0 cat "${acquisdir}/setup.foobar.yaml" + rune -0 yq '. head_comment=""' <(output) + assert_output - <<-EOT + filenames: + - /var/log/foobar/*.log + labels: + type: foobar + EOT + + rune -0 cat "${acquisdir}/setup.barbaz.yaml" + rune -0 yq '. head_comment=""' <(output) + assert_output - <<-EOT + filenames: + - /path/to/barbaz.log + labels: + type: barbaz + EOT + + rm -rf -- "${acquisdir:?}" + mkdir "$acquisdir" + + # having both filenames and journalctl does not generate two files: the datasource is copied as-is, even if incorrect + + rune -0 cscli setup datasources /dev/stdin --to-dir "$acquisdir" <<-EOT + setup: + - detected_service: apache2 + install: + collections: + - crowdsecurity/apache2 + datasource: + labels: + type: apache2 + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + - /var/log/httpd/*.log + journalctl_filter: + - _SYSTEMD_UNIT=apache2.service + EOT + + rune -0 cat "${acquisdir}/setup.apache2.yaml" + rune -0 yq '. head_comment=""' <(output) + assert_output - <<-EOT + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + - /var/log/httpd/*.log + journalctl_filter: + - _SYSTEMD_UNIT=apache2.service + labels: + type: apache2 + EOT + + # the directory must exist + rune -1 --separate-stderr cscli setup datasources /dev/stdin --to-dir /path/does/not/exist <<< '{}' + assert_stderr --partial "directory /path/does/not/exist does not exist" + + # of course it must be a directory + + touch "${acquisdir}/notadir" + + rune -1 --separate-stderr cscli setup datasources /dev/stdin --to-dir "${acquisdir}/notadir" <<-EOT + setup: + - detected_service: apache2 + datasource: + filenames: + - /var/log/apache2/*.log + EOT + assert_stderr --partial "open ${acquisdir}/notadir/setup.apache2.yaml: not a directory" + + rm -rf -- "${acquisdir:?}" +} + +@test "cscli setup datasources (disclaimer)" { + disclaimer="This file was automatically generated" + + rune -0 --separate-stderr cscli setup datasources /dev/stdin <<<"setup:" + rune -0 yq 'head_comment' <(output) + assert_output --partial "$disclaimer" + + rune -0 --separate-stderr cscli setup datasources /dev/stdin <<-EOT + setup: + - detected_service: something + datasource: + labels: + type: syslog + filenames: + - /var/log/something.log + EOT + rune -0 yq 'head_comment' <(output) + assert_output --partial "$disclaimer" +} + +@test "cscli setup (custom journalctl filter)" { + tempfile=$(TMPDIR="$BATS_TEST_TMPDIR" mktemp) + cat <<-EOT >"${tempfile}" + version: 1.0 + detect: + thewiz: + when: + - UnitFound("thewiz.service") + datasource: + source: journalctl + labels: + type: thewiz + journalctl_filter: + - "SYSLOG_IDENTIFIER=TheWiz" + EOT + + rune -0 --separate-stderr cscli setup detect --detect-config "$tempfile" --force-unit thewiz.service + rune -0 jq -cS '.' <(output) + assert_json '{setup:[{datasource:{source:"journalctl",journalctl_filter:["SYSLOG_IDENTIFIER=TheWiz"],labels:{type:"thewiz"}},detected_service:"thewiz"}]}' + rune -0 --separate-stderr cscli setup datasources <(output) + rune -0 yq '. head_comment=""' <(output) + assert_output - <<-EOT + journalctl_filter: + - SYSLOG_IDENTIFIER=TheWiz + labels: + type: thewiz + source: journalctl + EOT + + rm -f "$tempfile" +} + +@test "cscli setup validate" { + # an empty file is not enough + rune -1 --separate-stderr cscli setup validate /dev/null + assert_output "EOF" + assert_stderr --partial "invalid setup file" + + # this is ok; install nothing + rune -0 --separate-stderr cscli setup validate /dev/stdin <<-EOT + setup: + EOT + refute_output + + rune -1 --separate-stderr cscli setup validate /dev/stdin <<-EOT + se tup: + EOT + assert_output - <<-EOT + [1:1] unknown field "se tup" + > 1 | se tup: + ^ + EOT + assert_stderr --partial "invalid setup file" + + rune -1 --separate-stderr cscli setup validate /dev/stdin <<-EOT + setup: + alsdk al; sdf + EOT + assert_output "while unmarshaling setup file: yaml: line 2: could not find expected ':'" + assert_stderr --partial "invalid setup file" +} + diff --git a/tests/bats/testdata/07_setup/detect.yaml b/tests/bats/testdata/07_setup/detect.yaml new file mode 100644 index 000000000..704dfea8a --- /dev/null +++ b/tests/bats/testdata/07_setup/detect.yaml @@ -0,0 +1,88 @@ +# TODO: windows, use_time_machine, event support (see https://hub.crowdsec.net/author/crowdsecurity/collections/iis) + +--- +version: 1.0 + +detect: + apache2: + when: + - ProcessRunning("apache2") + install: + collections: + - crowdsecurity/apache2 + datasource: + source: file + labels: + type: apache2 + filenames: + - /var/log/apache2/*.log + - /var/log/*http*/*.log + - /var/log/httpd/*.log + + apache2-systemd: + when: + - UnitFound("apache2.service") + - OS.ID != "centos" + install: + collections: + - crowdsecurity/apache2 + datasource: + source: journalctl + journalctl_filter: + - "_SYSTEMD_UNIT=mock-apache2.service" + labels: + type: apache2 + + apache2-systemd-centos: + when: + - UnitFound("httpd.service") + - OS.ID == "centos" + install: + collections: + - crowdsecurity/apache2 + datasource: + source: journalctl + journalctl_filter: + - "_SYSTEMD_UNIT=httpd.service" + + ssh-systemd: + when: + - UnitFound("ssh.service") or UnitFound("ssh.socket") + install: + collections: + - crowdsecurity/apache2 + datasource: + source: journalctl + journalctl_filter: + - "_SYSTEMD_UNIT=ssh.service" + labels: + type: syslog + + linux: + when: + - OS.Family == "linux" + install: + collections: + - crowdsecurity/linux + datasource: + source: file + labels: + type: syslog + filenames: + - /var/log/syslog + - /var/log/kern.log + - /var/log/messages + + freebsd: + when: + - OS.Family == "freebsd" + install: + collections: + - crowdsecurity/freebsd + + windows: + when: + - OS.Family == "windows" + install: + collections: + - crowdsecurity/windows diff --git a/tests/lib/config/config-local b/tests/lib/config/config-local index 8fdd3898e..e7df60ec1 100755 --- a/tests/lib/config/config-local +++ b/tests/lib/config/config-local @@ -63,6 +63,9 @@ config_generate() { cp ../config/context.yaml "${CONFIG_DIR}/console/" + cp ../config/detect.yaml \ + "${HUB_DIR}" + # the default acquis file contains files that are not readable by everyone touch "$LOG_DIR/empty.log" cat <<-EOT >"$CONFIG_DIR/acquis.yaml"