diff --git a/Dockerfile b/Dockerfile index b3b958896..da1c3ab06 100644 --- a/Dockerfile +++ b/Dockerfile @@ -7,9 +7,19 @@ WORKDIR /go/src/crowdsec COPY . . +# Alpine does not ship a static version of re2, we can build it ourselves +# Later versions require 'abseil', which is likewise not available in its static form +ENV RE2_VERSION=2023-03-01 + # wizard.sh requires GNU coreutils -RUN apk add --no-cache git gcc libc-dev make bash gettext binutils-gold coreutils && \ +RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils icu-static re2-dev pkgconfig && \ + wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \ + tar -xzf ${RE2_VERSION}.tar.gz && \ + cd re2-${RE2_VERSION} && \ + make && \ + make install && \ echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \ + cd - && \ make clean release DOCKER_BUILD=1 && \ cd crowdsec-v* && \ ./wizard.sh --docker-mode && \ diff --git a/Dockerfile.debian b/Dockerfile.debian index b4cc4c9ec..10b06befd 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -12,7 +12,7 @@ ENV DEBCONF_NOWARNINGS="yes" # wizard.sh requires GNU coreutils RUN apt-get update && \ - apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \ + apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata libre2-dev && \ echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \ make clean release DOCKER_BUILD=1 && \ cd crowdsec-v* && \ diff --git a/Makefile b/Makefile index 52d5e3efe..d6f1b95f2 100644 --- a/Makefile +++ b/Makefile @@ -3,28 +3,35 @@ include mk/platform.mk BUILD_REQUIRE_GO_MAJOR ?= 1 BUILD_REQUIRE_GO_MINOR ?= 20 +GOCMD = go +GOTEST = $(GOCMD) test + BUILD_CODENAME ?= alphaga CROWDSEC_FOLDER = ./cmd/crowdsec CSCLI_FOLDER = ./cmd/crowdsec-cli/ -HTTP_PLUGIN_FOLDER = plugins/notifications/http -SLACK_PLUGIN_FOLDER = plugins/notifications/slack -SPLUNK_PLUGIN_FOLDER = plugins/notifications/splunk -EMAIL_PLUGIN_FOLDER = plugins/notifications/email -DUMMY_PLUGIN_FOLDER = plugins/notifications/dummy - -HTTP_PLUGIN_BIN = notification-http$(EXT) -SLACK_PLUGIN_BIN = notification-slack$(EXT) -SPLUNK_PLUGIN_BIN = notification-splunk$(EXT) -EMAIL_PLUGIN_BIN = notification-email$(EXT) -DUMMY_PLUGIN_BIN = notification-dummy$(EXT) +PLUGINS ?= $(patsubst ./plugins/notifications/%,%,$(wildcard ./plugins/notifications/*)) +PLUGINS_DIR = ./plugins/notifications CROWDSEC_BIN = crowdsec$(EXT) CSCLI_BIN = cscli$(EXT) +# Directory for the release files +RELDIR = crowdsec-$(BUILD_VERSION) + GO_MODULE_NAME = github.com/crowdsecurity/crowdsec +# see if we have libre2-dev installed for C++ optimizations +RE2_CHECK := $(shell pkg-config --libs re2 2>/dev/null) + +#-------------------------------------- +# +# Define MAKE_FLAGS and LD_OPTS for the sub-makefiles in cmd/ and plugins/ +# + +MAKE_FLAGS = --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)" + LD_OPTS_VARS= \ -X 'github.com/crowdsecurity/go-cs-lib/pkg/version.Version=$(BUILD_VERSION)' \ -X 'github.com/crowdsecurity/go-cs-lib/pkg/version.BuildDate=$(BUILD_TIMESTAMP)' \ @@ -37,33 +44,47 @@ ifneq (,$(DOCKER_BUILD)) LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.System=docker' endif -ifdef BUILD_STATIC -$(warning WARNING: The BUILD_STATIC variable is deprecated and has no effect. Builds are static by default since v1.5.0.) +GO_TAGS := netgo,osusergo,sqlite_omit_load_extension + +ifneq (,$(RE2_CHECK)) +# += adds a space that we don't want +GO_TAGS := $(GO_TAGS),re2_cgo +LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.Libre2=C++' endif export LD_OPTS=-ldflags "-s -w -extldflags '-static' $(LD_OPTS_VARS)" \ - -trimpath -tags netgo,osusergo,sqlite_omit_load_extension + -trimpath -tags $(GO_TAGS) ifneq (,$(TEST_COVERAGE)) LD_OPTS += -cover endif -GOCMD = go -GOTEST = $(GOCMD) test - -RELDIR = crowdsec-$(BUILD_VERSION) - -# flags for sub-makefiles -MAKE_FLAGS = --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)" +#-------------------------------------- .PHONY: build -build: goversion crowdsec cscli plugins +build: pre-build goversion crowdsec cscli plugins + +.PHONY: pre-build +pre-build: +ifdef BUILD_STATIC + $(warning WARNING: The BUILD_STATIC variable is deprecated and has no effect. Builds are static by default since v1.5.0.) +endif + $(info Building $(BUILD_VERSION) ($(BUILD_TAG)) for $(GOOS)/$(GOARCH)) +ifneq (,$(RE2_CHECK)) + $(info Using C++ regexp library) +else + $(info Fallback to WebAssembly regexp library. To use the C++ version, make sure you have installed libre2-dev and pkg-config.) +endif + $(info ) .PHONY: all all: clean test build .PHONY: plugins -plugins: http-plugin slack-plugin splunk-plugin email-plugin dummy-plugin +plugins: + @$(foreach plugin,$(PLUGINS), \ + $(MAKE) -C $(PLUGINS_DIR)/$(plugin) build $(MAKE_FLAGS); \ + ) .PHONY: clean clean: testclean @@ -73,34 +94,18 @@ clean: testclean @$(RM) $(CSCLI_BIN) $(WIN_IGNORE_ERR) @$(RM) *.log $(WIN_IGNORE_ERR) @$(RM) crowdsec-release.tgz $(WIN_IGNORE_ERR) - @$(RM) ./$(HTTP_PLUGIN_FOLDER)/$(HTTP_PLUGIN_BIN) $(WIN_IGNORE_ERR) - @$(RM) ./$(SLACK_PLUGIN_FOLDER)/$(SLACK_PLUGIN_BIN) $(WIN_IGNORE_ERR) - @$(RM) ./$(SPLUNK_PLUGIN_FOLDER)/$(SPLUNK_PLUGIN_BIN) $(WIN_IGNORE_ERR) - @$(RM) ./$(EMAIL_PLUGIN_FOLDER)/$(EMAIL_PLUGIN_BIN) $(WIN_IGNORE_ERR) - @$(RM) ./$(DUMMY_PLUGIN_FOLDER)/$(DUMMY_PLUGIN_BIN) $(WIN_IGNORE_ERR) - + @$(foreach plugin,$(PLUGINS), \ + $(MAKE) -C $(PLUGINS_DIR)/$(plugin) clean $(MAKE_FLAGS); \ + ) +.PHONY: cscli cscli: goversion @$(MAKE) -C $(CSCLI_FOLDER) build $(MAKE_FLAGS) +.PHONY: crowdsec crowdsec: goversion @$(MAKE) -C $(CROWDSEC_FOLDER) build $(MAKE_FLAGS) -http-plugin: goversion - @$(MAKE) -C ./$(HTTP_PLUGIN_FOLDER) build $(MAKE_FLAGS) - -slack-plugin: goversion - @$(MAKE) -C ./$(SLACK_PLUGIN_FOLDER) build $(MAKE_FLAGS) - -splunk-plugin: goversion - @$(MAKE) -C ./$(SPLUNK_PLUGIN_FOLDER) build $(MAKE_FLAGS) - -email-plugin: goversion - @$(MAKE) -C ./$(EMAIL_PLUGIN_FOLDER) build $(MAKE_FLAGS) - -dummy-plugin: goversion - $(MAKE) -C ./$(DUMMY_PLUGIN_FOLDER) build $(MAKE_FLAGS) - .PHONY: testclean testclean: bats-clean @$(RM) pkg/apiserver/ent $(WIN_IGNORE_ERR) @@ -132,35 +137,33 @@ localstack: localstack-stop: docker-compose -f test/localstack/docker-compose.yml down -package-common: +.PHONY: vendor +vendor: + @echo "Vendoring dependencies" + @$(GOCMD) mod vendor + @$(foreach plugin,$(PLUGINS), \ + $(MAKE) -C $(PLUGINS_DIR)/$(plugin) vendor $(MAKE_FLAGS); \ + ) + +.PHONY: package +package: @echo "Building Release to dir $(RELDIR)" @$(MKDIR) $(RELDIR)/cmd/crowdsec @$(MKDIR) $(RELDIR)/cmd/crowdsec-cli - @$(MKDIR) $(RELDIR)/$(HTTP_PLUGIN_FOLDER) - @$(MKDIR) $(RELDIR)/$(SLACK_PLUGIN_FOLDER) - @$(MKDIR) $(RELDIR)/$(SPLUNK_PLUGIN_FOLDER) - @$(MKDIR) $(RELDIR)/$(EMAIL_PLUGIN_FOLDER) - @$(CP) $(CROWDSEC_FOLDER)/$(CROWDSEC_BIN) $(RELDIR)/cmd/crowdsec @$(CP) $(CSCLI_FOLDER)/$(CSCLI_BIN) $(RELDIR)/cmd/crowdsec-cli - @$(CP) ./$(HTTP_PLUGIN_FOLDER)/$(HTTP_PLUGIN_BIN) $(RELDIR)/$(HTTP_PLUGIN_FOLDER) - @$(CP) ./$(SLACK_PLUGIN_FOLDER)/$(SLACK_PLUGIN_BIN) $(RELDIR)/$(SLACK_PLUGIN_FOLDER) - @$(CP) ./$(SPLUNK_PLUGIN_FOLDER)/$(SPLUNK_PLUGIN_BIN) $(RELDIR)/$(SPLUNK_PLUGIN_FOLDER) - @$(CP) ./$(EMAIL_PLUGIN_FOLDER)/$(EMAIL_PLUGIN_BIN) $(RELDIR)/$(EMAIL_PLUGIN_FOLDER) - - @$(CP) ./$(HTTP_PLUGIN_FOLDER)/http.yaml $(RELDIR)/$(HTTP_PLUGIN_FOLDER) - @$(CP) ./$(SLACK_PLUGIN_FOLDER)/slack.yaml $(RELDIR)/$(SLACK_PLUGIN_FOLDER) - @$(CP) ./$(SPLUNK_PLUGIN_FOLDER)/splunk.yaml $(RELDIR)/$(SPLUNK_PLUGIN_FOLDER) - @$(CP) ./$(EMAIL_PLUGIN_FOLDER)/email.yaml $(RELDIR)/$(EMAIL_PLUGIN_FOLDER) + @$(foreach plugin,$(PLUGINS), \ + $(MKDIR) $(RELDIR)/$(PLUGINS_DIR)/$(plugin); \ + $(CP) $(PLUGINS_DIR)/$(plugin)/notification-$(plugin)$(EXT) $(RELDIR)/$(PLUGINS_DIR)/$(plugin); \ + $(CP) $(PLUGINS_DIR)/$(plugin)/$(plugin).yaml $(RELDIR)/$(PLUGINS_DIR)/$(plugin)/; \ + ) @$(CPR) ./config $(RELDIR) @$(CP) wizard.sh $(RELDIR) @$(CP) scripts/test_env.sh $(RELDIR) @$(CP) scripts/test_env.ps1 $(RELDIR) -.PHONY: package -package: package-common @tar cvzf crowdsec-release.tgz $(RELDIR) .PHONY: check_release diff --git a/cmd/crowdsec-cli/metrics.go b/cmd/crowdsec-cli/metrics.go index f2577db8f..1ddf4ff66 100644 --- a/cmd/crowdsec-cli/metrics.go +++ b/cmd/crowdsec-cli/metrics.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "net/http" - "os" "strconv" "strings" "time" @@ -23,6 +22,7 @@ import ( // FormatPrometheusMetrics is a complete rip from prom2json func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error { mfChan := make(chan *dto.MetricFamily, 1024) + errChan := make(chan error, 1) // Start with the DefaultTransport for sane defaults. transport := http.DefaultTransport.(*http.Transport).Clone() @@ -35,14 +35,21 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error defer trace.CatchPanic("crowdsec/ShowPrometheus") err := prom2json.FetchMetricFamilies(url, mfChan, transport) if err != nil { - log.Fatalf("failed to fetch prometheus metrics : %v", err) + errChan <- fmt.Errorf("failed to fetch prometheus metrics: %w", err) + return } + errChan <- nil }() result := []*prom2json.Family{} for mf := range mfChan { result = append(result, prom2json.NewFamily(mf)) } + + if err := <-errChan; err != nil { + return err + } + log.Debugf("Finished reading prometheus output, %d entries", len(result)) /*walk*/ lapi_decisions_stats := map[string]struct { @@ -262,36 +269,44 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error var noUnit bool + +func runMetrics(cmd *cobra.Command, args []string) error { + if err := csConfig.LoadPrometheus(); err != nil { + return fmt.Errorf("failed to load prometheus config: %w", err) + } + + if csConfig.Prometheus == nil { + return fmt.Errorf("prometheus section missing, can't show metrics") + } + + if !csConfig.Prometheus.Enabled { + return fmt.Errorf("prometheus is not enabled, can't show metrics") + } + + if prometheusURL == "" { + prometheusURL = csConfig.Cscli.PrometheusUrl + } + + if prometheusURL == "" { + return fmt.Errorf("no prometheus url, please specify in %s or via -u", *csConfig.FilePath) + } + + err := FormatPrometheusMetrics(color.Output, prometheusURL+"/metrics", csConfig.Cscli.Output) + if err != nil { + return fmt.Errorf("could not fetch prometheus metrics: %w", err) + } + return nil +} + + func NewMetricsCmd() *cobra.Command { - var cmdMetrics = &cobra.Command{ + cmdMetrics := &cobra.Command{ Use: "metrics", Short: "Display crowdsec prometheus metrics.", Long: `Fetch metrics from the prometheus server and display them in a human-friendly way`, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { - if err := csConfig.LoadPrometheus(); err != nil { - log.Fatal(err) - } - if !csConfig.Prometheus.Enabled { - log.Warning("Prometheus is not enabled, can't show metrics") - os.Exit(1) - } - - if prometheusURL == "" { - prometheusURL = csConfig.Cscli.PrometheusUrl - } - - if prometheusURL == "" { - log.Errorf("No prometheus url, please specify in %s or via -u", *csConfig.FilePath) - os.Exit(1) - } - - err := FormatPrometheusMetrics(color.Output, prometheusURL+"/metrics", csConfig.Cscli.Output) - if err != nil { - log.Fatalf("could not fetch prometheus metrics: %s", err) - } - }, + RunE: runMetrics, } cmdMetrics.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url (http://:/metrics)") cmdMetrics.PersistentFlags().BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units") diff --git a/cmd/crowdsec/Makefile b/cmd/crowdsec/Makefile index beee418af..8242f1b49 100644 --- a/cmd/crowdsec/Makefile +++ b/cmd/crowdsec/Makefile @@ -57,7 +57,7 @@ install-conf: install-bin: install -v -m 755 -D "$(CROWDSEC_BIN)" "$(BIN_PREFIX)/$(CROWDSEC_BIN)" || exit -.PHONY: systemd"$(BIN_PREFI"$(BIN_PREFIX)/$(CROWDSEC_BIN)""$(BIN_PREFIX)/$(CROWDSEC_BIN)"X)/$(CROWDSEC_BIN)" +.PHONY: systemd systemd: install CFG=$(CFG_PREFIX) PID=$(PID_DIR) BIN=$(BIN_PREFIX)"/"$(CROWDSEC_BIN) envsubst < ../../config/crowdsec.service > "$(SYSTEMD_PATH_FILE)" systemctl daemon-reload diff --git a/go.mod b/go.mod index 5bed758f6..d64e1c075 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/sirupsen/logrus v1.9.2 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.3 - golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d + golang.org/x/crypto v0.1.0 golang.org/x/mod v0.8.0 google.golang.org/grpc v1.47.0 google.golang.org/protobuf v1.28.1 diff --git a/go.sum b/go.sum index 65c56c28e..8d4506758 100644 --- a/go.sum +++ b/go.sum @@ -1018,8 +1018,9 @@ golang.org/x/crypto v0.0.0-20201216223049-8b5274cf687f/go.mod h1:jdWPYTVW3xRLrWP golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= diff --git a/mk/platform/freebsd.mk b/mk/platform/freebsd.mk index c08c82d6e..600a3926a 100644 --- a/mk/platform/freebsd.mk +++ b/mk/platform/freebsd.mk @@ -1,5 +1,3 @@ # FreeBSD specific MAKE=gmake - -$(info building for FreeBSD) diff --git a/mk/platform/linux.mk b/mk/platform/linux.mk index 0c31e884a..02d38f873 100644 --- a/mk/platform/linux.mk +++ b/mk/platform/linux.mk @@ -1,5 +1,3 @@ # Linux specific MAKE=make - -$(info Building for linux) \ No newline at end of file diff --git a/mk/platform/openbsd.mk b/mk/platform/openbsd.mk index 145b8257f..682221353 100644 --- a/mk/platform/openbsd.mk +++ b/mk/platform/openbsd.mk @@ -1,5 +1,3 @@ # OpenBSD specific MAKE=gmake - -$(info building for OpenBSD) diff --git a/mk/platform/windows.mk b/mk/platform/windows.mk index e4c5e8a12..8e2cdf19b 100644 --- a/mk/platform/windows.mk +++ b/mk/platform/windows.mk @@ -18,5 +18,3 @@ CP=Copy-Item CPR=Copy-Item -Recurse MKDIR=New-Item -ItemType directory WIN_IGNORE_ERR=; exit 0 - -$(info Building for windows) diff --git a/pkg/acquisition/modules/waf/README.md b/pkg/acquisition/modules/waf/README.md index 61e02f8c9..459672f5f 100644 --- a/pkg/acquisition/modules/waf/README.md +++ b/pkg/acquisition/modules/waf/README.md @@ -3,26 +3,31 @@ Ongoing poc for Coraza For config: coraza_inband.conf: -``` +```shell SecRuleEngine On SecRule ARGS:id "@eq 0" "id:1, phase:1,deny, status:403,msg:'Invalid id',log,auditlog" SecRequestBodyAccess On -SecRule REQUEST_BODY "@contains password" "id:100, phase:2,deny, status:403,msg:'Invalid request body',log,auditlog" +SecRule REQUEST_BODY "@contains password" "id:2, phase:2,deny, status:403,msg:'Invalid request body',log,auditlog" ``` coraza_outofband.conf: -``` +```shell SecRuleEngine On -SecRule ARGS:id "@eq 2" "id:2, phase:1,deny, status:403,msg:'Invalid id',log,auditlog" +SecRule ARGS:id "@eq 1" "id:3,phase:1,log,msg:'Invalid id',log,auditlog" +SecRule ARGS:idd "@eq 2" "id:4,phase:1,log,msg:'Invalid id',log,auditlog" SecRequestBodyAccess On -SecRule REQUEST_BODY "@contains totolol" "id:100, phase:2,deny, status:403,msg:'Invalid request body',log,auditlog" +#We know that because we are not cloning the body in waf.go, the outofband rules cannot access body as it has been consumed. +#We are finding a way around this +#SecRule REQUEST_BODY "@contains totolol" "id:4, phase:2,deny,msg:'Invalid request body',log,auditlog" +#SecRule REQUEST_BODY "@contains password" "id:2, phase:2,deny, status:403,msg:'Invalid request body',log,auditlog" + ``` acquis.yaml : -``` +```yaml listen_addr: 127.0.0.1 listen_port: 4241 path: / @@ -30,3 +35,70 @@ source: waf labels: type: waf ``` + +Coraza parser: + +```yaml +onsuccess: next_stage +debug: true +filter: "evt.Parsed.program == 'waf'" +name: crowdsecurity/waf-logs +description: "Parse WAF logs" +statics: + - parsed: cloudtrail_parsed + expression: UnmarshalJSON(evt.Line.Raw, evt.Unmarshaled, 'waf') + - meta: req_uuid + expression: evt.Unmarshaled.waf.req_uuid + - meta: source_ip + expression: evt.Unmarshaled.waf.source_ip + - meta: rule_id + expression: evt.Unmarshaled.waf.rule_id + - meta: action + expression: evt.Unmarshaled.waf.rule_action + - meta: service + value: waf + - parsed: event_type + value: waf_match + +``` + +Coraza trigger scenario: + +```yaml +type: trigger +filter: evt.Parsed.event_type == "waf_match" && evt.Unmarshaled.waf.rule_type == "inband" +debug: true +name: coroza-triggger +description: here we go +blackhole: 2m +labels: + type: exploit + remediation: true +groupby: "evt.Meta.source_ip" +``` + +Coraza leaky scenario: + +```yaml +type: leaky +filter: evt.Parsed.event_type == "waf_match" && evt.Unmarshaled.waf.rule_type == "outofband" +debug: true +name: coroza-leaky +description: here we go +blackhole: 2m +leakspeed: 30s +capacity: 1 +labels: + type: exploit + remediation: true +groupby: "evt.Meta.source_ip" +distinct: evt.Meta.rule_id +``` + + + +To be solved: + - We need to solve the body cloning issue + - Merge w/ hub + + diff --git a/pkg/acquisition/modules/waf/waf.go b/pkg/acquisition/modules/waf/waf.go index 055e382b5..e875e7435 100644 --- a/pkg/acquisition/modules/waf/waf.go +++ b/pkg/acquisition/modules/waf/waf.go @@ -2,10 +2,13 @@ package wafacquisition import ( "context" + "encoding/json" "fmt" "io" "io/ioutil" "net/http" + "strings" + "time" "github.com/corazawaf/coraza/v3" corazatypes "github.com/corazawaf/coraza/v3/types" @@ -13,6 +16,8 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/waf" "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/davecgh/go-spew/spew" + "github.com/google/uuid" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" @@ -140,6 +145,8 @@ func (w *WafSource) Configure(yamlConfig []byte, logger *log.Entry) error { return errors.Wrap(err, "Cannot create WAF") } w.outOfBandWaf = outofbandwaf + log.Printf("OOB -> %s", spew.Sdump(w.outOfBandWaf)) + log.Printf("IB -> %s", spew.Sdump(w.inBandWaf)) //We donĀ“t use the wrapper provided by coraza because we want to fully control what happens when a rule match to send the information in crowdsec w.mux.HandleFunc(w.config.Path, w.wafHandler) @@ -195,12 +202,13 @@ func (w *WafSource) Dump() interface{} { return w } -func processReqWithEngine(waf coraza.WAF, r *http.Request) (*corazatypes.Interruption, error) { - tx := waf.NewTransaction() +func processReqWithEngine(waf coraza.WAF, r *http.Request, uuid string) (*corazatypes.Interruption, corazatypes.Transaction, error) { + var in *corazatypes.Interruption + tx := waf.NewTransactionWithID(uuid) if tx.IsRuleEngineOff() { log.Printf("engine is off") - return nil, nil + return nil, nil, nil } defer func() { @@ -236,63 +244,158 @@ func processReqWithEngine(waf coraza.WAF, r *http.Request) (*corazatypes.Interru tx.AddRequestHeader("Transfer-Encoding", r.TransferEncoding[0]) } - in := tx.ProcessRequestHeaders() + in = tx.ProcessRequestHeaders() + //if we're inband, we should stop here, but for outofband go to the end if in != nil { log.Printf("headerss") - return in, nil + return in, tx, nil } if tx.IsRequestBodyAccessible() { if r.Body != nil && r.Body != http.NoBody { _, _, err := tx.ReadRequestBodyFrom(r.Body) if err != nil { - return nil, errors.Wrap(err, "Cannot read request body") + return nil, nil, errors.Wrap(err, "Cannot read request body") } bodyReader, err := tx.RequestBodyReader() if err != nil { - return nil, errors.Wrap(err, "Cannot read request body") + return nil, nil, errors.Wrap(err, "Cannot read request body") } body := io.MultiReader(bodyReader, r.Body) r.Body = ioutil.NopCloser(body) in, err = tx.ProcessRequestBody() if err != nil { - return nil, errors.Wrap(err, "Cannot process request body") + return nil, nil, errors.Wrap(err, "Cannot process request body") } if in != nil { - log.Printf("nothing here") - return in, nil + log.Printf("exception while processing body") + return in, tx, nil } } } - log.Printf("done") + log.Printf("done -> %d", len(tx.MatchedRules())) + // if in != nil { + // log.Printf("exception while processing req") + // return in, tx, nil + // } + return nil, tx, nil +} - return nil, nil +func (w *WafSource) TxToEvents(tx corazatypes.Transaction, r *http.Request, kind string) ([]types.Event, error) { + evts := []types.Event{} + if tx == nil { + return nil, fmt.Errorf("tx is nil") + } + for idx, rule := range tx.MatchedRules() { + log.Printf("rule %d", idx) + evt, err := w.RuleMatchToEvent(rule, tx, r, kind) + if err != nil { + return nil, errors.Wrap(err, "Cannot convert rule match to event") + } + evts = append(evts, evt) + } + + return evts, nil +} + +// Transforms a coraza interruption to a crowdsec event +func (w *WafSource) RuleMatchToEvent(rule corazatypes.MatchedRule, tx corazatypes.Transaction, r *http.Request, kind string) (types.Event, error) { + evt := types.Event{} + //we might want to change this based on in-band vs out-of-band ? + evt.Type = types.LOG + evt.ExpectMode = types.LIVE + //def needs fixing + evt.Stage = "s00-raw" + evt.Process = true + + //we build a big-ass object that is going to be marshaled in line.raw and unmarshaled later. + //why ? because it's more consistent with the other data-sources etc. and it provides users with flexibility to alter our parsers + CorazaEvent := map[string]interface{}{ + //core rule info + "rule_type": kind, + "rule_id": rule.Rule().ID(), + //"rule_action": tx.Interruption().Action, + "rule_disruptive": rule.Disruptive(), + "rule_tags": rule.Rule().Tags(), + "rule_file": rule.Rule().File(), + "rule_file_line": rule.Rule().Line(), + "rule_revision": rule.Rule().Revision(), + "rule_secmark": rule.Rule().SecMark(), + "rule_accuracy": rule.Rule().Accuracy(), + + //http contextual infos + "upstream_addr": r.RemoteAddr, + "req_uuid": tx.ID(), + "source_ip": strings.Split(rule.ClientIPAddress(), ":")[0], + "uri": rule.URI(), + } + + if tx.Interruption() != nil { + CorazaEvent["rule_action"] = tx.Interruption().Action + } + corazaEventB, err := json.Marshal(CorazaEvent) + if err != nil { + return evt, fmt.Errorf("Unable to marshal coraza alert: %w", err) + } + evt.Line = types.Line{ + Time: time.Now(), + //should we add some info like listen addr/port/path ? + Labels: map[string]string{"type": "waf"}, + Process: true, + Module: "waf", + Src: "waf", + Raw: string(corazaEventB), + } + + return evt, nil } func (w *WafSource) wafHandler(rw http.ResponseWriter, r *http.Request) { log.Printf("yolo here %v", r) + //let's gen a transaction id to keep consistance accross in-band and out-of-band + uuid := uuid.New().String() //inband first - in, err := processReqWithEngine(w.inBandWaf, r) + in, tx, err := processReqWithEngine(w.inBandWaf, r, uuid) if err != nil { //things went south log.Errorf("Error while processing request : %s", err) rw.WriteHeader(http.StatusForbidden) return } if in != nil { - log.Infof("Request blocked by WAF : %+v", in) + events, err := w.TxToEvents(tx, r, "inband") + log.Infof("Request blocked by WAF, %d events to send", len(events)) + for _, evt := range events { + w.outChan <- evt + } + log.Infof("done") + if err != nil { + log.Errorf("Cannot convert transaction to events : %s", err) + rw.WriteHeader(http.StatusForbidden) + return + } rw.WriteHeader(http.StatusForbidden) return } rw.WriteHeader(http.StatusOK) //Now we can do out of band - in2, err := processReqWithEngine(w.outOfBandWaf, r) + in2, tx2, err := processReqWithEngine(w.outOfBandWaf, r, uuid) if err != nil { //things went south log.Errorf("Error while processing request : %s", err) return } - if in2 != nil { + if tx2 != nil && len(tx2.MatchedRules()) > 0 { + log.Printf("got events and stuff to do") + events, err := w.TxToEvents(tx2, r, "outofband") + log.Infof("Request triggered by WAF, %d events to send", len(events)) + for _, evt := range events { + w.outChan <- evt + } + if err != nil { + log.Errorf("Cannot convert transaction to events : %s", err) + } + log.Infof("done") log.Infof("WAF triggered : %+v", in2) return } diff --git a/pkg/csconfig/prometheus.go b/pkg/csconfig/prometheus.go index 31df85110..eea768ab7 100644 --- a/pkg/csconfig/prometheus.go +++ b/pkg/csconfig/prometheus.go @@ -2,7 +2,6 @@ package csconfig import "fmt" -/**/ type PrometheusCfg struct { Enabled bool `yaml:"enabled"` Level string `yaml:"level"` //aggregated|full @@ -16,6 +15,5 @@ func (c *Config) LoadPrometheus() error { c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d", c.Prometheus.ListenAddr, c.Prometheus.ListenPort) } } - return nil } diff --git a/pkg/csconfig/prometheus_test.go b/pkg/csconfig/prometheus_test.go index f7a483d32..3df9c298b 100644 --- a/pkg/csconfig/prometheus_test.go +++ b/pkg/csconfig/prometheus_test.go @@ -1,20 +1,19 @@ package csconfig import ( - "fmt" - "strings" "testing" - "github.com/stretchr/testify/assert" + "github.com/crowdsecurity/go-cs-lib/pkg/cstest" + + "github.com/stretchr/testify/require" ) func TestLoadPrometheus(t *testing.T) { - tests := []struct { - name string - Input *Config - expectedResult string - err string + name string + Input *Config + expectedURL string + expectedErr string }{ { name: "basic valid configuration", @@ -27,29 +26,17 @@ func TestLoadPrometheus(t *testing.T) { }, Cscli: &CscliCfg{}, }, - expectedResult: "http://127.0.0.1:6060", + expectedURL: "http://127.0.0.1:6060", }, } - for idx, test := range tests { - err := test.Input.LoadPrometheus() - if err == nil && test.err != "" { - fmt.Printf("TEST '%s': NOK\n", test.name) - t.Fatalf("%d/%d expected error, didn't get it", idx, len(tests)) - } else if test.err != "" { - if !strings.HasPrefix(fmt.Sprintf("%s", err), test.err) { - fmt.Printf("TEST '%s': NOK\n", test.name) - t.Fatalf("%d/%d expected '%s' got '%s'", idx, len(tests), - test.err, - fmt.Sprintf("%s", err)) - } - } + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + err := tc.Input.LoadPrometheus() + cstest.RequireErrorContains(t, err, tc.expectedErr) - isOk := assert.Equal(t, test.expectedResult, test.Input.Cscli.PrometheusUrl) - if !isOk { - t.Fatalf("test '%s' failed\n", test.name) - } else { - fmt.Printf("TEST '%s': OK\n", test.name) - } + require.Equal(t, tc.expectedURL, tc.Input.Cscli.PrometheusUrl) + }) } } diff --git a/pkg/csplugin/broker_win_test.go b/pkg/csplugin/broker_win_test.go index 8076ce67f..3d7498c0d 100644 --- a/pkg/csplugin/broker_win_test.go +++ b/pkg/csplugin/broker_win_test.go @@ -3,6 +3,9 @@ package csplugin import ( + "bytes" + "encoding/json" + "io" "os" "testing" "time" @@ -15,7 +18,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/models" - "github.com/crowdsecurity/crowdsec/pkg/types" ) /* @@ -81,5 +83,24 @@ func (s *PluginSuite) TestBrokerRun() { time.Sleep(time.Second * 4) assert.FileExists(t, ".\\out") - assert.Equal(t, types.GetLineCountForFile(".\\out"), 2) + + content, err := os.ReadFile("./out") + require.NoError(t, err, "Error reading file") + + decoder := json.NewDecoder(bytes.NewReader(content)) + + var alerts []models.Alert + + // two notifications, one alert each + + err = decoder.Decode(&alerts) + assert.NoError(t, err) + assert.Len(t, alerts, 1) + + err = decoder.Decode(&alerts) + assert.NoError(t, err) + assert.Len(t, alerts, 1) + + err = decoder.Decode(&alerts) + assert.Equal(t, err, io.EOF) } diff --git a/pkg/cwversion/version.go b/pkg/cwversion/version.go index f910159fa..aeac6f2f2 100644 --- a/pkg/cwversion/version.go +++ b/pkg/cwversion/version.go @@ -20,6 +20,7 @@ var ( Constraint_scenario = ">= 1.0, < 3.0" Constraint_api = "v1" Constraint_acquis = ">= 1.0, < 2.0" + Libre2 = "WebAssembly" ) func ShowStr() string { @@ -38,6 +39,7 @@ func Show() { log.Printf("BuildDate: %s", version.BuildDate) log.Printf("GoVersion: %s", version.GoVersion) log.Printf("Platform: %s\n", System) + log.Printf("libre2: %s\n", Libre2) log.Printf("Constraint_parser: %s", Constraint_parser) log.Printf("Constraint_scenario: %s", Constraint_scenario) log.Printf("Constraint_api: %s", Constraint_api) diff --git a/pkg/parser/node.go b/pkg/parser/node.go index 84ec073bb..2370d3796 100644 --- a/pkg/parser/node.go +++ b/pkg/parser/node.go @@ -274,6 +274,10 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri switch out := output.(type) { case string: gstr = out + case int: + gstr = fmt.Sprintf("%d", out) + case float64, float32: + gstr = fmt.Sprintf("%f", out) default: clog.Errorf("unexpected return type for RunTimeValue : %T", output) } diff --git a/pkg/parser/runtime.go b/pkg/parser/runtime.go index cbeee91ee..3c7d382d5 100644 --- a/pkg/parser/runtime.go +++ b/pkg/parser/runtime.go @@ -132,6 +132,8 @@ func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) er value = out case int: value = strconv.Itoa(out) + case float64, float32: + value = fmt.Sprintf("%f", out) case map[string]interface{}: clog.Warnf("Expression '%s' returned a map, please use ToJsonString() to convert it to string if you want to keep it as is, or refine your expression to extract a string", static.ExpValue) case []interface{}: @@ -139,7 +141,7 @@ func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) er case nil: clog.Debugf("Expression '%s' returned nil, skipping", static.ExpValue) default: - clog.Errorf("unexpected return type for RunTimeValue : %T", output) + clog.Errorf("unexpected return type for '%s' : %T", static.ExpValue, output) return errors.New("unexpected return type for RunTimeValue") } } diff --git a/plugins/notifications/dummy/Makefile b/plugins/notifications/dummy/Makefile index d7a74e485..612ec6c86 100644 --- a/plugins/notifications/dummy/Makefile +++ b/plugins/notifications/dummy/Makefile @@ -4,14 +4,20 @@ ifeq ($(OS), Windows_NT) EXT = .exe endif -# Go parameters +PLUGIN = dummy +BINARY_NAME = notification-$(PLUGIN)$(EXT) + GOCMD = go GOBUILD = $(GOCMD) build -BINARY_NAME = notification-dummy$(EXT) - build: clean $(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME) +.PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) + +.PHONY: vendor +vendor: + @echo "vendoring $(PLUGIN) plugin..." + @$(GOCMD) mod vendor diff --git a/plugins/notifications/email/Makefile b/plugins/notifications/email/Makefile index a58f5bc01..a386625ac 100644 --- a/plugins/notifications/email/Makefile +++ b/plugins/notifications/email/Makefile @@ -4,14 +4,20 @@ ifeq ($(OS), Windows_NT) EXT = .exe endif -# Go parameters +PLUGIN = email +BINARY_NAME = notification-$(PLUGIN)$(EXT) + GOCMD = go GOBUILD = $(GOCMD) build -BINARY_NAME = notification-email$(EXT) - build: clean $(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME) +.PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) + +.PHONY: vendor +vendor: + @echo "vendoring $(PLUGIN) plugin..." + @$(GOCMD) mod vendor diff --git a/plugins/notifications/http/Makefile b/plugins/notifications/http/Makefile index 2845d858d..44ee8c58f 100644 --- a/plugins/notifications/http/Makefile +++ b/plugins/notifications/http/Makefile @@ -4,14 +4,20 @@ ifeq ($(OS), Windows_NT) EXT = .exe endif -# Go parameters +PLUGIN=http +BINARY_NAME = notification-$(PLUGIN)$(EXT) + GOCMD = go GOBUILD = $(GOCMD) build -BINARY_NAME = notification-http$(EXT) - build: clean $(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME) +.PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) + +.PHONY: vendor +vendor: + @echo "vendoring $(PLUGIN) plugin..." + @$(GOCMD) mod vendor diff --git a/plugins/notifications/slack/Makefile b/plugins/notifications/slack/Makefile index ca6aa94db..e950eba92 100644 --- a/plugins/notifications/slack/Makefile +++ b/plugins/notifications/slack/Makefile @@ -4,14 +4,20 @@ ifeq ($(OS), Windows_NT) EXT = .exe endif -# Go parameters +PLUGIN=slack +BINARY_NAME = notification-$(PLUGIN)$(EXT) + GOCMD = go GOBUILD = $(GOCMD) build -BINARY_NAME = notification-slack$(EXT) - build: clean $(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME) +.PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) + +.PHONY: vendor +vendor: + @echo "vendoring $(PLUGIN) plugin..." + @$(GOCMD) mod vendor diff --git a/plugins/notifications/splunk/Makefile b/plugins/notifications/splunk/Makefile index a318cc8dd..a49c87bd6 100644 --- a/plugins/notifications/splunk/Makefile +++ b/plugins/notifications/splunk/Makefile @@ -4,14 +4,20 @@ ifeq ($(OS), Windows_NT) EXT = .exe endif -# Go parameters +PLUGIN=splunk +BINARY_NAME = notification-$(PLUGIN)$(EXT) + GOCMD = go GOBUILD = $(GOCMD) build -BINARY_NAME = notification-splunk$(EXT) - build: clean $(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME) +.PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) + +.PHONY: vendor +vendor: + @echo "vendoring $(PLUGIN) plugin..." + @$(GOCMD) mod vendor diff --git a/test/bats.mk b/test/bats.mk index d249f22b9..65bb4a286 100644 --- a/test/bats.mk +++ b/test/bats.mk @@ -24,7 +24,8 @@ DATA_DIR = $(LOCAL_DIR)/var/lib/crowdsec/data LOCAL_INIT_DIR = $(TEST_DIR)/local-init LOG_DIR = $(LOCAL_DIR)/var/log PID_DIR = $(LOCAL_DIR)/var/run -PLUGIN_DIR = $(LOCAL_DIR)/lib/crowdsec/plugins +# do not shadow $(PLUGINS_DIR) from the main Makefile +BATS_PLUGIN_DIR = $(LOCAL_DIR)/lib/crowdsec/plugins DB_BACKEND ?= sqlite CROWDSEC ?= $(BIN_DIR)/crowdsec @@ -43,7 +44,7 @@ export CONFIG_YAML="$(CONFIG_DIR)/config.yaml" export LOCAL_INIT_DIR="$(LOCAL_INIT_DIR)" export LOG_DIR="$(LOG_DIR)" export PID_DIR="$(PID_DIR)" -export PLUGIN_DIR="$(PLUGIN_DIR)" +export PLUGIN_DIR="$(BATS_PLUGIN_DIR)" export DB_BACKEND="$(DB_BACKEND)" export INIT_BACKEND="$(INIT_BACKEND)" export CONFIG_BACKEND="$(CONFIG_BACKEND)" @@ -66,10 +67,10 @@ bats-check-requirements: # Build and installs crowdsec in a local directory. Rebuilds if already exists. bats-build: bats-environment bats-check-requirements - @mkdir -p $(BIN_DIR) $(LOG_DIR) $(PID_DIR) $(PLUGIN_DIR) - @TEST_COVERAGE=$(TEST_COVERAGE) DEFAULT_CONFIGDIR=$(CONFIG_DIR) DEFAULT_DATADIR=$(DATA_DIR) $(MAKE) goversion crowdsec cscli plugins + @$(MKDIR) $(BIN_DIR) $(LOG_DIR) $(PID_DIR) $(BATS_PLUGIN_DIR) + @TEST_COVERAGE=$(TEST_COVERAGE) DEFAULT_CONFIGDIR=$(CONFIG_DIR) DEFAULT_DATADIR=$(DATA_DIR) $(MAKE) build @install -m 0755 cmd/crowdsec/crowdsec cmd/crowdsec-cli/cscli $(BIN_DIR)/ - @install -m 0755 plugins/notifications/*/notification-* $(PLUGIN_DIR)/ + @install -m 0755 plugins/notifications/*/notification-* $(BATS_PLUGIN_DIR)/ # Create a reusable package with initial configuration + data bats-fixture: @@ -99,10 +100,7 @@ bats-lint: @shellcheck --version >/dev/null 2>&1 || (echo "ERROR: shellcheck is required."; exit 1) @shellcheck -x $(TEST_DIR)/bats/*.bats - bats-test-package: bats-environment $(TEST_DIR)/instance-data make $(TEST_DIR)/run-tests $(TEST_DIR)/bats $(TEST_DIR)/run-tests $(TEST_DIR)/dyn-bats - -.PHONY: bats-environment diff --git a/test/bats/01_cscli.bats b/test/bats/01_cscli.bats index c765bf707..a01d936b7 100644 --- a/test/bats/01_cscli.bats +++ b/test/bats/01_cscli.bats @@ -228,7 +228,6 @@ teardown() { assert_output --partial "Route" assert_output --partial '/v1/watchers/login' assert_output --partial "Local Api Metrics:" - } @test "'cscli completion' with or without configuration file" { diff --git a/test/bats/08_metrics.bats b/test/bats/08_metrics.bats new file mode 100644 index 000000000..836e22048 --- /dev/null +++ b/test/bats/08_metrics.bats @@ -0,0 +1,60 @@ +#!/usr/bin/env bats +# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si: + +set -u + +setup_file() { + load "../lib/setup_file.sh" +} + +teardown_file() { + load "../lib/teardown_file.sh" +} + +setup() { + load "../lib/setup.sh" + ./instance-data load +} + +teardown() { + ./instance-crowdsec stop +} + +#---------- + +@test "cscli metrics (crowdsec not running)" { + rune -1 cscli metrics + # crowdsec is down + assert_stderr --partial "failed to fetch prometheus metrics" + assert_stderr --partial "connect: connection refused" +} + +@test "cscli metrics (bad configuration)" { + config_set '.prometheus.foo="bar"' + rune -1 cscli metrics + assert_stderr --partial "field foo not found in type csconfig.PrometheusCfg" +} + +@test "cscli metrics (.prometheus.enabled=false)" { + config_set '.prometheus.enabled=false' + rune -1 cscli metrics + assert_stderr --partial "prometheus is not enabled, can't show metrics" +} + +@test "cscli metrics (missing listen_addr)" { + config_set 'del(.prometheus.listen_addr)' + rune -1 cscli metrics + assert_stderr --partial "no prometheus url, please specify" +} + +@test "cscli metrics (missing listen_port)" { + config_set 'del(.prometheus.listen_addr)' + rune -1 cscli metrics + assert_stderr --partial "no prometheus url, please specify" +} + +@test "cscli metrics (missing prometheus section)" { + config_set 'del(.prometheus)' + rune -1 cscli metrics + assert_stderr --partial "prometheus section missing, can't show metrics" +}