Merge branch 'master' into handle_highAvailability

.github/workflows/bats-hub.yml

@@ -15,7 +15,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.6"]
+        go-version: ["1.21.1"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest
@@ -37,7 +37,6 @@ jobs:
       uses: actions/setup-go@v4
       with:
         go-version: ${{ matrix.go-version }}
-        cache-dependency-path: "**/go.sum"
 
     - name: "Install bats dependencies"
       env:

.github/workflows/bats-mysql.yml

@@ -14,7 +14,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.6"]
+        go-version: ["1.21.1"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest
@@ -44,7 +44,6 @@ jobs:
       uses: actions/setup-go@v4
       with:
         go-version: ${{ matrix.go-version }}
-        cache-dependency-path: "**/go.sum"
 
     - name: "Install bats dependencies"
       env:

.github/workflows/bats-postgres.yml

@@ -10,14 +10,14 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.6"]
+        go-version: ["1.21.1"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest
     timeout-minutes: 30
     services:
       database:
-        image: postgres:15
+        image: postgres:16
         env:
           POSTGRES_PASSWORD: "secret"
         ports:
@@ -30,13 +30,13 @@ jobs:
 
     steps:
 
-    - name: "Install pg_dump v15"
+    - name: "Install pg_dump v16"
       # we can remove this when it's released on ubuntu-latest
       run: |
           sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
           wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /etc/apt/trusted.gpg.d/pgdg.asc &>/dev/null
           sudo apt update
-          sudo apt -qq -y -o=Dpkg::Use-Pty=0 install postgresql-client-15
+          sudo apt -qq -y -o=Dpkg::Use-Pty=0 install postgresql-client-16
 
     - name: "Force machineid"
       run: |
@@ -53,7 +53,6 @@ jobs:
       uses: actions/setup-go@v4
       with:
         go-version: ${{ matrix.go-version }}
-        cache-dependency-path: "**/go.sum"
 
     - name: "Install bats dependencies"
       env:

.github/workflows/bats-sqlite-coverage.yml

@@ -11,7 +11,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.6"]
+        go-version: ["1.21.1"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest
@@ -34,7 +34,6 @@ jobs:
       uses: actions/setup-go@v4
       with:
         go-version: ${{ matrix.go-version }}
-        cache-dependency-path: "**/go.sum"
 
     - name: "Install bats dependencies"
       env:

.github/workflows/ci-windows-build-msi.yml

@@ -23,7 +23,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.6"]
+        go-version: ["1.21.1"]
 
     name: Build
     runs-on: windows-2019
@@ -40,7 +40,6 @@ jobs:
       uses: actions/setup-go@v4
       with:
         go-version: ${{ matrix.go-version }}
-        cache-dependency-path: "**/go.sum"
 
     - name: Build
       run: make windows_installer BUILD_RE2_WASM=1

.github/workflows/codeql-analysis.yml

@@ -45,6 +45,9 @@ jobs:
     steps:
     - name: Checkout repository
       uses: actions/checkout@v3
+      with:
+        # required to pick up tags for BUILD_VERSION
+        fetch-depth: 0
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
@@ -58,8 +61,8 @@ jobs:
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
-    - name: Autobuild
-      uses: github/codeql-action/autobuild@v2
+    # - name: Autobuild
+    #   uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -68,9 +71,14 @@ jobs:
     #    and modify them (or add more) to build your code if your project
     #    uses a compiled language
 
-    #- run: |
-    #   make bootstrap
-    #   make release
+    - name: "Set up Go"
+      uses: actions/setup-go@v4
+      with:
+        go-version: "1.21.0"
+        cache-dependency-path: "**/go.sum"
+
+    - run: |
+       make clean build BUILD_RE2_WASM=1
 
     - name: Perform CodeQL Analysis
       uses: github/codeql-action/analyze@v2

.github/workflows/go-tests-windows.yml

@@ -22,7 +22,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.6"]
+        go-version: ["1.21.1"]
 
     name: "Build + tests"
     runs-on: windows-2022
@@ -39,7 +39,6 @@ jobs:
       uses: actions/setup-go@v4
       with:
         go-version: ${{ matrix.go-version }}
-        cache-dependency-path: "**/go.sum"
 
     - name: Build
       run: |
@@ -61,7 +60,7 @@ jobs:
     - name: golangci-lint
       uses: golangci/golangci-lint-action@v3
       with:
-        version: v1.51
+        version: v1.54
         args: --issues-exit-code=1 --timeout 10m
         only-new-issues: false
         # the cache is already managed above, enabling it here

.github/workflows/go-tests.yml

@@ -34,7 +34,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.6"]
+        go-version: ["1.21.1"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest
@@ -120,7 +120,6 @@ jobs:
       uses: actions/setup-go@v4
       with:
         go-version: ${{ matrix.go-version }}
-        cache-dependency-path: "**/go.sum"
 
     - name: Build and run tests, static
       run: |
@@ -145,7 +144,7 @@ jobs:
     - name: golangci-lint
       uses: golangci/golangci-lint-action@v3
       with:
-        version: v1.51
+        version: v1.54
         args: --issues-exit-code=1 --timeout 10m
         only-new-issues: false
         # the cache is already managed above, enabling it here

.github/workflows/release_publish-package.yml

@@ -14,7 +14,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.20.6"]
+        go-version: ["1.21.1"]
 
     name: Build and upload binary package
     runs-on: ubuntu-latest
@@ -30,7 +30,6 @@ jobs:
         uses: actions/setup-go@v4
         with:
           go-version: ${{ matrix.go-version }}
-          cache-dependency-path: "**/go.sum"
 
       - name: Build the binaries
         run: |
@@ -42,4 +41,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           tag_name="${GITHUB_REF##*/}"
-          hub release edit -a crowdsec-release.tgz -a vendor.tgz -m "" "$tag_name"
+          hub release edit -a crowdsec-release.tgz -a vendor.tgz -a *-vendor.tar.xz -m "" "$tag_name"

.gitignore

@@ -41,11 +41,7 @@ vendor.tgz
 # crowdsec binaries
 cmd/crowdsec-cli/cscli
 cmd/crowdsec/crowdsec
-plugins/notifications/http/notification-http
-plugins/notifications/slack/notification-slack
-plugins/notifications/splunk/notification-splunk
-plugins/notifications/email/notification-email
-plugins/notifications/dummy/notification-dummy
+cmd/notification-*/notification-*
 
 # Test cache (downloaded files)
 .cache

.golangci.yml

@@ -105,6 +105,7 @@ linters:
     # Recommended? (easy)
     #
 
+    - depguard              # Go linter that checks if package imports are in a list of acceptable packages
     - dogsled               # Checks assignments with too many blank identifiers (e.g. x, _, _, _, := f())
     - errchkjson            # Checks types passed to the json encoding functions. Reports unsupported types and optionally reports occations, where the check for the returned error can be omitted.
     - errorlint             # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13.
@@ -121,10 +122,10 @@ linters:
     - nosprintfhostport     # Checks for misuse of Sprintf to construct a host with port in a URL.
     - promlinter            # Check Prometheus metrics naming via promlint
     - revive                # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint.
+    - tagalign              # check that struct tags are well aligned [fast: true, auto-fix: true]
     - thelper               # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers
     - wastedassign          # wastedassign finds wasted assignment statements.
     - wrapcheck             # Checks that errors returned from external packages are wrapped
-    - depguard              # Go linter that checks if package imports are in a list of acceptable packages
 
     #
     # Recommended? (requires some work)
@@ -198,6 +199,18 @@ issues:
         - govet
       text: "shadow: declaration of \"err\" shadows declaration"
 
+    #
+    # typecheck
+    #
+
+    - linters:
+        - typecheck
+      text: "undefined: min"
+
+    - linters:
+        - typecheck
+      text: "undefined: max"
+
     #
     # errcheck
     #

Dockerfile

@@ -1,5 +1,5 @@
 # vim: set ft=dockerfile:
-ARG GOVERSION=1.20.6
+ARG GOVERSION=1.21.1
 
 FROM golang:${GOVERSION}-alpine AS build
 
@@ -52,10 +52,11 @@ FROM slim as plugins
 
 # Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
 # The files are here for reference, as users will need to mount a new version to be actually able to use notifications
-COPY --from=build /go/src/crowdsec/plugins/notifications/email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
-COPY --from=build /go/src/crowdsec/plugins/notifications/http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
-COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
-COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml /staging/etc/crowdsec/notifications/sentinel.yaml
 COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
 
 FROM slim as geoip

Dockerfile.debian

@@ -1,5 +1,5 @@
 # vim: set ft=dockerfile:
-ARG GOVERSION=1.20.6
+ARG GOVERSION=1.21.1
 
 FROM golang:${GOVERSION}-bookworm AS build
 
@@ -68,10 +68,11 @@ FROM slim as plugins
 
 # Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
 # The files are here for reference, as users will need to mount a new version to be actually able to use notifications
-COPY --from=build /go/src/crowdsec/plugins/notifications/email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
-COPY --from=build /go/src/crowdsec/plugins/notifications/http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
-COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
-COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
+COPY --from=build /go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml /staging/etc/crowdsec/notifications/sentinel.yaml
 COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
 
 FROM slim as geoip

Makefile

@@ -23,22 +23,22 @@ BUILD_RE2_WASM ?= 0
 BUILD_STATIC ?= 0
 
 # List of plugins to build
-PLUGINS ?= $(patsubst ./plugins/notifications/%,%,$(wildcard ./plugins/notifications/*))
+PLUGINS ?= $(patsubst ./cmd/notification-%,%,$(wildcard ./cmd/notification-*))
 
 # Can be overriden, if you can deal with the consequences
 BUILD_REQUIRE_GO_MAJOR ?= 1
-BUILD_REQUIRE_GO_MINOR ?= 20
+BUILD_REQUIRE_GO_MINOR ?= 21
 
 #--------------------------------------
 
-GOCMD = go
-GOTEST = $(GOCMD) test
+GO = go
+GOTEST = $(GO) test
 
 BUILD_CODENAME ?= alphaga
 
 CROWDSEC_FOLDER = ./cmd/crowdsec
 CSCLI_FOLDER = ./cmd/crowdsec-cli/
-PLUGINS_DIR = ./plugins/notifications
+PLUGINS_DIR_PREFIX = ./cmd/notification-
 
 CROWDSEC_BIN = crowdsec$(EXT)
 CSCLI_BIN = cscli$(EXT)
@@ -64,15 +64,15 @@ bool = $(if $(filter $(call lc, $1),1 yes true),1,0)
 
 #--------------------------------------
 #
-# Define MAKE_FLAGS and LD_OPTS for the sub-makefiles in cmd/ and plugins/
+# Define MAKE_FLAGS and LD_OPTS for the sub-makefiles in cmd/
 #
 
 MAKE_FLAGS = --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
 
 LD_OPTS_VARS= \
--X 'github.com/crowdsecurity/go-cs-lib/pkg/version.Version=$(BUILD_VERSION)' \
--X 'github.com/crowdsecurity/go-cs-lib/pkg/version.BuildDate=$(BUILD_TIMESTAMP)' \
--X 'github.com/crowdsecurity/go-cs-lib/pkg/version.Tag=$(BUILD_TAG)' \
+-X 'github.com/crowdsecurity/go-cs-lib/version.Version=$(BUILD_VERSION)' \
+-X 'github.com/crowdsecurity/go-cs-lib/version.BuildDate=$(BUILD_TIMESTAMP)' \
+-X 'github.com/crowdsecurity/go-cs-lib/version.Tag=$(BUILD_TAG)' \
 -X '$(GO_MODULE_NAME)/pkg/cwversion.Codename=$(BUILD_CODENAME)' \
 -X '$(GO_MODULE_NAME)/pkg/csconfig.defaultConfigDir=$(DEFAULT_CONFIGDIR)' \
 -X '$(GO_MODULE_NAME)/pkg/csconfig.defaultDataDir=$(DEFAULT_DATADIR)'
@@ -92,7 +92,6 @@ ifeq ($(PKG_CONFIG),)
 endif
 
 ifeq ($(RE2_CHECK),)
-# we could detect the platform and suggest the command to install
 RE2_FAIL := "libre2-dev is not installed, please install it or set BUILD_RE2_WASM=1 to use the WebAssembly version"
 else
 # += adds a space that we don't want
@@ -101,6 +100,7 @@ LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.Libre2=C++'
 endif
 endif
 
+# Build static to avoid the runtime dependency on libre2.so
 ifeq ($(call bool,$(BUILD_STATIC)),1)
 BUILD_TYPE = static
 EXTLDFLAGS := -extldflags '-static'
@@ -109,10 +109,19 @@ BUILD_TYPE = dynamic
 EXTLDFLAGS :=
 endif
 
-export LD_OPTS=-ldflags "-s -w $(EXTLDFLAGS) $(LD_OPTS_VARS)" \
-	-trimpath -tags $(GO_TAGS)
+# Build with debug symbols, and disable optimizations + inlining, to use Delve
+ifeq ($(call bool,$(DEBUG)),1)
+STRIP_SYMBOLS :=
+DISABLE_OPTIMIZATION := -gcflags "-N -l"
+else
+STRIP_SYMBOLS := -s -w
+DISABLE_OPTIMIZATION :=
+endif
 
-ifneq (,$(TEST_COVERAGE))
+export LD_OPTS=-ldflags "$(STRIP_SYMBOLS) $(EXTLDFLAGS) $(LD_OPTS_VARS)" \
+	-trimpath -tags $(GO_TAGS) $(DISABLE_OPTIMIZATION)
+
+ifeq ($(call bool,$(TEST_COVERAGE)),1)
 LD_OPTS += -cover
 endif
 
@@ -135,19 +144,47 @@ ifneq (,$(RE2_CHECK))
 else
 	$(info Fallback to WebAssembly regexp library. To use the C++ version, make sure you have installed libre2-dev and pkg-config.)
 endif
+
+ifeq ($(call bool,$(DEBUG)),1)
+	$(info Building with debug symbols and disabled optimizations)
+endif
+
+ifeq ($(call bool,$(TEST_COVERAGE)),1)
+	$(info Test coverage collection enabled)
+endif
+
 	$(info )
 
+
 .PHONY: all
 all: clean test build
 
 .PHONY: plugins
 plugins:
 	@$(foreach plugin,$(PLUGINS), \
-		$(MAKE) -C $(PLUGINS_DIR)/$(plugin) build $(MAKE_FLAGS); \
+		$(MAKE) -C $(PLUGINS_DIR_PREFIX)$(plugin) build $(MAKE_FLAGS); \
 	)
 
+# same as "$(MAKE) -f debian/rules clean" but without the dependency on debhelper
+.PHONY: clean-debian
+clean-debian:
+	@$(RM) -r debian/crowdsec
+	@$(RM) -r debian/crowdsec
+	@$(RM) -r debian/files
+	@$(RM) -r debian/.debhelper
+	@$(RM) -r debian/*.substvars
+	@$(RM) -r debian/*-stamp
+
+.PHONY: clean-rpm
+clean-rpm:
+	@$(RM) -r rpm/BUILD
+	@$(RM) -r rpm/BUILDROOT
+	@$(RM) -r rpm/RPMS
+	@$(RM) -r rpm/SOURCES/*.tar.gz
+	@$(RM) -r rpm/SRPMS
+
 .PHONY: clean
-clean: testclean
+clean: clean-debian clean-rpm testclean
 	@$(MAKE) -C $(CROWDSEC_FOLDER) clean $(MAKE_FLAGS)
 	@$(MAKE) -C $(CSCLI_FOLDER) clean $(MAKE_FLAGS)
 	@$(RM) $(CROWDSEC_BIN) $(WIN_IGNORE_ERR)
@@ -155,7 +192,7 @@ clean: testclean
 	@$(RM) *.log $(WIN_IGNORE_ERR)
 	@$(RM) crowdsec-release.tgz $(WIN_IGNORE_ERR)
 	@$(foreach plugin,$(PLUGINS), \
-		$(MAKE) -C $(PLUGINS_DIR)/$(plugin) clean $(MAKE_FLAGS); \
+		$(MAKE) -C $(PLUGINS_DIR_PREFIX)$(plugin) clean $(MAKE_FLAGS); \
 	)
 
 .PHONY: cscli
@@ -166,6 +203,12 @@ cscli: goversion
 crowdsec: goversion
 	@$(MAKE) -C $(CROWDSEC_FOLDER) build $(MAKE_FLAGS)
 
+.PHONY: notification-email
+notification-email: goversion
+	@$(MAKE) -C cmd/notification-email build $(MAKE_FLAGS)
+
+
+
 .PHONY: testclean
 testclean: bats-clean
 	@$(RM) pkg/apiserver/ent $(WIN_IGNORE_ERR)
@@ -201,27 +244,17 @@ localstack:
 localstack-stop:
 	docker-compose -f test/localstack/docker-compose.yml down
 
-# list of plugins that contain go.mod
-PLUGIN_VENDOR = $(foreach plugin,$(PLUGINS),$(shell if [ -f $(PLUGINS_DIR)/$(plugin)/go.mod ]; then echo $(PLUGINS_DIR)/$(plugin); fi))
-
 # build vendor.tgz to be distributed with the release
 .PHONY: vendor
-vendor:
-	$(foreach plugin_dir,$(PLUGIN_VENDOR), \
-		cd $(plugin_dir) >/dev/null && \
-		$(GOCMD) mod vendor && \
-		cd - >/dev/null; \
-	)
-	$(GOCMD) mod vendor
-	tar -czf vendor.tgz vendor $(foreach plugin_dir,$(PLUGIN_VENDOR),$(plugin_dir)/vendor)
+vendor: vendor-remove
+	$(GO) mod vendor
+	tar czf vendor.tgz vendor
+	tar --create --auto-compress --file=$(RELDIR)-vendor.tar.xz vendor
 
 # remove vendor directories and vendor.tgz
 .PHONY: vendor-remove
 vendor-remove:
-	$(foreach plugin_dir,$(PLUGIN_VENDOR), \
-		$(RM) $(plugin_dir)/vendor; \
-	)
-	$(RM) vendor vendor.tgz
+	$(RM) vendor vendor.tgz *-vendor.tar.xz
 
 .PHONY: package
 package:
@@ -232,9 +265,9 @@ package:
 	@$(CP) $(CSCLI_FOLDER)/$(CSCLI_BIN) $(RELDIR)/cmd/crowdsec-cli
 
 	@$(foreach plugin,$(PLUGINS), \
-		$(MKDIR) $(RELDIR)/$(PLUGINS_DIR)/$(plugin); \
-		$(CP) $(PLUGINS_DIR)/$(plugin)/notification-$(plugin)$(EXT) $(RELDIR)/$(PLUGINS_DIR)/$(plugin); \
-		$(CP) $(PLUGINS_DIR)/$(plugin)/$(plugin).yaml $(RELDIR)/$(PLUGINS_DIR)/$(plugin)/; \
+		$(MKDIR) $(RELDIR)/$(PLUGINS_DIR_PREFIX)$(plugin); \
+		$(CP) $(PLUGINS_DIR_PREFIX)$(plugin)/notification-$(plugin)$(EXT) $(RELDIR)/$(PLUGINS_DIR_PREFIX)$(plugin); \
+		$(CP) $(PLUGINS_DIR_PREFIX)$(plugin)/$(plugin).yaml $(RELDIR)/$(PLUGINS_DIR_PREFIX)$(plugin)/; \
 	)
 
 	@$(CPR) ./config $(RELDIR)

azure-pipelines.yml

@@ -27,7 +27,7 @@ stages:
           - task: GoTool@0
             displayName: "Install Go 1.20"
             inputs:
-                version: '1.20.6'
+                version: '1.21.1'
 
           - pwsh: |
               choco install -y make

cmd/crowdsec-cli/Makefile

@@ -4,10 +4,8 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-# Go parameters
-GOCMD = go
-GOBUILD = $(GOCMD) build
-GOTEST = $(GOCMD) test
+GO = go
+GOBUILD = $(GO) build
 
 BINARY_NAME = cscli$(EXT)
 PREFIX ?= "/"
@@ -17,7 +15,7 @@ BIN_PREFIX = $(PREFIX)"/usr/local/bin/"
 all: clean build
 
 build: clean
-	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
+	$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
 
 .PHONY: install
 install: install-conf install-bin

cmd/crowdsec-cli/alerts.go

@@ -19,12 +19,14 @@ import (
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/version"
+	"github.com/crowdsecurity/go-cs-lib/version"
 
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/models"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
 func DecisionsFromAlert(alert *models.Alert) string {
@@ -124,6 +126,12 @@ func AlertsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
 		}
 		csvwriter.Flush()
 	} else if csConfig.Cscli.Output == "json" {
+		if *alerts == nil {
+			// avoid returning "null" in json
+			// could be cleaner if we used slice of alerts directly
+			fmt.Println("[]")
+			return nil
+		}
 		x, _ := json.MarshalIndent(alerts, "", " ")
 		fmt.Printf("%s", string(x))
 	} else if csConfig.Cscli.Output == "human" {
@@ -206,6 +214,7 @@ func NewAlertsCmd() *cobra.Command {
 		Short:             "Manage alerts",
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
+		Aliases:           []string{"alert"},
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			var err error
 			if err := csConfig.LoadAPIClient(); err != nil {
@@ -525,8 +534,8 @@ func NewAlertsFlushCmd() *cobra.Command {
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			var err error
-			if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
-				return fmt.Errorf("local API is disabled, please run this command on the local API machine")
+			if err := require.LAPI(csConfig); err != nil {
+				return err
 			}
 			dbClient, err = database.NewClient(csConfig.DbConfig)
 			if err != nil {

cmd/crowdsec-cli/bouncers.go

@@ -5,17 +5,20 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
+	"slices"
 	"strings"
 	"time"
 
+	"github.com/AlecAivazis/survey/v2"
 	"github.com/fatih/color"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"golang.org/x/exp/slices"
 
 	middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
 func getBouncers(out io.Writer, dbClient *database.Client) error {
@@ -58,8 +61,7 @@ func getBouncers(out io.Writer, dbClient *database.Client) error {
 func NewBouncersListCmd() *cobra.Command {
 	cmdBouncersList := &cobra.Command{
 		Use:               "list",
-		Short:             "List bouncers",
-		Long:              `List bouncers`,
+		Short:             "list all bouncers within the database",
 		Example:           `cscli bouncers list`,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
@@ -126,8 +128,7 @@ func runBouncersAdd(cmd *cobra.Command, args []string) error {
 func NewBouncersAddCmd() *cobra.Command {
 	cmdBouncersAdd := &cobra.Command{
 		Use:   "add MyBouncerName [--length 16]",
-		Short: "add bouncer",
-		Long:  `add bouncer`,
+		Short: "add a single bouncer to the database",
 		Example: `cscli bouncers add MyBouncerName
 cscli bouncers add MyBouncerName -l 24
 cscli bouncers add MyBouncerName -k <random-key>`,
@@ -159,7 +160,7 @@ func runBouncersDelete(cmd *cobra.Command, args []string) error {
 func NewBouncersDeleteCmd() *cobra.Command {
 	cmdBouncersDelete := &cobra.Command{
 		Use:               "delete MyBouncerName",
-		Short:             "delete bouncer",
+		Short:             "delete bouncer(s) from the database",
 		Args:              cobra.MinimumNArgs(1),
 		Aliases:           []string{"remove"},
 		DisableAutoGenTag: true,
@@ -188,11 +189,81 @@ func NewBouncersDeleteCmd() *cobra.Command {
 	return cmdBouncersDelete
 }
 
+func NewBouncersPruneCmd() *cobra.Command {
+	var parsedDuration time.Duration
+	cmdBouncersPrune := &cobra.Command{
+		Use:               "prune",
+		Short:             "prune multiple bouncers from the database",
+		Args:              cobra.NoArgs,
+		DisableAutoGenTag: true,
+		Example: `cscli bouncers prune -d 60m
+cscli bouncers prune -d 60m --force`,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			dur, _ := cmd.Flags().GetString("duration")
+			var err error
+			parsedDuration, err = time.ParseDuration(fmt.Sprintf("-%s", dur))
+			if err != nil {
+				return fmt.Errorf("unable to parse duration '%s': %s", dur, err)
+			}
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			force, _ := cmd.Flags().GetBool("force")
+			if parsedDuration >= 0-2*time.Minute {
+				var answer bool
+				prompt := &survey.Confirm{
+					Message: "The duration you provided is less than or equal 2 minutes this may remove active bouncers continue ?",
+					Default: false,
+				}
+				if err := survey.AskOne(prompt, &answer); err != nil {
+					return fmt.Errorf("unable to ask about prune check: %s", err)
+				}
+				if !answer {
+					fmt.Println("user aborted prune no changes were made")
+					return nil
+				}
+			}
+			bouncers, err := dbClient.QueryBouncersLastPulltimeLT(time.Now().UTC().Add(parsedDuration))
+			if err != nil {
+				return fmt.Errorf("unable to query bouncers: %s", err)
+			}
+			if len(bouncers) == 0 {
+				fmt.Println("no bouncers to prune")
+				return nil
+			}
+			getBouncersTable(color.Output, bouncers)
+			if !force {
+				var answer bool
+				prompt := &survey.Confirm{
+					Message: "You are about to PERMANENTLY remove the above bouncers from the database these will NOT be recoverable, continue ?",
+					Default: false,
+				}
+				if err := survey.AskOne(prompt, &answer); err != nil {
+					return fmt.Errorf("unable to ask about prune check: %s", err)
+				}
+				if !answer {
+					fmt.Println("user aborted prune no changes were made")
+					return nil
+				}
+			}
+			nbDeleted, err := dbClient.BulkDeleteBouncers(bouncers)
+			if err != nil {
+				return fmt.Errorf("unable to prune bouncers: %s", err)
+			}
+			fmt.Printf("successfully delete %d bouncers\n", nbDeleted)
+			return nil
+		},
+	}
+	cmdBouncersPrune.Flags().StringP("duration", "d", "60m", "duration of time since last pull")
+	cmdBouncersPrune.Flags().Bool("force", false, "force prune without asking for confirmation")
+	return cmdBouncersPrune
+}
+
 func NewBouncersCmd() *cobra.Command {
 	var cmdBouncers = &cobra.Command{
 		Use:   "bouncers [action]",
 		Short: "Manage bouncers [requires local API]",
-		Long: `To list/add/delete bouncers.
+		Long: `To list/add/delete/prune bouncers.
 Note: This command requires database direct access, so is intended to be run on Local API/master.
 `,
 		Args:              cobra.MinimumNArgs(1),
@@ -200,9 +271,10 @@ Note: This command requires database direct access, so is intended to be run on
 		DisableAutoGenTag: true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			var err error
-			if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
-				return fmt.Errorf("local API is disabled, please run this command on the local API machine")
+			if err = require.LAPI(csConfig); err != nil {
+				return err
 			}
+
 			dbClient, err = database.NewClient(csConfig.DbConfig)
 			if err != nil {
 				return fmt.Errorf("unable to create new database client: %s", err)
@@ -214,6 +286,7 @@ Note: This command requires database direct access, so is intended to be run on
 	cmdBouncers.AddCommand(NewBouncersListCmd())
 	cmdBouncers.AddCommand(NewBouncersAddCmd())
 	cmdBouncers.AddCommand(NewBouncersDeleteCmd())
+	cmdBouncers.AddCommand(NewBouncersPruneCmd())
 
 	return cmdBouncers
 }

cmd/crowdsec-cli/capi.go

@@ -11,7 +11,7 @@ import (
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/version"
+	"github.com/crowdsecurity/go-cs-lib/version"
 
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
@@ -19,6 +19,8 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/fflag"
 	"github.com/crowdsecurity/crowdsec/pkg/models"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
 const CAPIBaseURL string = "https://api.crowdsec.net/"
@@ -31,14 +33,12 @@ func NewCapiCmd() *cobra.Command {
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadAPIServer(); err != nil {
-				return fmt.Errorf("local API is disabled, please run this command on the local API machine: %w", err)
-			}
-			if csConfig.DisableAPI {
-				return nil
+			if err := require.LAPI(csConfig); err != nil {
+				return err
 			}
-			if csConfig.API.Server.OnlineClient == nil {
-				log.Fatalf("no configuration for Central API in '%s'", *csConfig.FilePath)
+
+			if err := require.CAPI(csConfig); err != nil {
+				return err
 			}
 
 			return nil
@@ -134,10 +134,6 @@ func NewCapiStatusCmd() *cobra.Command {
 		Args:              cobra.MinimumNArgs(0),
 		DisableAutoGenTag: true,
 		Run: func(cmd *cobra.Command, args []string) {
-			var err error
-			if csConfig.API.Server == nil {
-				log.Fatal("There is no configuration on 'api.server:'")
-			}
 			if csConfig.API.Server.OnlineClient == nil {
 				log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
 			}
@@ -160,7 +156,7 @@ func NewCapiStatusCmd() *cobra.Command {
 				log.Info("Run 'sudo cscli hub update' to get the hub index")
 				log.Fatalf("Failed to load hub index : %s", err)
 			}
-			scenarios, err := cwhub.GetInstalledScenariosAsString()
+			scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
 			if err != nil {
 				log.Fatalf("failed to get scenarios : %s", err)
 			}

cmd/crowdsec-cli/collections.go

@@ -7,6 +7,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
@@ -20,20 +21,8 @@ func NewCollectionsCmd() *cobra.Command {
 		Aliases:           []string{"collection"},
 		DisableAutoGenTag: true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadHub(); err != nil {
-				log.Fatal(err)
-			}
-			if csConfig.Hub == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			if err := cwhub.SetHubBranch(); err != nil {
-				return fmt.Errorf("error while setting hub branch: %s", err)
-			}
-
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to get Hub index : %v", err)
+			if err := require.Hub(csConfig); err != nil {
+				return err
 			}
 
 			return nil
@@ -47,6 +36,7 @@ func NewCollectionsCmd() *cobra.Command {
 	}
 
 	var ignoreError bool
+
 	var cmdCollectionsInstall = &cobra.Command{
 		Use:     "install collection",
 		Short:   "Install given collection(s)",
@@ -57,7 +47,7 @@ func NewCollectionsCmd() *cobra.Command {
 			return compAllItems(cwhub.COLLECTIONS, args, toComplete)
 		},
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			for _, name := range args {
 				t := cwhub.GetItem(cwhub.COLLECTIONS, name)
 				if t == nil {
@@ -67,11 +57,12 @@ func NewCollectionsCmd() *cobra.Command {
 				}
 				if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
 					if !ignoreError {
-						log.Fatalf("Error while installing '%s': %s", name, err)
+						return fmt.Errorf("error while installing '%s': %w", name, err)
 					}
 					log.Errorf("Error while installing '%s': %s", name, err)
 				}
 			}
+			return nil
 		},
 	}
 	cmdCollectionsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
@@ -89,21 +80,21 @@ func NewCollectionsCmd() *cobra.Command {
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if all {
 				cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, forceAction)
-				return
+				return nil
 			}
 
 			if len(args) == 0 {
-				log.Fatal("Specify at least one collection to remove or '--all' flag.")
+				return fmt.Errorf("specify at least one collection to remove or '--all'")
 			}
 
 			for _, name := range args {
 				if !forceAction {
 					item := cwhub.GetItem(cwhub.COLLECTIONS, name)
 					if item == nil {
-						log.Fatalf("unable to retrieve: %s\n", name)
+						return fmt.Errorf("unable to retrieve: %s", name)
 					}
 					if len(item.BelongsToCollections) > 0 {
 						log.Warningf("%s belongs to other collections :\n%s\n", name, item.BelongsToCollections)
@@ -113,6 +104,7 @@ func NewCollectionsCmd() *cobra.Command {
 				}
 				cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, forceAction)
 			}
+			return nil
 		},
 	}
 	cmdCollectionsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
@@ -129,17 +121,18 @@ func NewCollectionsCmd() *cobra.Command {
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if all {
 				cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
 			} else {
 				if len(args) == 0 {
-					log.Fatalf("no target collection to upgrade")
+					return fmt.Errorf("specify at least one collection to upgrade or '--all'")
 				}
 				for _, name := range args {
 					cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, forceAction)
 				}
 			}
+			return nil
 		},
 	}
 	cmdCollectionsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the collections")

cmd/crowdsec-cli/config_backup.go

@@ -8,10 +8,11 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
-/* Backup crowdsec configurations to directory <dirPath> :
+/*
+	Backup crowdsec configurations to directory <dirPath>:
 
 - Main config (config.yaml)
 - Profiles config (profiles.yaml)
@@ -19,6 +20,7 @@ import (
 - Backup of API credentials (local API and online API)
 - List of scenarios, parsers, postoverflows and collections that are up-to-date
 - Tainted/local/out-of-date scenarios, parsers, postoverflows and collections
+- Acquisition files (acquis.yaml, acquis.d/*.yaml)
 */
 func backupConfigToDirectory(dirPath string) error {
 	var err error
@@ -128,15 +130,10 @@ func backupConfigToDirectory(dirPath string) error {
 }
 
 func runConfigBackup(cmd *cobra.Command, args []string) error {
-	if err := csConfig.LoadHub(); err != nil {
+	if err := require.Hub(csConfig); err != nil {
 		return err
 	}
 
-	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-		log.Info("Run 'sudo cscli hub update' to get the hub index")
-		return fmt.Errorf("failed to get Hub index: %w", err)
-	}
-
 	if err := backupConfigToDirectory(args[0]); err != nil {
 		return fmt.Errorf("failed to backup config: %w", err)
 	}

cmd/crowdsec-cli/config_restore.go

@@ -11,8 +11,8 @@ import (
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
 type OldAPICfg struct {
@@ -20,7 +20,8 @@ type OldAPICfg struct {
 	Password  string `json:"password"`
 }
 
-/* Restore crowdsec configurations to directory <dirPath> :
+/*
+	Restore crowdsec configurations to directory <dirPath>:
 
 - Main config (config.yaml)
 - Profiles config (profiles.yaml)
@@ -28,6 +29,7 @@ type OldAPICfg struct {
 - Backup of API credentials (local API and online API)
 - List of scenarios, parsers, postoverflows and collections that are up-to-date
 - Tainted/local/out-of-date scenarios, parsers, postoverflows and collections
+- Acquisition files (acquis.yaml, acquis.d/*.yaml)
 */
 func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
 	var err error
@@ -111,7 +113,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
 
 	/*if there is a acquisition dir, restore its content*/
 	if csConfig.Crowdsec.AcquisitionDirPath != "" {
-		if err = os.Mkdir(csConfig.Crowdsec.AcquisitionDirPath, 0o700); err != nil {
+		if err = os.MkdirAll(csConfig.Crowdsec.AcquisitionDirPath, 0o700); err != nil {
 			return fmt.Errorf("error while creating %s : %s", csConfig.Crowdsec.AcquisitionDirPath, err)
 		}
 	}
@@ -181,15 +183,10 @@ func runConfigRestore(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	if err := csConfig.LoadHub(); err != nil {
+	if err := require.Hub(csConfig); err != nil {
 		return err
 	}
 
-	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-		log.Info("Run 'sudo cscli hub update' to get the hub index")
-		return fmt.Errorf("failed to get Hub index: %w", err)
-	}
-
 	if err := restoreConfigFromDirectory(args[0], oldBackup); err != nil {
 		return fmt.Errorf("failed to restore config from %s: %w", args[0], err)
 	}

cmd/crowdsec-cli/config_show.go

@@ -57,7 +57,6 @@ func showConfigKey(key string) error {
 var configShowTemplate = `Global:
 
 {{- if .ConfigPaths }}
-   - Configuration Folder   : {{.ConfigPaths.ConfigDir}}
    - Configuration Folder   : {{.ConfigPaths.ConfigDir}}
    - Data Folder            : {{.ConfigPaths.DataDir}}
    - Hub Folder             : {{.ConfigPaths.HubDir}}
@@ -71,7 +70,7 @@ var configShowTemplate = `Global:
 {{- end }}
 
 {{- if .Crowdsec }}
-Crowdsec:
+Crowdsec{{if and .Crowdsec.Enable (not (ValueBool .Crowdsec.Enable))}} (disabled){{end}}:
   - Acquisition File        : {{.Crowdsec.AcquisitionFilePath}}
   - Parsers routines        : {{.Crowdsec.ParserRoutinesCount}}
 {{- if .Crowdsec.AcquisitionDirPath }}
@@ -97,7 +96,7 @@ API Client:
 {{- end }}
 
 {{- if .API.Server }}
-Local API Server:
+Local API Server{{if and .API.Server.Enable (not (ValueBool .API.Server.Enable))}} (disabled){{end}}:
   - Listen URL              : {{.API.Server.ListenURI}}
   - Profile File            : {{.API.Server.ProfilesPath}}
 
@@ -164,6 +163,12 @@ Central API:
       - User                : {{.DbConfig.User}}
       - DB Name             : {{.DbConfig.DbName}}
 {{- end }}
+{{- if .DbConfig.MaxOpenConns }}
+      - Max Open Conns      : {{.DbConfig.MaxOpenConns}}
+{{- end }}
+{{- if ne .DbConfig.DecisionBulkSize 0 }}
+      - Decision Bulk Size  : {{.DbConfig.DecisionBulkSize}}
+{{- end }}
 {{- if .DbConfig.Flush }}
 {{- if .DbConfig.Flush.MaxAge }}
       - Flush age           : {{.DbConfig.Flush.MaxAge}}
@@ -194,7 +199,15 @@ func runConfigShow(cmd *cobra.Command, args []string) error {
 
 	switch csConfig.Cscli.Output {
 	case "human":
-		tmp, err := template.New("config").Parse(configShowTemplate)
+		// The tests on .Enable look funny because the option has a true default which has
+		// not been set yet (we don't really load the LAPI) and go templates don't dereference
+		// pointers in boolean tests. Prefix notation is the cherry on top.
+		funcs := template.FuncMap{
+			// can't use generics here
+			"ValueBool": func(b *bool) bool { return b!=nil && *b },
+		}
+
+		tmp, err := template.New("config").Funcs(funcs).Parse(configShowTemplate)
 		if err != nil {
 			return err
 		}

cmd/crowdsec-cli/console.go

@@ -4,9 +4,7 @@ import (
 	"context"
 	"encoding/csv"
 	"encoding/json"
-	"errors"
 	"fmt"
-	"io/fs"
 	"net/url"
 	"os"
 
@@ -16,14 +14,16 @@ import (
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v3"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/ptr"
-	"github.com/crowdsecurity/go-cs-lib/pkg/version"
+	"github.com/crowdsecurity/go-cs-lib/ptr"
+	"github.com/crowdsecurity/go-cs-lib/version"
 
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/fflag"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
 func NewConsoleCmd() *cobra.Command {
@@ -33,24 +33,14 @@ func NewConsoleCmd() *cobra.Command {
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
-				var fdErr *fs.PathError
-				if errors.As(err, &fdErr) {
-					log.Fatalf("Unable to load Local API : %s", fdErr)
-				}
-				if err != nil {
-					log.Fatalf("Unable to load required Local API Configuration : %s", err)
-				}
-				log.Fatal("Local API is disabled, please run this command on the local API machine")
+			if err := require.LAPI(csConfig); err != nil {
+				return err
 			}
-			if csConfig.DisableAPI {
-				log.Fatal("Local API is disabled, please run this command on the local API machine")
+			if err := require.CAPI(csConfig); err != nil {
+				return err
 			}
-			if csConfig.API.Server.OnlineClient == nil {
-				log.Fatalf("No configuration for Central API (CAPI) in '%s'", *csConfig.FilePath)
-			}
-			if csConfig.API.Server.OnlineClient.Credentials == nil {
-				log.Fatal("You must configure Central API (CAPI) with `cscli capi register` before accessing console features.")
+			if err := require.CAPIRegistered(csConfig); err != nil {
+				return err
 			}
 			return nil
 		},
@@ -74,25 +64,20 @@ After running this command your will need to validate the enrollment in the weba
 `,
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
 			apiURL, err := url.Parse(csConfig.API.Server.OnlineClient.Credentials.URL)
 			if err != nil {
-				log.Fatalf("Could not parse CAPI URL : %s", err)
-			}
-
-			if err := csConfig.LoadHub(); err != nil {
-				log.Fatal(err)
+				return fmt.Errorf("could not parse CAPI URL: %s", err)
 			}
 
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Fatalf("Failed to load hub index : %s", err)
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
+			if err := require.Hub(csConfig); err != nil {
+				return err
 			}
 
-			scenarios, err := cwhub.GetInstalledScenariosAsString()
+			scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
 			if err != nil {
-				log.Fatalf("failed to get scenarios : %s", err)
+				return fmt.Errorf("failed to get installed scenarios: %s", err)
 			}
 
 			if len(scenarios) == 0 {
@@ -109,20 +94,21 @@ After running this command your will need to validate the enrollment in the weba
 			})
 			resp, err := c.Auth.EnrollWatcher(context.Background(), args[0], name, tags, overwrite)
 			if err != nil {
-				log.Fatalf("Could not enroll instance: %s", err)
+				return fmt.Errorf("could not enroll instance: %s", err)
 			}
 			if resp.Response.StatusCode == 200 && !overwrite {
 				log.Warning("Instance already enrolled. You can use '--overwrite' to force enroll")
-				return
+				return nil
 			}
 
-			SetConsoleOpts(csconfig.CONSOLE_CONFIGS, true)
-			if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
-				log.Fatalf("failed writing console config : %s", err)
+			if err := SetConsoleOpts([]string{csconfig.SEND_MANUAL_SCENARIOS, csconfig.SEND_TAINTED_SCENARIOS}, true); err != nil {
+				return err
 			}
-			log.Infof("Enabled tainted&manual alerts sharing, see 'cscli console status'.")
-			log.Infof("Watcher successfully enrolled. Visit https://app.crowdsec.net to accept it.")
-			log.Infof("Please restart crowdsec after accepting the enrollment.")
+
+			log.Info("Enabled tainted&manual alerts sharing, see 'cscli console status'.")
+			log.Info("Watcher successfully enrolled. Visit https://app.crowdsec.net to accept it.")
+			log.Info("Please restart crowdsec after accepting the enrollment.")
+			return nil
 		},
 	}
 	cmdEnroll.Flags().StringVarP(&name, "name", "n", "", "Name to display in the console")
@@ -140,21 +126,23 @@ After running this command your will need to validate the enrollment in the weba
 Enable given information push to the central API. Allows to empower the console`,
 		ValidArgs:         csconfig.CONSOLE_CONFIGS,
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if enableAll {
-				SetConsoleOpts(csconfig.CONSOLE_CONFIGS, true)
+				if err := SetConsoleOpts(csconfig.CONSOLE_CONFIGS, true); err != nil {
+					return err
+				}
 				log.Infof("All features have been enabled successfully")
 			} else {
 				if len(args) == 0 {
-					log.Fatalf("You must specify at least one feature to enable")
+					return fmt.Errorf("you must specify at least one feature to enable")
+				}
+				if err := SetConsoleOpts(args, true); err != nil {
+					return err
 				}
-				SetConsoleOpts(args, true)
 				log.Infof("%v have been enabled", args)
 			}
-			if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
-				log.Fatalf("failed writing console config : %s", err)
-			}
 			log.Infof(ReloadMessage())
+			return nil
 		},
 	}
 	cmdEnable.Flags().BoolVarP(&enableAll, "all", "a", false, "Enable all console options")
@@ -167,49 +155,55 @@ Enable given information push to the central API. Allows to empower the console`
 		Long: `
 Disable given information push to the central API.`,
 		ValidArgs:         csconfig.CONSOLE_CONFIGS,
-		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			if disableAll {
-				SetConsoleOpts(csconfig.CONSOLE_CONFIGS, false)
-			} else {
-				SetConsoleOpts(args, false)
-			}
-
-			if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
-				log.Fatalf("failed writing console config : %s", err)
-			}
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if disableAll {
+				if err := SetConsoleOpts(csconfig.CONSOLE_CONFIGS, false); err != nil {
+					return err
+				}
 				log.Infof("All features have been disabled")
 			} else {
+				if err := SetConsoleOpts(args, false); err != nil {
+					return err
+				}
 				log.Infof("%v have been disabled", args)
 			}
+
 			log.Infof(ReloadMessage())
+			return nil
 		},
 	}
 	cmdDisable.Flags().BoolVarP(&disableAll, "all", "a", false, "Disable all console options")
 	cmdConsole.AddCommand(cmdDisable)
 
 	cmdConsoleStatus := &cobra.Command{
-		Use:               "status [option]",
-		Short:             "Shows status of one or all console options",
+		Use:               "status",
+		Short:             "Shows status of the console options",
 		Example:           `sudo cscli console status`,
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			switch csConfig.Cscli.Output {
 			case "human":
 				cmdConsoleStatusTable(color.Output, *csConfig)
 			case "json":
-				data, err := json.MarshalIndent(csConfig.API.Server.ConsoleConfig, "", "  ")
+				c := csConfig.API.Server.ConsoleConfig
+				out := map[string](*bool){
+					csconfig.SEND_MANUAL_SCENARIOS: c.ShareManualDecisions,
+					csconfig.SEND_CUSTOM_SCENARIOS: c.ShareCustomScenarios,
+					csconfig.SEND_TAINTED_SCENARIOS: c.ShareTaintedScenarios,
+					csconfig.SEND_CONTEXT: c.ShareContext,
+					csconfig.CONSOLE_MANAGEMENT: c.ConsoleManagement,
+				}
+				data, err := json.MarshalIndent(out, "", "  ")
 				if err != nil {
-					log.Fatalf("failed to marshal configuration: %s", err)
+					return fmt.Errorf("failed to marshal configuration: %s", err)
 				}
-				fmt.Printf("%s\n", string(data))
+				fmt.Println(string(data))
 			case "raw":
 				csvwriter := csv.NewWriter(os.Stdout)
 				err := csvwriter.Write([]string{"option", "enabled"})
 				if err != nil {
-					log.Fatal(err)
+					return err
 				}
 
 				rows := [][]string{
@@ -222,11 +216,12 @@ Disable given information push to the central API.`,
 				for _, row := range rows {
 					err = csvwriter.Write(row)
 					if err != nil {
-						log.Fatal(err)
+						return err
 					}
 				}
 				csvwriter.Flush()
 			}
+			return nil
 		},
 	}
 	cmdConsole.AddCommand(cmdConsoleStatus)
@@ -234,7 +229,7 @@ Disable given information push to the central API.`,
 	return cmdConsole
 }
 
-func SetConsoleOpts(args []string, wanted bool) {
+func SetConsoleOpts(args []string, wanted bool) error {
 	for _, arg := range args {
 		switch arg {
 		case csconfig.CONSOLE_MANAGEMENT:
@@ -265,12 +260,12 @@ func SetConsoleOpts(args []string, wanted bool) {
 				if changed {
 					fileContent, err := yaml.Marshal(csConfig.API.Server.OnlineClient.Credentials)
 					if err != nil {
-						log.Fatalf("Cannot marshal credentials: %s", err)
+						return fmt.Errorf("cannot marshal credentials: %s", err)
 					}
 					log.Infof("Updating credentials file: %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
 					err = os.WriteFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, fileContent, 0600)
 					if err != nil {
-						log.Fatalf("Cannot write credentials file: %s", err)
+						return fmt.Errorf("cannot write credentials file: %s", err)
 					}
 				}
 			}
@@ -327,8 +322,13 @@ func SetConsoleOpts(args []string, wanted bool) {
 				csConfig.API.Server.ConsoleConfig.ShareContext = ptr.Of(wanted)
 			}
 		default:
-			log.Fatalf("unknown flag %s", arg)
+			return fmt.Errorf("unknown flag %s", arg)
 		}
 	}
 
+	if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
+		return fmt.Errorf("failed writing console config: %s", err)
+	}
+
+	return nil
 }

cmd/crowdsec-cli/dashboard.go

@@ -1,7 +1,6 @@
 package main
 
 import (
-	"errors"
 	"fmt"
 	"math"
 	"os"
@@ -18,6 +17,8 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/crowdsecurity/crowdsec/pkg/metabase"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
 var (
@@ -27,6 +28,7 @@ var (
 	metabaseConfigPath   string
 	metabaseConfigFolder = "metabase/"
 	metabaseConfigFile   = "metabase.yaml"
+	metabaseImage        = "metabase/metabase:v0.46.6.1"
 	/**/
 	metabaseListenAddress = "127.0.0.1"
 	metabaseListenPort    = "3000"
@@ -54,23 +56,23 @@ cscli dashboard start
 cscli dashboard stop
 cscli dashboard remove
 `,
-		PersistentPreRun: func(cmd *cobra.Command, args []string) {
-			if err := metabase.TestAvailability(); err != nil {
-				log.Fatalf("%s", err)
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := require.LAPI(csConfig); err != nil {
+				return err
 			}
 
-			if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
-				log.Fatal("Local API is disabled, please run this command on the local API machine")
+			if err := metabase.TestAvailability(); err != nil {
+				return err
 			}
 
 			metabaseConfigFolderPath := filepath.Join(csConfig.ConfigPaths.ConfigDir, metabaseConfigFolder)
 			metabaseConfigPath = filepath.Join(metabaseConfigFolderPath, metabaseConfigFile)
 			if err := os.MkdirAll(metabaseConfigFolderPath, os.ModePerm); err != nil {
-				log.Fatal(err)
+				return err
 			}
-			if err := csConfig.LoadDBConfig(); err != nil {
-				log.Errorf("This command requires direct database access (must be run on the local API machine)")
-				log.Fatal(err)
+
+			if err := require.DB(csConfig); err != nil {
+				return err
 			}
 
 			/*
@@ -84,6 +86,7 @@ cscli dashboard remove
 					metabaseContainerID = oldContainerID
 				}
 			}
+			return nil
 		},
 	}
 
@@ -96,7 +99,6 @@ cscli dashboard remove
 	return cmdDashboard
 }
 
-
 func NewDashboardSetupCmd() *cobra.Command {
 	var force bool
 
@@ -111,7 +113,7 @@ cscli dashboard setup
 cscli dashboard setup --listen 0.0.0.0
 cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
  `,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if metabaseDbPath == "" {
 				metabaseDbPath = csConfig.ConfigPaths.DataDir
 			}
@@ -123,70 +125,23 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
 					isValid = passwordIsValid(metabasePassword)
 				}
 			}
-			var answer bool
-			if valid, err := checkSystemMemory(); err == nil && !valid {
-				if !forceYes {
-					prompt := &survey.Confirm{
-						Message: "Metabase requires 1-2GB of RAM, your system is below this requirement continue ?",
-						Default: true,
-					}
-					if err := survey.AskOne(prompt, &answer); err != nil {
-						log.Warnf("unable to ask about RAM check: %s", err)
-					}
-					if !answer {
-						log.Fatal("Unable to continue due to RAM requirement")
-					}
-				} else {
-					log.Warnf("Metabase requires 1-2GB of RAM, your system is below this requirement")
-				}
-			}
-			groupExist := false
-			dockerGroup, err := user.LookupGroup(crowdsecGroup)
-			if err == nil {
-				groupExist = true
+			if err := checkSystemMemory(&forceYes); err != nil {
+				return err
 			}
-			if !forceYes && !groupExist {
-				prompt := &survey.Confirm{
-					Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup),
-					Default: true,
-				}
-				if err := survey.AskOne(prompt, &answer); err != nil {
-					log.Fatalf("unable to ask to force: %s", err)
-				}
+			warnIfNotLoopback(metabaseListenAddress)
+			if err := disclaimer(&forceYes); err != nil {
+				return err
 			}
-			if !answer && !forceYes && !groupExist {
-				log.Fatalf("unable to continue without creating '%s' group", crowdsecGroup)
-			}
-			if !groupExist {
-				groupAddCmd, err := exec.LookPath("groupadd")
-				if err != nil {
-					log.Fatalf("unable to find 'groupadd' command, can't continue")
-				}
-
-				groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}}
-				if err := groupAdd.Run(); err != nil {
-					log.Fatalf("unable to add group '%s': %s", dockerGroup, err)
-				}
-				dockerGroup, err = user.LookupGroup(crowdsecGroup)
-				if err != nil {
-					log.Fatalf("unable to lookup '%s' group: %+v", dockerGroup, err)
-				}
-			}
-			intID, err := strconv.Atoi(dockerGroup.Gid)
+			dockerGroup, err := checkGroups(&forceYes)
 			if err != nil {
-				log.Fatalf("unable to convert group ID to int: %s", err)
+				return err
 			}
-			if err := os.Chown(csConfig.DbConfig.DbPath, 0, intID); err != nil {
-				log.Fatalf("unable to chown sqlite db file '%s': %s", csConfig.DbConfig.DbPath, err)
-			}
-
-			mb, err := metabase.SetupMetabase(csConfig.API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDbPath, dockerGroup.Gid, metabaseContainerID)
+			mb, err := metabase.SetupMetabase(csConfig.API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDbPath, dockerGroup.Gid, metabaseContainerID, metabaseImage)
 			if err != nil {
-				log.Fatal(err)
+				return err
 			}
-
 			if err := mb.DumpConfig(metabaseConfigPath); err != nil {
-				log.Fatal(err)
+				return err
 			}
 
 			log.Infof("Metabase is ready")
@@ -194,11 +149,13 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
 			fmt.Printf("\tURL       : '%s'\n", mb.Config.ListenURL)
 			fmt.Printf("\tusername  : '%s'\n", mb.Config.Username)
 			fmt.Printf("\tpassword  : '%s'\n", mb.Config.Password)
+			return nil
 		},
 	}
 	cmdDashSetup.Flags().BoolVarP(&force, "force", "f", false, "Force setup : override existing files")
 	cmdDashSetup.Flags().StringVarP(&metabaseDbPath, "dir", "d", "", "Shared directory with metabase container")
 	cmdDashSetup.Flags().StringVarP(&metabaseListenAddress, "listen", "l", metabaseListenAddress, "Listen address of container")
+	cmdDashSetup.Flags().StringVar(&metabaseImage, "metabase-image", metabaseImage, "Metabase image to use")
 	cmdDashSetup.Flags().StringVarP(&metabaseListenPort, "port", "p", metabaseListenPort, "Listen port of container")
 	cmdDashSetup.Flags().BoolVarP(&forceYes, "yes", "y", false, "force  yes")
 	//cmdDashSetup.Flags().StringVarP(&metabaseUser, "user", "u", "crowdsec@crowdsec.net", "metabase user")
@@ -214,18 +171,24 @@ func NewDashboardStartCmd() *cobra.Command {
 		Long:              `Stats the metabase container using docker.`,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			mb, err := metabase.NewMetabase(metabaseConfigPath, metabaseContainerID)
 			if err != nil {
-				log.Fatal(err)
+				return err
+			}
+			warnIfNotLoopback(mb.Config.ListenAddr)
+			if err := disclaimer(&forceYes); err != nil {
+				return err
 			}
 			if err := mb.Container.Start(); err != nil {
-				log.Fatalf("Failed to start metabase container : %s", err)
+				return fmt.Errorf("failed to start metabase container : %s", err)
 			}
 			log.Infof("Started metabase")
-			log.Infof("url : http://%s:%s", metabaseListenAddress, metabaseListenPort)
+			log.Infof("url : http://%s:%s", mb.Config.ListenAddr, mb.Config.ListenPort)
+			return nil
 		},
 	}
+	cmdDashStart.Flags().BoolVarP(&forceYes, "yes", "y", false, "force  yes")
 	return cmdDashStart
 }
 
@@ -236,33 +199,33 @@ func NewDashboardStopCmd() *cobra.Command {
 		Long:              `Stops the metabase container using docker.`,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if err := metabase.StopContainer(metabaseContainerID); err != nil {
-				log.Fatalf("unable to stop container '%s': %s", metabaseContainerID, err)
+				return fmt.Errorf("unable to stop container '%s': %s", metabaseContainerID, err)
 			}
+			return nil
 		},
 	}
 	return cmdDashStop
 }
 
-
 func NewDashboardShowPasswordCmd() *cobra.Command {
 	var cmdDashShowPassword = &cobra.Command{Use: "show-password",
 		Short:             "displays password of metabase.",
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			m := metabase.Metabase{}
 			if err := m.LoadConfig(metabaseConfigPath); err != nil {
-				log.Fatal(err)
+				return err
 			}
 			log.Printf("'%s'", m.Config.Password)
+			return nil
 		},
 	}
 	return cmdDashShowPassword
 }
 
-
 func NewDashboardRemoveCmd() *cobra.Command {
 	var force bool
 
@@ -276,53 +239,59 @@ func NewDashboardRemoveCmd() *cobra.Command {
 cscli dashboard remove
 cscli dashboard remove --force
  `,
-		Run: func(cmd *cobra.Command, args []string) {
-			answer := true
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if !forceYes {
+				var answer bool
 				prompt := &survey.Confirm{
 					Message: "Do you really want to remove crowdsec dashboard? (all your changes will be lost)",
 					Default: true,
 				}
 				if err := survey.AskOne(prompt, &answer); err != nil {
-					log.Fatalf("unable to ask to force: %s", err)
+					return fmt.Errorf("unable to ask to force: %s", err)
+				}
+				if !answer {
+					return fmt.Errorf("user stated no to continue")
 				}
 			}
-			if answer {
-				if metabase.IsContainerExist(metabaseContainerID) {
-					log.Debugf("Stopping container %s", metabaseContainerID)
-					if err := metabase.StopContainer(metabaseContainerID); err != nil {
-						log.Warningf("unable to stop container '%s': %s", metabaseContainerID, err)
-					}
-					dockerGroup, err := user.LookupGroup(crowdsecGroup)
-					if err == nil { // if group exist, remove it
-						groupDelCmd, err := exec.LookPath("groupdel")
-						if err != nil {
-							log.Fatalf("unable to find 'groupdel' command, can't continue")
-						}
-
-						groupDel := &exec.Cmd{Path: groupDelCmd, Args: []string{groupDelCmd, crowdsecGroup}}
-						if err := groupDel.Run(); err != nil {
-							log.Errorf("unable to delete group '%s': %s", dockerGroup, err)
-						}
+			if metabase.IsContainerExist(metabaseContainerID) {
+				log.Debugf("Stopping container %s", metabaseContainerID)
+				if err := metabase.StopContainer(metabaseContainerID); err != nil {
+					log.Warningf("unable to stop container '%s': %s", metabaseContainerID, err)
+				}
+				dockerGroup, err := user.LookupGroup(crowdsecGroup)
+				if err == nil { // if group exist, remove it
+					groupDelCmd, err := exec.LookPath("groupdel")
+					if err != nil {
+						return fmt.Errorf("unable to find 'groupdel' command, can't continue")
 					}
-					log.Debugf("Removing container %s", metabaseContainerID)
-					if err := metabase.RemoveContainer(metabaseContainerID); err != nil {
-						log.Warningf("unable to remove container '%s': %s", metabaseContainerID, err)
+
+					groupDel := &exec.Cmd{Path: groupDelCmd, Args: []string{groupDelCmd, crowdsecGroup}}
+					if err := groupDel.Run(); err != nil {
+						log.Warnf("unable to delete group '%s': %s", dockerGroup, err)
 					}
-					log.Infof("container %s stopped & removed", metabaseContainerID)
 				}
-				log.Debugf("Removing metabase db %s", csConfig.ConfigPaths.DataDir)
-				if err := metabase.RemoveDatabase(csConfig.ConfigPaths.DataDir); err != nil {
-					log.Warningf("failed to remove metabase internal db : %s", err)
+				log.Debugf("Removing container %s", metabaseContainerID)
+				if err := metabase.RemoveContainer(metabaseContainerID); err != nil {
+					log.Warnf("unable to remove container '%s': %s", metabaseContainerID, err)
+				}
+				log.Infof("container %s stopped & removed", metabaseContainerID)
+			}
+			log.Debugf("Removing metabase db %s", csConfig.ConfigPaths.DataDir)
+			if err := metabase.RemoveDatabase(csConfig.ConfigPaths.DataDir); err != nil {
+				log.Warnf("failed to remove metabase internal db : %s", err)
+			}
+			if force {
+				m := metabase.Metabase{}
+				if err := m.LoadConfig(metabaseConfigPath); err != nil {
+					return err
 				}
-				if force {
-					if err := metabase.RemoveImageContainer(); err != nil {
-						if !strings.Contains(err.Error(), "No such image") {
-							log.Fatalf("removing docker image: %s", err)
-						}
+				if err := metabase.RemoveImageContainer(m.Config.Image); err != nil {
+					if !strings.Contains(err.Error(), "No such image") {
+						return fmt.Errorf("removing docker image: %s", err)
 					}
 				}
 			}
+			return nil
 		},
 	}
 	cmdDashRemove.Flags().BoolVarP(&force, "force", "f", false, "Remove also the metabase image")
@@ -347,13 +316,95 @@ func passwordIsValid(password string) bool {
 
 }
 
-func checkSystemMemory() (bool, error) {
+func checkSystemMemory(forceYes *bool) error {
 	totMem := memory.TotalMemory()
-	if totMem == 0 {
-		return true, errors.New("Unable to get system total memory")
+	if totMem >= uint64(math.Pow(2, 30)) {
+		return nil
+	}
+	if !*forceYes {
+		var answer bool
+		prompt := &survey.Confirm{
+			Message: "Metabase requires 1-2GB of RAM, your system is below this requirement continue ?",
+			Default: true,
+		}
+		if err := survey.AskOne(prompt, &answer); err != nil {
+			return fmt.Errorf("unable to ask about RAM check: %s", err)
+		}
+		if !answer {
+			return fmt.Errorf("user stated no to continue")
+		}
+		return nil
+	}
+	log.Warn("Metabase requires 1-2GB of RAM, your system is below this requirement")
+	return nil
+}
+
+func warnIfNotLoopback(addr string) {
+	if addr == "127.0.0.1" || addr == "::1" {
+		return
+	}
+	log.Warnf("You are potentially exposing your metabase port to the internet (addr: %s), please consider using a reverse proxy", addr)
+}
+
+func disclaimer(forceYes *bool) error {
+	if !*forceYes {
+		var answer bool
+		prompt := &survey.Confirm{
+			Message: "CrowdSec takes no responsibility for the security of your metabase instance. Do you accept these responsibilities ?",
+			Default: true,
+		}
+		if err := survey.AskOne(prompt, &answer); err != nil {
+			return fmt.Errorf("unable to ask to question: %s", err)
+		}
+		if !answer {
+			return fmt.Errorf("user stated no to responsibilities")
+		}
+		return nil
+	}
+	log.Warn("CrowdSec takes no responsibility for the security of your metabase instance. You used force yes, so you accept this disclaimer")
+	return nil
+}
+
+func checkGroups(forceYes *bool) (*user.Group, error) {
+	groupExist := false
+	dockerGroup, err := user.LookupGroup(crowdsecGroup)
+	if err == nil {
+		groupExist = true
+	}
+	if !groupExist {
+		if !*forceYes {
+			var answer bool
+			prompt := &survey.Confirm{
+				Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup),
+				Default: true,
+			}
+			if err := survey.AskOne(prompt, &answer); err != nil {
+				return dockerGroup, fmt.Errorf("unable to ask to question: %s", err)
+			}
+			if !answer {
+				return dockerGroup, fmt.Errorf("unable to continue without creating '%s' group", crowdsecGroup)
+			}
+		}
+		groupAddCmd, err := exec.LookPath("groupadd")
+		if err != nil {
+			return dockerGroup, fmt.Errorf("unable to find 'groupadd' command, can't continue")
+		}
+
+		groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}}
+		if err := groupAdd.Run(); err != nil {
+			return dockerGroup, fmt.Errorf("unable to add group '%s': %s", dockerGroup, err)
+		}
+		dockerGroup, err = user.LookupGroup(crowdsecGroup)
+		if err != nil {
+			return dockerGroup, fmt.Errorf("unable to lookup '%s' group: %+v", dockerGroup, err)
+		}
+	}
+	intID, err := strconv.Atoi(dockerGroup.Gid)
+	if err != nil {
+		return dockerGroup, fmt.Errorf("unable to convert group ID to int: %s", err)
 	}
-	if uint64(math.Pow(2, 30)) >= totMem {
-		return false, nil
+	if err := os.Chown(csConfig.DbConfig.DbPath, 0, intID); err != nil {
+		return dockerGroup, fmt.Errorf("unable to chown sqlite db file '%s': %s", csConfig.DbConfig.DbPath, err)
 	}
-	return true, nil
+	return dockerGroup, nil
 }

cmd/crowdsec-cli/decisions.go

@@ -16,7 +16,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/version"
+	"github.com/crowdsecurity/go-cs-lib/version"
 
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/models"
@@ -81,6 +81,12 @@ func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error
 		}
 		csvwriter.Flush()
 	} else if csConfig.Cscli.Output == "json" {
+		if *alerts == nil {
+			// avoid returning "null" in `json"
+			// could be cleaner if we used slice of alerts directly
+			fmt.Println("[]")
+			return nil
+		}
 		x, _ := json.MarshalIndent(alerts, "", " ")
 		fmt.Printf("%s", string(x))
 	} else if csConfig.Cscli.Output == "human" {

cmd/crowdsec-cli/decisions_import.go

@@ -15,8 +15,8 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/ptr"
-	"github.com/crowdsecurity/go-cs-lib/pkg/slicetools"
+	"github.com/crowdsecurity/go-cs-lib/ptr"
+	"github.com/crowdsecurity/go-cs-lib/slicetools"
 
 	"github.com/crowdsecurity/crowdsec/pkg/models"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
@@ -188,7 +188,9 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error  {
 		}
 	}
 
-	alerts := models.AddAlertsRequest{}
+	if len(decisions) > 1000 {
+		log.Infof("You are about to add %d decisions, this may take a while", len(decisions))
+	}
 
 	for _, chunk := range slicetools.Chunks(decisions, batchSize) {
 		log.Debugf("Processing chunk of %d decisions", len(chunk))
@@ -212,16 +214,11 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error  {
 			ScenarioVersion: ptr.Of(""),
 			Decisions:       chunk,
 		}
-		alerts = append(alerts, &importAlert)
-	}
-
-	if len(decisions) > 1000 {
-		log.Infof("You are about to add %d decisions, this may take a while", len(decisions))
-	}
 
-	_, _, err = Client.Alerts.Add(context.Background(), alerts)
-	if err != nil {
-		return err
+		_, _, err = Client.Alerts.Add(context.Background(), models.AddAlertsRequest{&importAlert})
+		if err != nil {
+			return err
+		}
 	}
 
 	log.Infof("Imported %d decisions", len(decisions))

cmd/crowdsec-cli/explain.go

@@ -12,9 +12,22 @@ import (
 	"github.com/spf13/cobra"
 
 	"github.com/crowdsecurity/crowdsec/pkg/hubtest"
-	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
+func GetLineCountForFile(filepath string) (int, error) {
+	f, err := os.Open(filepath)
+	if err != nil {
+		return 0, err
+	}
+	defer f.Close()
+	lc := 0
+	fs := bufio.NewScanner(f)
+	for fs.Scan() {
+		lc++
+	}
+	return lc, nil
+}
+
 func runExplain(cmd *cobra.Command, args []string) error {
 	flags := cmd.Flags()
 
@@ -61,6 +74,11 @@ func runExplain(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	labels, err := flags.GetString("labels")
+	if err != nil {
+		return err
+	}
+
 	fileInfo, _ := os.Stdin.Stat()
 
 	if logType == "" || (logLine == "" && logFile == "" && dsn == "") {
@@ -123,9 +141,12 @@ func runExplain(cmd *cobra.Command, args []string) error {
 			return fmt.Errorf("unable to get absolute path of '%s', exiting", logFile)
 		}
 		dsn = fmt.Sprintf("file://%s", absolutePath)
-		lineCount := types.GetLineCountForFile(absolutePath)
+		lineCount, err := GetLineCountForFile(absolutePath)
+		if err != nil {
+			return err
+		}
 		if lineCount > 100 {
-			log.Warnf("log file contains %d lines. This may take lot of resources.", lineCount)
+			log.Warnf("The log file contains %d lines. This may take a lot of resources.", lineCount)
 		}
 	}
 
@@ -134,6 +155,10 @@ func runExplain(cmd *cobra.Command, args []string) error {
 	}
 
 	cmdArgs := []string{"-c", ConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", dir, "-no-api"}
+	if labels != "" {
+		log.Debugf("adding labels %s", labels)
+		cmdArgs = append(cmdArgs, "-label", labels)
+	}
 	crowdsecCmd := exec.Command(crowdsec, cmdArgs...)
 	output, err := crowdsecCmd.CombinedOutput()
 	if err != nil {
@@ -193,6 +218,7 @@ tail -n 5 myfile.log | cscli explain --type nginx -f -
 	flags.StringP("dsn", "d", "", "DSN to test")
 	flags.StringP("log", "l", "", "Log line to test")
 	flags.StringP("type", "t", "", "Type of the acquisition to test")
+	flags.String("labels", "", "Additional labels to add to the acquisition format (key:value,key2:value2)")
 	flags.BoolP("verbose", "v", false, "Display individual changes")
 	flags.Bool("failures", false, "Only show failed lines")
 	flags.Bool("only-successful-parsers", false, "Only show successful parsers")

cmd/crowdsec-cli/hub.go

@@ -8,6 +8,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
@@ -50,16 +51,12 @@ func NewHubListCmd() *cobra.Command {
 		Short:             "List installed configs",
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-
-			if err := csConfig.LoadHub(); err != nil {
-				log.Fatal(err)
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if err := require.Hub(csConfig); err != nil {
+				return err
 			}
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-			//use LocalSync to get warnings about tainted / outdated items
+
+			// use LocalSync to get warnings about tainted / outdated items
 			_, warn := cwhub.LocalSync(csConfig.Hub)
 			for _, v := range warn {
 				log.Info(v)
@@ -68,6 +65,8 @@ func NewHubListCmd() *cobra.Command {
 			ListItems(color.Output, []string{
 				cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW,
 			}, args, true, false, all)
+
+			return nil
 		},
 	}
 	cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
@@ -94,19 +93,18 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde
 			}
 			return nil
 		},
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if err := csConfig.LoadHub(); err != nil {
-				log.Fatal(err)
+				return err
 			}
 			if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
-				if errors.Is(err, cwhub.ErrIndexNotFound) {
-					log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch)
-					cwhub.HubBranch = "master"
-					if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
-						log.Fatalf("Failed to get Hub index after retry : %v", err)
-					}
-				} else {
-					log.Fatalf("Failed to get Hub index : %v", err)
+				if !errors.Is(err, cwhub.ErrIndexNotFound) {
+					return fmt.Errorf("failed to get Hub index : %w", err)
+				}
+				log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch)
+				cwhub.HubBranch = "master"
+				if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
+					return fmt.Errorf("failed to get Hub index after retry: %w", err)
 				}
 			}
 			//use LocalSync to get warnings about tainted / outdated items
@@ -114,6 +112,8 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde
 			for _, v := range warn {
 				log.Info(v)
 			}
+
+			return nil
 		},
 	}
 
@@ -139,13 +139,9 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
 			}
 			return nil
 		},
-		Run: func(cmd *cobra.Command, args []string) {
-			if err := csConfig.LoadHub(); err != nil {
-				log.Fatal(err)
-			}
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to get Hub index : %v", err)
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if err := require.Hub(csConfig); err != nil {
+				return err
 			}
 
 			log.Infof("Upgrading collections")
@@ -156,6 +152,8 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
 			cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
 			log.Infof("Upgrading postoverflows")
 			cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
+
+			return nil
 		},
 	}
 	cmdHubUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")

cmd/crowdsec-cli/lapi.go

@@ -5,17 +5,18 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"slices"
 	"sort"
 	"strings"
 
 	"github.com/go-openapi/strfmt"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"golang.org/x/exp/slices"
 	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/version"
+	"github.com/crowdsecurity/go-cs-lib/version"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
@@ -36,15 +37,12 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
 	}
-	if err := csConfig.LoadHub(); err != nil {
+
+	if err := require.Hub(csConfig); err != nil {
 		log.Fatal(err)
 	}
 
-	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-		log.Info("Run 'sudo cscli hub update' to get the hub index")
-		log.Fatalf("Failed to load hub index : %s", err)
-	}
-	scenarios, err := cwhub.GetInstalledScenariosAsString()
+	scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
 	if err != nil {
 		log.Fatalf("failed to get scenarios : %s", err)
 	}
@@ -216,6 +214,29 @@ func NewLapiCmd() *cobra.Command {
 	return cmdLapi
 }
 
+func AddContext(key string, values []string) error {
+	if err := alertcontext.ValidateContextExpr(key, values); err != nil {
+		return fmt.Errorf("invalid context configuration :%s", err)
+	}
+	if _, ok := csConfig.Crowdsec.ContextToSend[key]; !ok {
+		csConfig.Crowdsec.ContextToSend[key] = make([]string, 0)
+		log.Infof("key '%s' added", key)
+	}
+	data := csConfig.Crowdsec.ContextToSend[key]
+	for _, val := range values {
+		if !slices.Contains(data, val) {
+			log.Infof("value '%s' added to key '%s'", val, key)
+			data = append(data, val)
+		}
+		csConfig.Crowdsec.ContextToSend[key] = data
+	}
+	if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func NewLapiContextCmd() *cobra.Command {
 	cmdContext := &cobra.Command{
 		Use:               "context [command]",
@@ -246,32 +267,29 @@ func NewLapiContextCmd() *cobra.Command {
 		Short: "Add context to send with alerts. You must specify the output key with the expr value you want",
 		Example: `cscli lapi context add --key source_ip --value evt.Meta.source_ip
 cscli lapi context add --key file_source --value evt.Line.Src
+cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user 
 		`,
 		DisableAutoGenTag: true,
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := alertcontext.ValidateContextExpr(keyToAdd, valuesToAdd); err != nil {
-				log.Fatalf("invalid context configuration :%s", err)
-			}
-			if _, ok := csConfig.Crowdsec.ContextToSend[keyToAdd]; !ok {
-				csConfig.Crowdsec.ContextToSend[keyToAdd] = make([]string, 0)
-				log.Infof("key '%s' added", keyToAdd)
-			}
-			data := csConfig.Crowdsec.ContextToSend[keyToAdd]
-			for _, val := range valuesToAdd {
-				if !slices.Contains(data, val) {
-					log.Infof("value '%s' added to key '%s'", val, keyToAdd)
-					data = append(data, val)
+			if keyToAdd != "" {
+				if err := AddContext(keyToAdd, valuesToAdd); err != nil {
+					log.Fatalf(err.Error())
 				}
-				csConfig.Crowdsec.ContextToSend[keyToAdd] = data
+				return
 			}
-			if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
-				log.Fatalf(err.Error())
+
+			for _, v := range valuesToAdd {
+				keySlice := strings.Split(v, ".")
+				key := keySlice[len(keySlice)-1]
+				value := []string{v}
+				if err := AddContext(key, value); err != nil {
+					log.Fatalf(err.Error())
+				}
 			}
 		},
 	}
 	cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
 	cmdContextAdd.Flags().StringSliceVar(&valuesToAdd, "value", []string{}, "The expr fields to associate with the key")
-	cmdContextAdd.MarkFlagRequired("key")
 	cmdContextAdd.MarkFlagRequired("value")
 	cmdContext.AddCommand(cmdContextAdd)
 

cmd/crowdsec-cli/machines.go

@@ -8,6 +8,7 @@ import (
 	"io"
 	"math/big"
 	"os"
+	"slices"
 	"strings"
 	"time"
 
@@ -17,7 +18,6 @@ import (
 	"github.com/google/uuid"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"golang.org/x/exp/slices"
 	"gopkg.in/yaml.v2"
 
 	"github.com/crowdsecurity/machineid"
@@ -26,6 +26,8 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/database/ent"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
 var (
@@ -144,20 +146,11 @@ func getAgents(out io.Writer, dbClient *database.Client) error {
 func NewMachinesListCmd() *cobra.Command {
 	cmdMachinesList := &cobra.Command{
 		Use:               "list",
-		Short:             "List machines",
-		Long:              `List `,
+		Short:             "list all machines in the database",
+		Long:              `list all machines in the database with their status and last heartbeat`,
 		Example:           `cscli machines list`,
-		Args:              cobra.MaximumNArgs(1),
+		Args:              cobra.NoArgs,
 		DisableAutoGenTag: true,
-		PreRunE: func(cmd *cobra.Command, args []string) error {
-			var err error
-			dbClient, err = database.NewClient(csConfig.DbConfig)
-			if err != nil {
-				return fmt.Errorf("unable to create new database client: %s", err)
-			}
-
-			return nil
-		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			err := getAgents(color.Output, dbClient)
 			if err != nil {
@@ -174,7 +167,7 @@ func NewMachinesListCmd() *cobra.Command {
 func NewMachinesAddCmd() *cobra.Command {
 	cmdMachinesAdd := &cobra.Command{
 		Use:               "add",
-		Short:             "add machine to the database.",
+		Short:             "add a single machine to the database",
 		DisableAutoGenTag: true,
 		Long:              `Register a new machine in the database. cscli should be on the same machine as LAPI.`,
 		Example: `
@@ -182,15 +175,6 @@ cscli machines add --auto
 cscli machines add MyTestMachine --auto
 cscli machines add MyTestMachine --password MyPassword
 `,
-		PreRunE: func(cmd *cobra.Command, args []string) error {
-			var err error
-			dbClient, err = database.NewClient(csConfig.DbConfig)
-			if err != nil {
-				return fmt.Errorf("unable to create new database client: %s", err)
-			}
-
-			return nil
-		},
 		RunE: runMachinesAdd,
 	}
 
@@ -318,26 +302,12 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
 func NewMachinesDeleteCmd() *cobra.Command {
 	cmdMachinesDelete := &cobra.Command{
 		Use:               "delete [machine_name]...",
-		Short:             "delete machines",
+		Short:             "delete machine(s) by name",
 		Example:           `cscli machines delete "machine1" "machine2"`,
 		Args:              cobra.MinimumNArgs(1),
 		Aliases:           []string{"remove"},
 		DisableAutoGenTag: true,
-		PreRunE: func(cmd *cobra.Command, args []string) error {
-			var err error
-			dbClient, err = database.NewClient(csConfig.DbConfig)
-			if err != nil {
-				return fmt.Errorf("unable to create new database client: %s", err)
-			}
-			return nil
-		},
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			var err error
-			dbClient, err = getDBClient()
-			if err != nil {
-				cobra.CompError("unable to create new database client: " + err.Error())
-				return nil, cobra.ShellCompDirectiveNoFileComp
-			}
 			machines, err := dbClient.ListMachines()
 			if err != nil {
 				cobra.CompError("unable to list machines " + err.Error())
@@ -369,6 +339,86 @@ func runMachinesDelete(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
+func NewMachinesPruneCmd() *cobra.Command {
+	var parsedDuration time.Duration
+	cmdMachinesPrune := &cobra.Command{
+		Use:   "prune",
+		Short: "prune multiple machines from the database",
+		Long:  `prune multiple machines that are not validated or have not connected to the local API in a given duration.`,
+		Example: `cscli machines prune
+cscli machines prune --duration 1h
+cscli machines prune --not-validated-only --force`,
+		Args:              cobra.NoArgs,
+		DisableAutoGenTag: true,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			dur, _ := cmd.Flags().GetString("duration")
+			var err error
+			parsedDuration, err = time.ParseDuration(fmt.Sprintf("-%s", dur))
+			if err != nil {
+				return fmt.Errorf("unable to parse duration '%s': %s", dur, err)
+			}
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			notValidOnly, _ := cmd.Flags().GetBool("not-validated-only")
+			force, _ := cmd.Flags().GetBool("force")
+			if parsedDuration >= 0-60*time.Second && !notValidOnly {
+				var answer bool
+				prompt := &survey.Confirm{
+					Message: "The duration you provided is less than or equal 60 seconds this can break installations do you want to continue ?",
+					Default: false,
+				}
+				if err := survey.AskOne(prompt, &answer); err != nil {
+					return fmt.Errorf("unable to ask about prune check: %s", err)
+				}
+				if !answer {
+					fmt.Println("user aborted prune no changes were made")
+					return nil
+				}
+			}
+			machines := make([]*ent.Machine, 0)
+			if pending, err := dbClient.QueryPendingMachine(); err == nil {
+				machines = append(machines, pending...)
+			}
+			if !notValidOnly {
+				if pending, err := dbClient.QueryLastValidatedHeartbeatLT(time.Now().UTC().Add(parsedDuration)); err == nil {
+					machines = append(machines, pending...)
+				}
+			}
+			if len(machines) == 0 {
+				fmt.Println("no machines to prune")
+				return nil
+			}
+			getAgentsTable(color.Output, machines)
+			if !force {
+				var answer bool
+				prompt := &survey.Confirm{
+					Message: "You are about to PERMANENTLY remove the above machines from the database these will NOT be recoverable, continue ?",
+					Default: false,
+				}
+				if err := survey.AskOne(prompt, &answer); err != nil {
+					return fmt.Errorf("unable to ask about prune check: %s", err)
+				}
+				if !answer {
+					fmt.Println("user aborted prune no changes were made")
+					return nil
+				}
+			}
+			nbDeleted, err := dbClient.BulkDeleteWatchers(machines)
+			if err != nil {
+				return fmt.Errorf("unable to prune machines: %s", err)
+			}
+			fmt.Printf("successfully delete %d machines\n", nbDeleted)
+			return nil
+		},
+	}
+	cmdMachinesPrune.Flags().StringP("duration", "d", "10m", "duration of time since validated machine last heartbeat")
+	cmdMachinesPrune.Flags().Bool("not-validated-only", false, "only prune machines that are not validated")
+	cmdMachinesPrune.Flags().Bool("force", false, "force prune without asking for confirmation")
+
+	return cmdMachinesPrune
+}
+
 func NewMachinesValidateCmd() *cobra.Command {
 	cmdMachinesValidate := &cobra.Command{
 		Use:               "validate",
@@ -377,15 +427,6 @@ func NewMachinesValidateCmd() *cobra.Command {
 		Example:           `cscli machines validate "machine_name"`,
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
-		PreRunE: func(cmd *cobra.Command, args []string) error {
-			var err error
-			dbClient, err = database.NewClient(csConfig.DbConfig)
-			if err != nil {
-				return fmt.Errorf("unable to create new database client: %s", err)
-			}
-
-			return nil
-		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			machineID := args[0]
 			if err := dbClient.ValidateMachine(machineID); err != nil {
@@ -404,20 +445,21 @@ func NewMachinesCmd() *cobra.Command {
 	var cmdMachines = &cobra.Command{
 		Use:   "machines [action]",
 		Short: "Manage local API machines [requires local API]",
-		Long: `To list/add/delete/validate machines.
+		Long: `To list/add/delete/validate/prune machines.
 Note: This command requires database direct access, so is intended to be run on the local API machine.
 `,
 		Example:           `cscli machines [action]`,
 		DisableAutoGenTag: true,
 		Aliases:           []string{"machine"},
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
-				if err != nil {
-					log.Errorf("local api : %s", err)
-				}
-				return fmt.Errorf("local API is disabled, please run this command on the local API machine")
+			var err error
+			if err := require.LAPI(csConfig); err != nil {
+				return err
+			}
+			dbClient, err = database.NewClient(csConfig.DbConfig)
+			if err != nil {
+				return fmt.Errorf("unable to create new database client: %s", err)
 			}
-
 			return nil
 		},
 	}
@@ -426,6 +468,7 @@ Note: This command requires database direct access, so is intended to be run on
 	cmdMachines.AddCommand(NewMachinesAddCmd())
 	cmdMachines.AddCommand(NewMachinesDeleteCmd())
 	cmdMachines.AddCommand(NewMachinesValidateCmd())
+	cmdMachines.AddCommand(NewMachinesPruneCmd())
 
 	return cmdMachines
 }

cmd/crowdsec-cli/main.go

@@ -3,8 +3,8 @@ package main
 import (
 	"fmt"
 	"os"
-	"path"
 	"path/filepath"
+	"slices"
 	"strings"
 
 	"github.com/fatih/color"
@@ -12,7 +12,6 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/cobra/doc"
-	"golang.org/x/exp/slices"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
@@ -54,11 +53,11 @@ func initConfig() {
 	}
 
 	if !slices.Contains(NoNeedConfig, os.Args[1]) {
+		log.Debugf("Using %s as configuration file", ConfigFilePath)
 		csConfig, mergedConfig, err = csconfig.NewConfig(ConfigFilePath, false, false, true)
 		if err != nil {
 			log.Fatal(err)
 		}
-		log.Debugf("Using %s as configuration file", ConfigFilePath)
 		if err := csConfig.LoadCSCLI(); err != nil {
 			log.Fatal(err)
 		}
@@ -116,7 +115,7 @@ title: %s
 ---
 `
 	name := filepath.Base(filename)
-	base := strings.TrimSuffix(name, path.Ext(name))
+	base := strings.TrimSuffix(name, filepath.Ext(name))
 	return fmt.Sprintf(header, base, strings.ReplaceAll(base, "_", " "))
 }
 
@@ -189,7 +188,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	/*usage*/
 	var cmdVersion = &cobra.Command{
 		Use:               "version",
-		Short:             "Display version and exit.",
+		Short:             "Display version",
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
 		Run: func(cmd *cobra.Command, args []string) {

cmd/crowdsec-cli/metrics.go

@@ -16,7 +16,7 @@ import (
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v3"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
+	"github.com/crowdsecurity/go-cs-lib/trace"
 )
 
 // FormatPrometheusMetrics is a complete rip from prom2json

cmd/crowdsec-cli/notifications.go

@@ -19,12 +19,14 @@ import (
 	"github.com/spf13/cobra"
 	"gopkg.in/tomb.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/version"
+	"github.com/crowdsecurity/go-cs-lib/version"
 
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/csplugin"
 	"github.com/crowdsecurity/crowdsec/pkg/csprofiles"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
 type NotificationsCfg struct {
@@ -41,16 +43,18 @@ func NewNotificationsCmd() *cobra.Command {
 		Args:              cobra.MinimumNArgs(1),
 		Aliases:           []string{"notifications", "notification"},
 		DisableAutoGenTag: true,
-		PersistentPreRun: func(cmd *cobra.Command, args []string) {
-			var (
-				err error
-			)
-			if err = csConfig.API.Server.LoadProfiles(); err != nil {
-				log.Fatal(err)
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := require.LAPI(csConfig); err != nil {
+				return err
 			}
-			if csConfig.ConfigPaths.NotificationDir == "" {
-				log.Fatalf("config_paths.notification_dir is not set in crowdsec config")
+			if err := require.Profiles(csConfig); err != nil {
+				return err
 			}
+			if err := require.Notifications(csConfig); err != nil {
+				return err
+			}
+
+			return nil
 		},
 	}
 

cmd/crowdsec-cli/papi.go

@@ -1,17 +1,18 @@
 package main
 
 import (
-	"fmt"
 	"time"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"gopkg.in/tomb.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/ptr"
+	"github.com/crowdsecurity/go-cs-lib/ptr"
 
 	"github.com/crowdsecurity/crowdsec/pkg/apiserver"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
 func NewPapiCmd() *cobra.Command {
@@ -21,14 +22,14 @@ func NewPapiCmd() *cobra.Command {
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
-				return fmt.Errorf("Local API is disabled, please run this command on the local API machine: %w", err)
+			if err := require.LAPI(csConfig); err != nil {
+				return err
 			}
-			if csConfig.API.Server.OnlineClient == nil {
-				log.Fatalf("no configuration for Central API in '%s'", *csConfig.FilePath)
+			if err := require.CAPI(csConfig); err != nil {
+				return err
 			}
-			if csConfig.API.Server.OnlineClient.Credentials.PapiURL == "" {
-				log.Fatalf("no PAPI URL in configuration")
+			if err := require.PAPI(csConfig); err != nil {
+				return err
 			}
 			return nil
 		},

cmd/crowdsec-cli/parsers.go

@@ -7,10 +7,10 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-
 func NewParsersCmd() *cobra.Command {
 	var cmdParsers = &cobra.Command{
 		Use:   "parsers [action] [config]",
@@ -25,21 +25,10 @@ cscli parsers remove crowdsecurity/sshd-logs
 		Aliases:           []string{"parser"},
 		DisableAutoGenTag: true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadHub(); err != nil {
-				log.Fatal(err)
-			}
-			if csConfig.Hub == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			if err := cwhub.SetHubBranch(); err != nil {
-				return fmt.Errorf("error while setting hub branch: %s", err)
+			if err := require.Hub(csConfig); err != nil {
+				return err
 			}
 
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
 			return nil
 		},
 		PersistentPostRun: func(cmd *cobra.Command, args []string) {
@@ -59,7 +48,6 @@ cscli parsers remove crowdsecurity/sshd-logs
 	return cmdParsers
 }
 
-
 func NewParsersInstallCmd() *cobra.Command {
 	var ignoreError bool
 
@@ -73,7 +61,7 @@ func NewParsersInstallCmd() *cobra.Command {
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compAllItems(cwhub.PARSERS, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			for _, name := range args {
 				t := cwhub.GetItem(cwhub.PARSERS, name)
 				if t == nil {
@@ -82,15 +70,16 @@ func NewParsersInstallCmd() *cobra.Command {
 					continue
 				}
 				if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
-					if ignoreError {
-						log.Errorf("Error while installing '%s': %s", name, err)
-					} else {
-						log.Fatalf("Error while installing '%s': %s", name, err)
+					if !ignoreError {
+						return fmt.Errorf("error while installing '%s': %w", name, err)
 					}
+					log.Errorf("Error while installing '%s': %s", name, err)
 				}
 			}
+			return nil
 		},
 	}
+
 	cmdParsersInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
 	cmdParsersInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
 	cmdParsersInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple parsers")
@@ -98,33 +87,35 @@ func NewParsersInstallCmd() *cobra.Command {
 	return cmdParsersInstall
 }
 
-
 func NewParsersRemoveCmd() *cobra.Command {
-	var cmdParsersRemove = &cobra.Command{
+	cmdParsersRemove := &cobra.Command{
 		Use:               "remove [config]",
 		Short:             "Remove given parser(s)",
 		Long:              `Remove given parse(s) from hub`,
-		Aliases:           []string{"delete"},
 		Example:           `cscli parsers remove crowdsec/xxx crowdsec/xyz`,
+		Aliases:           []string{"delete"},
 		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.PARSERS, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if all {
 				cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, forceAction)
-				return
+				return nil
 			}
 
 			if len(args) == 0 {
-				log.Fatalf("Specify at least one parser to remove or '--all' flag.")
+				return fmt.Errorf("specify at least one parser to remove or '--all'")
 			}
 
 			for _, name := range args {
 				cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, forceAction)
 			}
+
+			return nil
 		},
 	}
+
 	cmdParsersRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
 	cmdParsersRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
 	cmdParsersRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the parsers")
@@ -132,9 +123,8 @@ func NewParsersRemoveCmd() *cobra.Command {
 	return cmdParsersRemove
 }
 
-
 func NewParsersUpgradeCmd() *cobra.Command {
-	var cmdParsersUpgrade = &cobra.Command{
+	cmdParsersUpgrade := &cobra.Command{
 		Use:               "upgrade [config]",
 		Short:             "Upgrade given parser(s)",
 		Long:              `Fetch and upgrade given parser(s) from hub`,
@@ -143,26 +133,27 @@ func NewParsersUpgradeCmd() *cobra.Command {
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.PARSERS, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if all {
 				cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
 			} else {
 				if len(args) == 0 {
-					log.Fatalf("no target parser to upgrade")
+					return fmt.Errorf("specify at least one parser to upgrade or '--all'")
 				}
 				for _, name := range args {
 					cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, forceAction)
 				}
 			}
+			return nil
 		},
 	}
+
 	cmdParsersUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the parsers")
 	cmdParsersUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
 
 	return cmdParsersUpgrade
 }
 
-
 func NewParsersInspectCmd() *cobra.Command {
 	var cmdParsersInspect = &cobra.Command{
 		Use:               "inspect [name]",
@@ -178,12 +169,12 @@ func NewParsersInspectCmd() *cobra.Command {
 			InspectItem(args[0], cwhub.PARSERS)
 		},
 	}
+
 	cmdParsersInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
 
 	return cmdParsersInspect
 }
 
-
 func NewParsersListCmd() *cobra.Command {
 	var cmdParsersList = &cobra.Command{
 		Use:   "list [name]",
@@ -196,6 +187,7 @@ cscli parser list crowdsecurity/xxx`,
 			ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all)
 		},
 	}
+
 	cmdParsersList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
 
 	return cmdParsersList

cmd/crowdsec-cli/postoverflows.go

@@ -7,9 +7,46 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
+func NewPostOverflowsCmd() *cobra.Command {
+	cmdPostOverflows := &cobra.Command{
+		Use:   "postoverflows [action] [config]",
+		Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub",
+		Example: `cscli postoverflows install crowdsecurity/cdn-whitelist
+		cscli postoverflows inspect crowdsecurity/cdn-whitelist
+		cscli postoverflows upgrade crowdsecurity/cdn-whitelist
+		cscli postoverflows list
+		cscli postoverflows remove crowdsecurity/cdn-whitelist`,
+		Args:              cobra.MinimumNArgs(1),
+		Aliases:           []string{"postoverflow"},
+		DisableAutoGenTag: true,
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := require.Hub(csConfig); err != nil {
+				return err
+			}
+
+			return nil
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			if cmd.Name() == "inspect" || cmd.Name() == "list" {
+				return
+			}
+			log.Infof(ReloadMessage())
+		},
+	}
+
+	cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd())
+	cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd())
+	cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd())
+	cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd())
+	cmdPostOverflows.AddCommand(NewPostOverflowsListCmd())
+
+	return cmdPostOverflows
+}
+
 func NewPostOverflowsInstallCmd() *cobra.Command {
 	var ignoreError bool
 
@@ -23,7 +60,7 @@ func NewPostOverflowsInstallCmd() *cobra.Command {
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compAllItems(cwhub.PARSERS_OVFLW, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			for _, name := range args {
 				t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name)
 				if t == nil {
@@ -32,13 +69,13 @@ func NewPostOverflowsInstallCmd() *cobra.Command {
 					continue
 				}
 				if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil {
-					if ignoreError {
-						log.Errorf("Error while installing '%s': %s", name, err)
-					} else {
-						log.Fatalf("Error while installing '%s': %s", name, err)
+					if !ignoreError {
+						return fmt.Errorf("error while installing '%s': %w", name, err)
 					}
+					log.Errorf("Error while installing '%s': %s", name, err)
 				}
 			}
+			return nil
 		},
 	}
 
@@ -55,24 +92,26 @@ func NewPostOverflowsRemoveCmd() *cobra.Command {
 		Short:             "Remove given postoverflow(s)",
 		Long:              `remove given postoverflow(s)`,
 		Example:           `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`,
-		DisableAutoGenTag: true,
 		Aliases:           []string{"delete"},
+		DisableAutoGenTag: true,
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if all {
 				cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, forceAction)
-				return
+				return nil
 			}
 
 			if len(args) == 0 {
-				log.Fatalf("Specify at least one postoverflow to remove or '--all' flag.")
+				return fmt.Errorf("specify at least one postoverflow to remove or '--all'")
 			}
 
 			for _, name := range args {
 				cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, forceAction)
 			}
+
+			return nil
 		},
 	}
 
@@ -93,17 +132,18 @@ func NewPostOverflowsUpgradeCmd() *cobra.Command {
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
 		},
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if all {
 				cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
 			} else {
 				if len(args) == 0 {
-					log.Fatalf("no target postoverflow to upgrade")
+					return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'")
 				}
 				for _, name := range args {
 					cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, forceAction)
 				}
 			}
+			return nil
 		},
 	}
 
@@ -120,10 +160,10 @@ func NewPostOverflowsInspectCmd() *cobra.Command {
 		Long:              `Inspect given postoverflow`,
 		Example:           `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`,
 		DisableAutoGenTag: true,
+		Args:              cobra.MinimumNArgs(1),
 		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
 		},
-		Args: cobra.MinimumNArgs(1),
 		Run: func(cmd *cobra.Command, args []string) {
 			InspectItem(args[0], cwhub.PARSERS_OVFLW)
 		},
@@ -149,52 +189,3 @@ cscli postoverflows list crowdsecurity/xxx`,
 
 	return cmdPostOverflowsList
 }
-
-
-
-func NewPostOverflowsCmd() *cobra.Command {
-	cmdPostOverflows := &cobra.Command{
-		Use:   "postoverflows [action] [config]",
-		Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub",
-		Example: `cscli postoverflows install crowdsecurity/cdn-whitelist
-		cscli postoverflows inspect crowdsecurity/cdn-whitelist
-		cscli postoverflows upgrade crowdsecurity/cdn-whitelist
-		cscli postoverflows list
-		cscli postoverflows remove crowdsecurity/cdn-whitelist`,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"postoverflow"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadHub(); err != nil {
-				log.Fatal(err)
-			}
-			if csConfig.Hub == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			if err := cwhub.SetHubBranch(); err != nil {
-				return fmt.Errorf("error while setting hub branch: %s", err)
-			}
-
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			if cmd.Name() == "inspect" || cmd.Name() == "list" {
-				return
-			}
-			log.Infof(ReloadMessage())
-		},
-	}
-
-	cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsListCmd())
-
-	return cmdPostOverflows
-}

cmd/crowdsec-cli/require/require.go

@@ -0,0 +1,85 @@
+package require
+
+import (
+	"fmt"
+
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func LAPI(c *csconfig.Config) error {
+	if err := c.LoadAPIServer(); err != nil {
+		return fmt.Errorf("failed to load Local API: %w", err)
+	}
+
+	if c.DisableAPI {
+		return fmt.Errorf("local API is disabled -- this command must be run on the local API machine")
+	}
+
+	return nil
+}
+
+func CAPI(c *csconfig.Config) error {
+	if c.API.Server.OnlineClient == nil {
+		return fmt.Errorf("no configuration for Central API (CAPI) in '%s'", *c.FilePath)
+	}
+	return nil
+}
+
+func PAPI(c *csconfig.Config) error {
+	if c.API.Server.OnlineClient.Credentials.PapiURL == "" {
+		return fmt.Errorf("no PAPI URL in configuration")
+	}
+	return nil
+}
+
+func CAPIRegistered(c *csconfig.Config) error {
+	if c.API.Server.OnlineClient.Credentials == nil {
+		return fmt.Errorf("the Central API (CAPI) must be configured with 'cscli capi register'")
+	}
+
+	return nil
+}
+
+func DB(c *csconfig.Config) error {
+	if err := c.LoadDBConfig(); err != nil {
+		return fmt.Errorf("this command requires direct database access (must be run on the local API machine): %w", err)
+	}
+	return nil
+}
+
+func Profiles(c *csconfig.Config) error {
+	if err := c.API.Server.LoadProfiles(); err != nil {
+		return fmt.Errorf("while loading profiles: %w", err)
+	}
+
+	return nil
+}
+
+func Notifications(c *csconfig.Config) error {
+	if c.ConfigPaths.NotificationDir == "" {
+		return fmt.Errorf("config_paths.notification_dir is not set in crowdsec config")
+	}
+
+	return nil
+}
+
+func Hub (c *csconfig.Config) error {
+	if err := c.LoadHub(); err != nil {
+		return err
+	}
+
+	if c.Hub == nil {
+		return fmt.Errorf("you must configure cli before interacting with hub")
+	}
+
+	if err := cwhub.SetHubBranch(); err != nil {
+		return fmt.Errorf("while setting hub branch: %w", err)
+	}
+
+	if err := cwhub.GetHubIdx(c.Hub); err != nil {
+		return fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err)
+	}
+
+	return nil
+}

cmd/crowdsec-cli/scenarios.go

@@ -7,6 +7,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
@@ -24,20 +25,8 @@ cscli scenarios remove crowdsecurity/ssh-bf
 		Aliases:           []string{"scenario"},
 		DisableAutoGenTag: true,
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadHub(); err != nil {
-				log.Fatal(err)
-			}
-			if csConfig.Hub == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			if err := cwhub.SetHubBranch(); err != nil {
-				return fmt.Errorf("while setting hub branch: %w", err)
-			}
-
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to get Hub index : %v", err)
+			if err := require.Hub(csConfig); err != nil {
+				return err
 			}
 
 			return nil
@@ -72,7 +61,7 @@ func NewCmdScenariosInstall() *cobra.Command {
 			return compAllItems(cwhub.SCENARIOS, args, toComplete)
 		},
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			for _, name := range args {
 				t := cwhub.GetItem(cwhub.SCENARIOS, name)
 				if t == nil {
@@ -81,13 +70,13 @@ func NewCmdScenariosInstall() *cobra.Command {
 					continue
 				}
 				if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
-					if ignoreError {
-						log.Errorf("Error while installing '%s': %s", name, err)
-					} else {
-						log.Fatalf("Error while installing '%s': %s", name, err)
+					if !ignoreError {
+						return fmt.Errorf("error while installing '%s': %w", name, err)
 					}
+					log.Errorf("Error while installing '%s': %s", name, err)
 				}
 			}
+			return nil
 		},
 	}
 	cmdScenariosInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
@@ -108,19 +97,20 @@ func NewCmdScenariosRemove() *cobra.Command {
 			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
 		},
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if all {
 				cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, forceAction)
-				return
+				return nil
 			}
 
 			if len(args) == 0 {
-				log.Fatalf("Specify at least one scenario to remove or '--all' flag.")
+				return fmt.Errorf("specify at least one scenario to remove or '--all'")
 			}
 
 			for _, name := range args {
 				cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, forceAction)
 			}
+			return nil
 		},
 	}
 	cmdScenariosRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
@@ -140,17 +130,18 @@ func NewCmdScenariosUpgrade() *cobra.Command {
 			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
 		},
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if all {
 				cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
 			} else {
 				if len(args) == 0 {
-					log.Fatalf("no target scenario to upgrade")
+					return fmt.Errorf("specify at least one scenario to upgrade or '--all'")
 				}
 				for _, name := range args {
 					cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, forceAction)
 				}
 			}
+			return nil
 		},
 	}
 	cmdScenariosUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the scenarios")

cmd/crowdsec-cli/setup.go

@@ -112,6 +112,20 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
+	var detectReader *os.File
+
+	switch detectConfigFile {
+	case "-":
+		log.Tracef("Reading detection rules from stdin")
+		detectReader = os.Stdin
+	default:
+		log.Tracef("Reading detection rules: %s", detectConfigFile)
+		detectReader, err = os.Open(detectConfigFile)
+		if err != nil {
+			return err
+		}
+	}
+
 	listSupportedServices, err := flags.GetBool("list-supported-services")
 	if err != nil {
 		return err
@@ -171,7 +185,7 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
 	}
 
 	if listSupportedServices {
-		supported, err := setup.ListSupported(detectConfigFile)
+		supported, err := setup.ListSupported(detectReader)
 		if err != nil {
 			return err
 		}
@@ -195,7 +209,7 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
 		SnubSystemd:  snubSystemd,
 	}
 
-	hubSetup, err := setup.Detect(detectConfigFile, opts)
+	hubSetup, err := setup.Detect(detectReader, opts)
 	if err != nil {
 		return fmt.Errorf("detecting services: %w", err)
 	}

cmd/crowdsec-cli/simulation.go

@@ -3,12 +3,13 @@ package main
 import (
 	"fmt"
 	"os"
+	"slices"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"golang.org/x/exp/slices"
 	"gopkg.in/yaml.v2"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
@@ -144,13 +145,9 @@ func NewSimulationEnableCmd() *cobra.Command {
 		Example:           `cscli simulation enable`,
 		DisableAutoGenTag: true,
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := csConfig.LoadHub(); err != nil {
+			if err := require.Hub(csConfig); err != nil {
 				log.Fatal(err)
 			}
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Info("Run 'sudo cscli hub update' to get the hub index")
-				log.Fatalf("Failed to get Hub index : %v", err)
-			}
 
 			if len(args) > 0 {
 				for _, scenario := range args {

cmd/crowdsec-cli/support.go

@@ -18,8 +18,9 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/version"
+	"github.com/crowdsecurity/go-cs-lib/version"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
@@ -131,24 +132,6 @@ func collectOSInfo() ([]byte, error) {
 	return w.Bytes(), nil
 }
 
-func initHub() error {
-	if err := csConfig.LoadHub(); err != nil {
-		return fmt.Errorf("cannot load hub: %s", err)
-	}
-	if csConfig.Hub == nil {
-		return fmt.Errorf("hub not configured")
-	}
-
-	if err := cwhub.SetHubBranch(); err != nil {
-		return fmt.Errorf("cannot set hub branch: %s", err)
-	}
-
-	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-		return fmt.Errorf("no hub index found: %s", err)
-	}
-	return nil
-}
-
 func collectHubItems(itemType string) []byte {
 	out := bytes.NewBuffer(nil)
 	log.Infof("Collecting %s list", itemType)
@@ -184,7 +167,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
 	if err != nil {
 		return []byte(fmt.Sprintf("cannot parse API URL: %s", err))
 	}
-	scenarios, err := cwhub.GetInstalledScenariosAsString()
+	scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
 	if err != nil {
 		return []byte(fmt.Sprintf("could not collect scenarios: %s", err))
 	}
@@ -312,8 +295,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				skipAgent = true
 			}
 
-			err = initHub()
-			if err != nil {
+			if err := require.Hub(csConfig); err != nil {
 				log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
 				skipHub = true
 				infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())

cmd/crowdsec-cli/utils.go

@@ -9,6 +9,7 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -19,10 +20,9 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/agext/levenshtein"
-	"golang.org/x/exp/slices"
 	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
+	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
@@ -120,25 +120,12 @@ func compInstalledItems(itemType string, args []string, toComplete string) ([]st
 		return nil, cobra.ShellCompDirectiveDefault
 	}
 
-	var items []string
-	var err error
-	switch itemType {
-	case cwhub.PARSERS:
-		items, err = cwhub.GetInstalledParsersAsString()
-	case cwhub.SCENARIOS:
-		items, err = cwhub.GetInstalledScenariosAsString()
-	case cwhub.PARSERS_OVFLW:
-		items, err = cwhub.GetInstalledPostOverflowsAsString()
-	case cwhub.COLLECTIONS:
-		items, err = cwhub.GetInstalledCollectionsAsString()
-	default:
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
+	items, err := cwhub.GetInstalledItemsAsString(itemType)
 	if err != nil {
 		cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
 		return nil, cobra.ShellCompDirectiveDefault
 	}
+
 	comp := make([]string, 0)
 
 	if toComplete != "" {

cmd/crowdsec/Makefile

@@ -4,10 +4,9 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-# Go parameters
-GOCMD = go
-GOBUILD = $(GOCMD) build
-GOTEST = $(GOCMD) test
+GO = go
+GOBUILD = $(GO) build
+GOTEST = $(GO) test
 
 CROWDSEC_BIN = crowdsec$(EXT)
 # names longer than 15 chars break 'pgrep'
@@ -23,7 +22,7 @@ SYSTEMD_PATH_FILE = "/etc/systemd/system/crowdsec.service"
 all: clean test build
 
 build: clean
-	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(CROWDSEC_BIN)
+	$(GOBUILD) $(LD_OPTS) -o $(CROWDSEC_BIN)
 
 test:
 	$(GOTEST) $(LD_OPTS) -v ./...

cmd/crowdsec/api.go

@@ -8,7 +8,7 @@ import (
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
+	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/apiserver"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"

cmd/crowdsec/crowdsec.go

@@ -10,7 +10,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
+	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"

cmd/crowdsec/main.go

@@ -138,11 +138,13 @@ func (l *labelsMap) String() string {
 }
 
 func (l labelsMap) Set(label string) error {
-	split := strings.Split(label, ":")
-	if len(split) != 2 {
-		return errors.Wrapf(errors.New("Bad Format"), "for Label '%s'", label)
+	for _, pair := range strings.Split(label, ",") {
+		split := strings.Split(pair, ":")
+		if len(split) != 2 {
+			return fmt.Errorf("invalid format for label '%s', must be key:value", pair)
+		}
+		l[split[0]] = split[1]
 	}
-	l[split[0]] = split[1]
 	return nil
 }
 
@@ -249,13 +251,13 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 		return nil, err
 	}
 
-	if !flags.DisableAgent {
+	if !cConfig.DisableAgent {
 		if err := cConfig.LoadCrowdsec(); err != nil {
 			return nil, err
 		}
 	}
 
-	if !flags.DisableAPI {
+	if !cConfig.DisableAPI {
 		if err := cConfig.LoadAPIServer(); err != nil {
 			return nil, err
 		}
@@ -290,7 +292,7 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 			cConfig.API.Server.OnlineClient = nil
 		}
 		/*if the api is disabled as well, just read file and exit, don't daemonize*/
-		if flags.DisableAPI {
+		if cConfig.DisableAPI {
 			cConfig.Common.Daemonize = false
 		}
 		log.Infof("single file mode : log_media=%s daemonize=%t", cConfig.Common.LogMedia, cConfig.Common.Daemonize)

cmd/crowdsec/metrics.go

@@ -9,8 +9,8 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
-	"github.com/crowdsecurity/go-cs-lib/pkg/version"
+	"github.com/crowdsecurity/go-cs-lib/trace"
+	"github.com/crowdsecurity/go-cs-lib/version"
 
 	v1 "github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers/v1"
 	"github.com/crowdsecurity/crowdsec/pkg/cache"

cmd/crowdsec/output.go

@@ -10,7 +10,7 @@ import (
 	"github.com/go-openapi/strfmt"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/version"
+	"github.com/crowdsecurity/go-cs-lib/version"
 
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
@@ -70,7 +70,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
 	var cache []types.RuntimeAlert
 	var cacheMutex sync.Mutex
 
-	scenarios, err := cwhub.GetInstalledScenariosAsString()
+	scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
 	if err != nil {
 		return fmt.Errorf("loading list of installed hub scenarios: %w", err)
 	}
@@ -93,7 +93,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
 		URL:            apiURL,
 		PapiURL:        papiURL,
 		VersionPrefix:  "v1",
-		UpdateScenario: cwhub.GetInstalledScenariosAsString,
+		UpdateScenario: func() ([]string, error) {return cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)},
 	})
 	if err != nil {
 		return fmt.Errorf("new client api: %w", err)

cmd/crowdsec/run_in_svc.go

@@ -10,8 +10,8 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/sirupsen/logrus/hooks/writer"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
-	"github.com/crowdsecurity/go-cs-lib/pkg/version"
+	"github.com/crowdsecurity/go-cs-lib/trace"
+	"github.com/crowdsecurity/go-cs-lib/version"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/database"

cmd/crowdsec/run_in_svc_windows.go

@@ -6,8 +6,8 @@ import (
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/windows/svc"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
-	"github.com/crowdsecurity/go-cs-lib/pkg/version"
+	"github.com/crowdsecurity/go-cs-lib/trace"
+	"github.com/crowdsecurity/go-cs-lib/version"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/database"

cmd/crowdsec/serve.go

@@ -10,8 +10,8 @@ import (
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/tomb.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/csdaemon"
-	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
+	"github.com/crowdsecurity/go-cs-lib/csdaemon"
+	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
@@ -141,12 +141,24 @@ func ShutdownCrowdsecRoutines() error {
 	time.Sleep(1 * time.Second) // ugly workaround for now
 	outputsTomb.Kill(nil)
 
-	if err := outputsTomb.Wait(); err != nil {
-		log.Warningf("Ouputs returned error : %s", err)
-		reterr = err
+	done := make(chan error, 1)
+	go func() {
+		done <- outputsTomb.Wait()
+	}()
+
+	// wait for outputs to finish, max 3 seconds
+	select {
+	case err := <-done:
+		if err != nil {
+			log.Warningf("Outputs returned error : %s", err)
+			reterr = err
+		}
+		log.Debugf("outputs are done")
+	case <-time.After(3 * time.Second):
+		// this can happen if outputs are stuck in a http retry loop
+		log.Warningf("Outputs didn't finish in time, some events may have not been flushed")
 	}
 
-	log.Debugf("outputs are done")
 	// He's dead, Jim.
 	crowdsecTomb.Kill(nil)
 

plugins/notifications/http/Makefile → cmd/notification-dummy/Makefile

@@ -4,14 +4,13 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-PLUGIN=http
-BINARY_NAME = notification-$(PLUGIN)$(EXT)
+GO = go
+GOBUILD = $(GO) build
 
-GOCMD = go
-GOBUILD = $(GOCMD) build
+BINARY_NAME = notification-dummy$(EXT)
 
 build: clean
-	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
+	$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
 
 .PHONY: clean
 clean:

plugins/notifications/slack/Makefile → cmd/notification-email/Makefile

@@ -4,14 +4,13 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-PLUGIN=slack
-BINARY_NAME = notification-$(PLUGIN)$(EXT)
+GO = go
+GOBUILD = $(GO) build
 
-GOCMD = go
-GOBUILD = $(GOCMD) build
+BINARY_NAME = notification-email$(EXT)
 
 build: clean
-	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
+	$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
 
 .PHONY: clean
 clean:

plugins/notifications/email/email.yaml → cmd/notification-email/email.yaml

@@ -15,12 +15,14 @@ timeout: 20s          # Time to wait for response from the plugin before conside
 # The following template receives a list of models.Alert objects
 # The output goes in the email message body
 format: |
+  <html><body>
   {{range . -}}
     {{$alert := . -}}
     {{range .Decisions -}}
-      <html><body><p><a href=https://www.whois.com/whois/{{.Value}}>{{.Value}}</a> will get <b>{{.Type}}</b> for next <b>{{.Duration}}</b> for triggering <b>{{.Scenario}}</b> on machine <b>{{$alert.MachineID}}</b>.</p> <p><a href=https://app.crowdsec.net/cti/{{.Value}}>CrowdSec CTI</a></p></body></html>
+      <p><a href="https://www.whois.com/whois/{{.Value}}">{{.Value}}</a> will get <b>{{.Type}}</b> for next <b>{{.Duration}}</b> for triggering <b>{{.Scenario}}</b> on machine <b>{{$alert.MachineID}}</b>.</p> <p><a href="https://app.crowdsec.net/cti/{{.Value}}">CrowdSec CTI</a></p>
     {{end -}}
   {{end -}}
+  </body></html>
 
 smtp_host:            # example: smtp.gmail.com
 smtp_username:        # Replace with your actual username
@@ -35,7 +37,15 @@ receiver_emails:
 # - email2@gmail.com
 
 # One of "ssltls", "starttls", "none"
-encryption_type: ssltls
+encryption_type: "ssltls"
+
+# If you need to set the HELO hostname:
+# helo_host: "localhost"
+
+# If the email server is hitting the default timeouts (10 seconds), you can increase them here
+#
+# connect_timeout: 10s
+# send_timeout: 10s
 
 ---
 

plugins/notifications/email/main.go → cmd/notification-email/main.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"time"
 
 	"github.com/crowdsecurity/crowdsec/pkg/protobufs"
 	"github.com/hashicorp/go-hclog"
@@ -47,6 +48,8 @@ type PluginConfig struct {
 	EncryptionType string   `yaml:"encryption_type"`
 	AuthType       string   `yaml:"auth_type"`
 	HeloHost       string   `yaml:"helo_host"`
+	ConnectTimeout string   `yaml:"connect_timeout"`
+	SendTimeout    string   `yaml:"send_timeout"`
 }
 
 type EmailPlugin struct {
@@ -77,7 +80,7 @@ func (n *EmailPlugin) Configure(ctx context.Context, config *protobufs.Config) (
 	}
 
 	if d.ReceiverEmails == nil || len(d.ReceiverEmails) == 0 {
-		return nil, fmt.Errorf("Receiver emails are not set")
+		return nil, fmt.Errorf("receiver emails are not set")
 	}
 
 	n.ConfigByName[d.Name] = d
@@ -108,6 +111,24 @@ func (n *EmailPlugin) Notify(ctx context.Context, notification *protobufs.Notifi
 	server.Authentication = AuthStringToType[cfg.AuthType]
 	server.Helo = cfg.HeloHost
 
+	var err error
+
+	if cfg.ConnectTimeout != "" {
+		server.ConnectTimeout, err = time.ParseDuration(cfg.ConnectTimeout)
+		if err != nil {
+			logger.Warn(fmt.Sprintf("invalid connect timeout '%s', using default '10s'", cfg.ConnectTimeout))
+			server.ConnectTimeout = 10 * time.Second
+		}
+	}
+
+	if cfg.SendTimeout != "" {
+		server.SendTimeout, err = time.ParseDuration(cfg.SendTimeout)
+		if err != nil {
+			logger.Warn(fmt.Sprintf("invalid send timeout '%s', using default '10s'", cfg.SendTimeout))
+			server.SendTimeout = 10 * time.Second
+		}
+	}
+
 	logger.Debug("making smtp connection")
 	smtpClient, err := server.Connect()
 	if err != nil {

plugins/notifications/splunk/Makefile → cmd/notification-http/Makefile

@@ -4,14 +4,13 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-PLUGIN=splunk
-BINARY_NAME = notification-$(PLUGIN)$(EXT)
+GO = go
+GOBUILD = $(GO) build
 
-GOCMD = go
-GOBUILD = $(GOCMD) build
+BINARY_NAME = notification-http$(EXT)
 
 build: clean
-	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
+	$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
 
 .PHONY: clean
 clean:

plugins/notifications/http/main.go → cmd/notification-http/main.go

@@ -63,7 +63,7 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
 		logger.Debug(fmt.Sprintf("adding header %s: %s", headerName, headerValue))
 		request.Header.Add(headerName, headerValue)
 	}
-	logger.Debug(fmt.Sprintf("making HTTP %s call to %s with body %s", cfg.Method, cfg.URL, string(notification.Text)))
+	logger.Debug(fmt.Sprintf("making HTTP %s call to %s with body %s", cfg.Method, cfg.URL, notification.Text))
 	resp, err := client.Do(request)
 	if err != nil {
 		logger.Error(fmt.Sprintf("Failed to make HTTP request : %s", err))

plugins/notifications/dummy/Makefile → cmd/notification-sentinel/Makefile

@@ -4,14 +4,13 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-PLUGIN = dummy
-BINARY_NAME = notification-$(PLUGIN)$(EXT)
+GO = go
+GOBUILD = $(GO) build
 
-GOCMD = go
-GOBUILD = $(GOCMD) build
+BINARY_NAME = notification-sentinel$(EXT)
 
 build: clean
-	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
+	$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
 
 .PHONY: clean
 clean:

cmd/notification-sentinel/main.go

@@ -0,0 +1,133 @@
+package main
+
+import (
+	"context"
+	"crypto/hmac"
+	"crypto/sha256"
+	"encoding/base64"
+	"fmt"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/crowdsecurity/crowdsec/pkg/protobufs"
+	"github.com/hashicorp/go-hclog"
+	"github.com/hashicorp/go-plugin"
+	"gopkg.in/yaml.v3"
+)
+
+type PluginConfig struct {
+	Name       string  `yaml:"name"`
+	CustomerID string  `yaml:"customer_id"`
+	SharedKey  string  `yaml:"shared_key"`
+	LogType    string  `yaml:"log_type"`
+	LogLevel   *string `yaml:"log_level"`
+}
+
+type SentinelPlugin struct {
+	PluginConfigByName map[string]PluginConfig
+}
+
+var logger hclog.Logger = hclog.New(&hclog.LoggerOptions{
+	Name:       "sentinel-plugin",
+	Level:      hclog.LevelFromString("INFO"),
+	Output:     os.Stderr,
+	JSONFormat: true,
+})
+
+func (s *SentinelPlugin) getAuthorizationHeader(now string, length int, pluginName string) (string, error) {
+	xHeaders := "x-ms-date:" + now
+
+	stringToHash := fmt.Sprintf("POST\n%d\napplication/json\n%s\n/api/logs", length, xHeaders)
+	decodedKey, _ := base64.StdEncoding.DecodeString(s.PluginConfigByName[pluginName].SharedKey)
+
+	h := hmac.New(sha256.New, decodedKey)
+	h.Write([]byte(stringToHash))
+
+	encodedHash := base64.StdEncoding.EncodeToString(h.Sum(nil))
+	authorization := "SharedKey " + s.PluginConfigByName[pluginName].CustomerID + ":" + encodedHash
+
+	logger.Trace("authorization header", "header", authorization)
+
+	return authorization, nil
+}
+
+func (s *SentinelPlugin) Notify(ctx context.Context, notification *protobufs.Notification) (*protobufs.Empty, error) {
+
+	if _, ok := s.PluginConfigByName[notification.Name]; !ok {
+		return nil, fmt.Errorf("invalid plugin config name %s", notification.Name)
+	}
+	cfg := s.PluginConfigByName[notification.Name]
+
+	if cfg.LogLevel != nil && *cfg.LogLevel != "" {
+		logger.SetLevel(hclog.LevelFromString(*cfg.LogLevel))
+	}
+
+	logger.Info("received notification for sentinel config", "name", notification.Name)
+
+	url := fmt.Sprintf("https://%s.ods.opinsights.azure.com/api/logs?api-version=2016-04-01", s.PluginConfigByName[notification.Name].CustomerID)
+	body := strings.NewReader(notification.Text)
+
+	//Cannot use time.RFC1123 as azure wants GMT, not UTC
+	now := time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT")
+
+	authorization, err := s.getAuthorizationHeader(now, len(notification.Text), notification.Name)
+
+	if err != nil {
+		return &protobufs.Empty{}, err
+	}
+
+	req, err := http.NewRequest(http.MethodPost, url, body)
+	if err != nil {
+		logger.Error("failed to create request", "error", err)
+		return &protobufs.Empty{}, err
+	}
+
+	req.Header.Set("Content-Type", "application/json")
+	req.Header.Set("Log-Type", s.PluginConfigByName[notification.Name].LogType)
+	req.Header.Set("Authorization", authorization)
+	req.Header.Set("x-ms-date", now)
+
+	client := &http.Client{}
+	resp, err := client.Do(req)
+	if err != nil {
+		logger.Error("failed to send request", "error", err)
+		return &protobufs.Empty{}, err
+	}
+	defer resp.Body.Close()
+	logger.Debug("sent notification to sentinel", "status", resp.Status)
+
+	if resp.StatusCode != http.StatusOK {
+		return &protobufs.Empty{}, fmt.Errorf("failed to send notification to sentinel: %s", resp.Status)
+	}
+
+	return &protobufs.Empty{}, nil
+}
+
+func (s *SentinelPlugin) Configure(ctx context.Context, config *protobufs.Config) (*protobufs.Empty, error) {
+	d := PluginConfig{}
+	err := yaml.Unmarshal(config.Config, &d)
+	s.PluginConfigByName[d.Name] = d
+	return &protobufs.Empty{}, err
+}
+
+func main() {
+	var handshake = plugin.HandshakeConfig{
+		ProtocolVersion:  1,
+		MagicCookieKey:   "CROWDSEC_PLUGIN_KEY",
+		MagicCookieValue: os.Getenv("CROWDSEC_PLUGIN_KEY"),
+	}
+
+	sp := &SentinelPlugin{PluginConfigByName: make(map[string]PluginConfig)}
+	plugin.Serve(&plugin.ServeConfig{
+		HandshakeConfig: handshake,
+		Plugins: map[string]plugin.Plugin{
+			"sentinel": &protobufs.NotifierPlugin{
+				Impl: sp,
+			},
+		},
+		GRPCServer: plugin.DefaultGRPCServer,
+		Logger:     logger,
+	})
+}

cmd/notification-sentinel/sentinel.yaml

@@ -0,0 +1,21 @@
+type: sentinel          # Don't change
+name: sentinel_default  # Must match the registered plugin in the profile
+
+# One of "trace", "debug", "info", "warn", "error", "off"
+log_level: info
+# group_wait:         # Time to wait collecting alerts before relaying a message to this plugin, eg "30s"
+# group_threshold:    # Amount of alerts that triggers a message before <group_wait> has expired, eg "10"
+# max_retry:          # Number of attempts to relay messages to plugins in case of error
+# timeout:            # Time to wait for response from the plugin before considering the attempt a failure, eg "10s"
+
+#-------------------------
+# plugin-specific options
+
+# The following template receives a list of models.Alert objects
+# The output goes in the http request body
+format: |
+  {{.|toJson}}
+
+customer_id: XXX-XXX
+shared_key: XXXXXXX
+log_type: crowdsec

cmd/notification-slack/Makefile

@@ -0,0 +1,17 @@
+ifeq ($(OS), Windows_NT)
+	SHELL := pwsh.exe
+	.SHELLFLAGS := -NoProfile -Command
+	EXT = .exe
+endif
+
+GO = go
+GOBUILD = $(GO) build
+
+BINARY_NAME = notification-slack$(EXT)
+
+build: clean
+	$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
+
+.PHONY: clean
+clean:
+	@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)

cmd/notification-splunk/Makefile

@@ -0,0 +1,17 @@
+ifeq ($(OS), Windows_NT)
+	SHELL := pwsh.exe
+	.SHELLFLAGS := -NoProfile -Command
+	EXT = .exe
+endif
+
+GO = go
+GOBUILD = $(GO) build
+
+BINARY_NAME = notification-splunk$(EXT)
+
+build: clean
+	$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
+
+.PHONY: clean
+clean:
+	@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)

plugins/notifications/splunk/main.go → cmd/notification-splunk/main.go

@@ -58,7 +58,7 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio
 		return &protobufs.Empty{}, err
 	}
 
-	req, err := http.NewRequest("POST", cfg.URL, strings.NewReader(string(data)))
+	req, err := http.NewRequest(http.MethodPost, cfg.URL, strings.NewReader(string(data)))
 	if err != nil {
 		return &protobufs.Empty{}, err
 	}
@@ -70,7 +70,7 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio
 		return &protobufs.Empty{}, err
 	}
 
-	if resp.StatusCode != 200 {
+	if resp.StatusCode != http.StatusOK {
 		content, err := io.ReadAll(resp.Body)
 		if err != nil {
 			return &protobufs.Empty{}, fmt.Errorf("got non 200 response and failed to read error %s", err)

debian/crowdsec.service

@@ -5,7 +5,7 @@ After=syslog.target network.target remote-fs.target nss-lookup.target
 [Service]
 Type=notify
 Environment=LC_ALL=C LANG=C
-ExecStartPre=/usr/bin/crowdsec -c /etc/crowdsec/config.yaml -t
+ExecStartPre=/usr/bin/crowdsec -c /etc/crowdsec/config.yaml -t -error
 ExecStart=/usr/bin/crowdsec -c /etc/crowdsec/config.yaml
 #ExecStartPost=/bin/sleep 0.1
 ExecReload=/bin/kill -HUP $MAINPID

debian/install

@@ -6,7 +6,8 @@ config/patterns/*       etc/crowdsec/patterns
 config/crowdsec.service lib/systemd/system
 
 # Referenced configs:
-plugins/notifications/slack/slack.yaml      etc/crowdsec/notifications/
-plugins/notifications/http/http.yaml        etc/crowdsec/notifications/
-plugins/notifications/splunk/splunk.yaml    etc/crowdsec/notifications/
-plugins/notifications/email/email.yaml      etc/crowdsec/notifications/
+cmd/notification-slack/slack.yaml        etc/crowdsec/notifications/
+cmd/notification-http/http.yaml          etc/crowdsec/notifications/
+cmd/notification-splunk/splunk.yaml      etc/crowdsec/notifications/
+cmd/notification-email/email.yaml        etc/crowdsec/notifications/
+cmd/notification-sentinel/sentinel.yaml  etc/crowdsec/notifications/

debian/rules

@@ -25,10 +25,11 @@ override_dh_auto_install:
 	mkdir -p debian/crowdsec/usr/lib/crowdsec/plugins/
 	mkdir -p debian/crowdsec/etc/crowdsec/notifications/
 
-	install -m 551 plugins/notifications/slack/notification-slack debian/crowdsec/usr/lib/crowdsec/plugins/
-	install -m 551 plugins/notifications/http/notification-http debian/crowdsec/usr/lib/crowdsec/plugins/
-	install -m 551 plugins/notifications/splunk/notification-splunk debian/crowdsec/usr/lib/crowdsec/plugins/
-	install -m 551 plugins/notifications/email/notification-email debian/crowdsec/usr/lib/crowdsec/plugins/
+	install -m 551 cmd/notification-slack/notification-slack debian/crowdsec/usr/lib/crowdsec/plugins/
+	install -m 551 cmd/notification-http/notification-http debian/crowdsec/usr/lib/crowdsec/plugins/
+	install -m 551 cmd/notification-splunk/notification-splunk debian/crowdsec/usr/lib/crowdsec/plugins/
+	install -m 551 cmd/notification-email/notification-email debian/crowdsec/usr/lib/crowdsec/plugins/
+	install -m 551 cmd/notification-sentinel/notification-sentinel debian/crowdsec/usr/lib/crowdsec/plugins/
 
 	cp cmd/crowdsec/crowdsec debian/crowdsec/usr/bin
 	cp cmd/crowdsec-cli/cscli debian/crowdsec/usr/bin

docker/docker_start.sh

@@ -243,7 +243,7 @@ if istrue "$DISABLE_ONLINE_API"; then
 fi
 
 # registration to online API for signal push
-if isfalse "$DISABLE_ONLINE_API" ; then
+if isfalse "$DISABLE_LOCAL_API" && isfalse "$DISABLE_ONLINE_API" ; then
     CONFIG_DIR=$(conf_get '.config_paths.config_dir')
     export CONFIG_DIR
     config_exists=$(conf_get '.api.server.online_client | has("credentials_path")')
@@ -255,7 +255,7 @@ if isfalse "$DISABLE_ONLINE_API" ; then
 fi
 
 # Enroll instance if enroll key is provided
-if isfalse "$DISABLE_ONLINE_API" && [ "$ENROLL_KEY" != "" ]; then
+if isfalse "$DISABLE_LOCAL_API" && isfalse "$DISABLE_ONLINE_API" && [ "$ENROLL_KEY" != "" ]; then
     enroll_args=""
     if [ "$ENROLL_INSTANCE_NAME" != "" ]; then
         enroll_args="--name $ENROLL_INSTANCE_NAME"
@@ -273,13 +273,14 @@ fi
 # crowdsec sqlite database permissions
 if [ "$GID" != "" ]; then
     if istrue "$(conf_get '.db_config.type == "sqlite"')"; then
-        chown ":$GID" "$(conf_get '.db_config.db_path')"
-        echo "sqlite database permissions updated"
+        # don't fail if the db is not there yet
+        chown -f ":$GID" "$(conf_get '.db_config.db_path')" 2>/dev/null \
+            && echo "sqlite database permissions updated" \
+            || true
     fi
 fi
 
-# XXX only with LAPI
-if istrue "$USE_TLS"; then
+if isfalse "$DISABLE_LOCAL_API" && istrue "$USE_TLS"; then
     agents_allowed_yaml=$(csv2yaml "$AGENTS_ALLOWED_OU")
     export agents_allowed_yaml
     bouncers_allowed_yaml=$(csv2yaml "$BOUNCERS_ALLOWED_OU")
@@ -358,7 +359,7 @@ shopt -s nullglob extglob
 for BOUNCER in /run/secrets/@(bouncer_key|BOUNCER_KEY)* ; do
     KEY=$(cat "${BOUNCER}")
     NAME=$(echo "${BOUNCER}" | awk -F "/" '{printf $NF}' | cut -d_  -f2-)
-    if [[ -n $KEY ]] && [[ -n $NAME ]]; then    
+    if [[ -n $KEY ]] && [[ -n $NAME ]]; then
         register_bouncer "$NAME" "$KEY"
     fi
 done
@@ -369,6 +370,12 @@ shopt -u nullglob extglob
 conf_set_if "$CAPI_WHITELISTS_PATH" '.api.server.capi_whitelists_path = strenv(CAPI_WHITELISTS_PATH)'
 conf_set_if "$METRICS_PORT" '.prometheus.listen_port=env(METRICS_PORT)'
 
+if istrue "$DISABLE_LOCAL_API"; then
+    conf_set '.api.server.enable=false'
+else
+    conf_set '.api.server.enable=true'
+fi
+
 ARGS=""
 if [ "$CONFIG_FILE" != "" ]; then
     ARGS="-c $CONFIG_FILE"
@@ -390,10 +397,6 @@ if istrue "$DISABLE_AGENT"; then
     ARGS="$ARGS -no-cs"
 fi
 
-if istrue "$DISABLE_LOCAL_API"; then
-    ARGS="$ARGS -no-api"
-fi
-
 if istrue "$LEVEL_TRACE"; then
     ARGS="$ARGS -trace"
 fi

docker/test/Pipfile

@@ -1,7 +1,7 @@
 [packages]
 pytest-dotenv = "0.5.2"
 pytest-xdist = "3.3.1"
-pytest-cs = {ref = "0.7.16", git = "https://github.com/crowdsecurity/pytest-cs.git"}
+pytest-cs = {ref = "0.7.18", git = "https://github.com/crowdsecurity/pytest-cs.git"}
 
 [dev-packages]
 gnureadline = "8.1.2"

docker/test/Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "78f693678e411b7bdb5dd0280b7d6f8d9880069b331d44d96d32ba697275e30d"
+            "sha256": "64085783c9fec3a9eda976b7700b5bad7abd2b7a0f0670fa2209c52f3647be7f"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -18,11 +18,11 @@
     "default": {
         "certifi": {
             "hashes": [
-                "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7",
-                "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"
+                "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082",
+                "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2023.5.7"
+            "version": "==2023.7.22"
         },
         "cffi": {
             "hashes": [
@@ -176,32 +176,32 @@
         },
         "cryptography": {
             "hashes": [
-                "sha256:01f1d9e537f9a15b037d5d9ee442b8c22e3ae11ce65ea1f3316a41c78756b711",
-                "sha256:079347de771f9282fbfe0e0236c716686950c19dee1b76240ab09ce1624d76d7",
-                "sha256:182be4171f9332b6741ee818ec27daff9fb00349f706629f5cbf417bd50e66fd",
-                "sha256:192255f539d7a89f2102d07d7375b1e0a81f7478925b3bc2e0549ebf739dae0e",
-                "sha256:2a034bf7d9ca894720f2ec1d8b7b5832d7e363571828037f9e0c4f18c1b58a58",
-                "sha256:342f3767e25876751e14f8459ad85e77e660537ca0a066e10e75df9c9e9099f0",
-                "sha256:439c3cc4c0d42fa999b83ded80a9a1fb54d53c58d6e59234cfe97f241e6c781d",
-                "sha256:49c3222bb8f8e800aead2e376cbef687bc9e3cb9b58b29a261210456a7783d83",
-                "sha256:674b669d5daa64206c38e507808aae49904c988fa0a71c935e7006a3e1e83831",
-                "sha256:7a9a3bced53b7f09da251685224d6a260c3cb291768f54954e28f03ef14e3766",
-                "sha256:7af244b012711a26196450d34f483357e42aeddb04128885d95a69bd8b14b69b",
-                "sha256:7d230bf856164de164ecb615ccc14c7fc6de6906ddd5b491f3af90d3514c925c",
-                "sha256:84609ade00a6ec59a89729e87a503c6e36af98ddcd566d5f3be52e29ba993182",
-                "sha256:9a6673c1828db6270b76b22cc696f40cde9043eb90373da5c2f8f2158957f42f",
-                "sha256:9b6d717393dbae53d4e52684ef4f022444fc1cce3c48c38cb74fca29e1f08eaa",
-                "sha256:9c3fe6534d59d071ee82081ca3d71eed3210f76ebd0361798c74abc2bcf347d4",
-                "sha256:a719399b99377b218dac6cf547b6ec54e6ef20207b6165126a280b0ce97e0d2a",
-                "sha256:b332cba64d99a70c1e0836902720887fb4529ea49ea7f5462cf6640e095e11d2",
-                "sha256:d124682c7a23c9764e54ca9ab5b308b14b18eba02722b8659fb238546de83a76",
-                "sha256:d73f419a56d74fef257955f51b18d046f3506270a5fd2ac5febbfa259d6c0fa5",
-                "sha256:f0dc40e6f7aa37af01aba07277d3d64d5a03dc66d682097541ec4da03cc140ee",
-                "sha256:f14ad275364c8b4e525d018f6716537ae7b6d369c094805cae45300847e0894f",
-                "sha256:f772610fe364372de33d76edcd313636a25684edb94cee53fd790195f5989d14"
+                "sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67",
+                "sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311",
+                "sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8",
+                "sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13",
+                "sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143",
+                "sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f",
+                "sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829",
+                "sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd",
+                "sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397",
+                "sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac",
+                "sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d",
+                "sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a",
+                "sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839",
+                "sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e",
+                "sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6",
+                "sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9",
+                "sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860",
+                "sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca",
+                "sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91",
+                "sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d",
+                "sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714",
+                "sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb",
+                "sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==41.0.2"
+            "version": "==41.0.4"
         },
         "docker": {
             "hashes": [
@@ -245,11 +245,11 @@
         },
         "pluggy": {
             "hashes": [
-                "sha256:c2fd55a7d7a3863cba1a013e4e2414658b1d07b6bc57b3919e0c63c9abb99849",
-                "sha256:d12f0c4b579b15f5e054301bb226ee85eeeba08ffec228092f8defbaa3a4c4b3"
+                "sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12",
+                "sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.2.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.3.0"
         },
         "psutil": {
             "hashes": [
@@ -280,15 +280,15 @@
         },
         "pytest": {
             "hashes": [
-                "sha256:78bf16451a2eb8c7a2ea98e32dc119fd2aa758f1d5d66dbf0a59d69a3969df32",
-                "sha256:b4bf8c45bd59934ed84001ad51e11b4ee40d40a1229d2c79f9c592b0a3f6bd8a"
+                "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002",
+                "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==7.4.0"
+            "version": "==7.4.2"
         },
         "pytest-cs": {
             "git": "https://github.com/crowdsecurity/pytest-cs.git",
-            "ref": "4a3451084215053af8a48ff37507b4f86bf75c10"
+            "ref": "df835beabc539be7f7f627b21caa0d6ad333daae"
         },
         "pytest-datadir": {
             "hashes": [
@@ -324,49 +324,59 @@
         },
         "pyyaml": {
             "hashes": [
-                "sha256:01b45c0191e6d66c470b6cf1b9531a771a83c1c4208272ead47a3ae4f2f603bf",
-                "sha256:0283c35a6a9fbf047493e3a0ce8d79ef5030852c51e9d911a27badfde0605293",
-                "sha256:055d937d65826939cb044fc8c9b08889e8c743fdc6a32b33e2390f66013e449b",
-                "sha256:07751360502caac1c067a8132d150cf3d61339af5691fe9e87803040dbc5db57",
-                "sha256:0b4624f379dab24d3725ffde76559cff63d9ec94e1736b556dacdfebe5ab6d4b",
-                "sha256:0ce82d761c532fe4ec3f87fc45688bdd3a4c1dc5e0b4a19814b9009a29baefd4",
-                "sha256:1e4747bc279b4f613a09eb64bba2ba602d8a6664c6ce6396a4d0cd413a50ce07",
-                "sha256:213c60cd50106436cc818accf5baa1aba61c0189ff610f64f4a3e8c6726218ba",
-                "sha256:231710d57adfd809ef5d34183b8ed1eeae3f76459c18fb4a0b373ad56bedcdd9",
-                "sha256:277a0ef2981ca40581a47093e9e2d13b3f1fbbeffae064c1d21bfceba2030287",
-                "sha256:2cd5df3de48857ed0544b34e2d40e9fac445930039f3cfe4bcc592a1f836d513",
-                "sha256:40527857252b61eacd1d9af500c3337ba8deb8fc298940291486c465c8b46ec0",
-                "sha256:432557aa2c09802be39460360ddffd48156e30721f5e8d917f01d31694216782",
-                "sha256:473f9edb243cb1935ab5a084eb238d842fb8f404ed2193a915d1784b5a6b5fc0",
-                "sha256:48c346915c114f5fdb3ead70312bd042a953a8ce5c7106d5bfb1a5254e47da92",
-                "sha256:50602afada6d6cbfad699b0c7bb50d5ccffa7e46a3d738092afddc1f9758427f",
-                "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2",
-                "sha256:77f396e6ef4c73fdc33a9157446466f1cff553d979bd00ecb64385760c6babdc",
-                "sha256:81957921f441d50af23654aa6c5e5eaf9b06aba7f0a19c18a538dc7ef291c5a1",
-                "sha256:819b3830a1543db06c4d4b865e70ded25be52a2e0631ccd2f6a47a2822f2fd7c",
-                "sha256:897b80890765f037df3403d22bab41627ca8811ae55e9a722fd0392850ec4d86",
-                "sha256:98c4d36e99714e55cfbaaee6dd5badbc9a1ec339ebfc3b1f52e293aee6bb71a4",
-                "sha256:9df7ed3b3d2e0ecfe09e14741b857df43adb5a3ddadc919a2d94fbdf78fea53c",
-                "sha256:9fa600030013c4de8165339db93d182b9431076eb98eb40ee068700c9c813e34",
-                "sha256:a80a78046a72361de73f8f395f1f1e49f956c6be882eed58505a15f3e430962b",
-                "sha256:afa17f5bc4d1b10afd4466fd3a44dc0e245382deca5b3c353d8b757f9e3ecb8d",
-                "sha256:b3d267842bf12586ba6c734f89d1f5b871df0273157918b0ccefa29deb05c21c",
-                "sha256:b5b9eccad747aabaaffbc6064800670f0c297e52c12754eb1d976c57e4f74dcb",
-                "sha256:bfaef573a63ba8923503d27530362590ff4f576c626d86a9fed95822a8255fd7",
-                "sha256:c5687b8d43cf58545ade1fe3e055f70eac7a5a1a0bf42824308d868289a95737",
-                "sha256:cba8c411ef271aa037d7357a2bc8f9ee8b58b9965831d9e51baf703280dc73d3",
-                "sha256:d15a181d1ecd0d4270dc32edb46f7cb7733c7c508857278d3d378d14d606db2d",
-                "sha256:d4b0ba9512519522b118090257be113b9468d804b19d63c71dbcf4a48fa32358",
-                "sha256:d4db7c7aef085872ef65a8fd7d6d09a14ae91f691dec3e87ee5ee0539d516f53",
-                "sha256:d4eccecf9adf6fbcc6861a38015c2a64f38b9d94838ac1810a9023a0609e1b78",
-                "sha256:d67d839ede4ed1b28a4e8909735fc992a923cdb84e618544973d7dfc71540803",
-                "sha256:daf496c58a8c52083df09b80c860005194014c3698698d1a57cbcfa182142a3a",
-                "sha256:dbad0e9d368bb989f4515da330b88a057617d16b6a8245084f1b05400f24609f",
-                "sha256:e61ceaab6f49fb8bdfaa0f92c4b57bcfbea54c09277b1b4f7ac376bfb7a7c174",
-                "sha256:f84fbc98b019fef2ee9a1cb3ce93e3187a6df0b2538a651bfb890254ba9f90b5"
+                "sha256:04ac92ad1925b2cff1db0cfebffb6ffc43457495c9b3c39d3fcae417d7125dc5",
+                "sha256:062582fca9fabdd2c8b54a3ef1c978d786e0f6b3a1510e0ac93ef59e0ddae2bc",
+                "sha256:0d3304d8c0adc42be59c5f8a4d9e3d7379e6955ad754aa9d6ab7a398b59dd1df",
+                "sha256:1635fd110e8d85d55237ab316b5b011de701ea0f29d07611174a1b42f1444741",
+                "sha256:184c5108a2aca3c5b3d3bf9395d50893a7ab82a38004c8f61c258d4428e80206",
+                "sha256:18aeb1bf9a78867dc38b259769503436b7c72f7a1f1f4c93ff9a17de54319b27",
+                "sha256:1d4c7e777c441b20e32f52bd377e0c409713e8bb1386e1099c2415f26e479595",
+                "sha256:1e2722cc9fbb45d9b87631ac70924c11d3a401b2d7f410cc0e3bbf249f2dca62",
+                "sha256:1fe35611261b29bd1de0070f0b2f47cb6ff71fa6595c077e42bd0c419fa27b98",
+                "sha256:28c119d996beec18c05208a8bd78cbe4007878c6dd15091efb73a30e90539696",
+                "sha256:326c013efe8048858a6d312ddd31d56e468118ad4cdeda36c719bf5bb6192290",
+                "sha256:40df9b996c2b73138957fe23a16a4f0ba614f4c0efce1e9406a184b6d07fa3a9",
+                "sha256:42f8152b8dbc4fe7d96729ec2b99c7097d656dc1213a3229ca5383f973a5ed6d",
+                "sha256:49a183be227561de579b4a36efbb21b3eab9651dd81b1858589f796549873dd6",
+                "sha256:4fb147e7a67ef577a588a0e2c17b6db51dda102c71de36f8549b6816a96e1867",
+                "sha256:50550eb667afee136e9a77d6dc71ae76a44df8b3e51e41b77f6de2932bfe0f47",
+                "sha256:510c9deebc5c0225e8c96813043e62b680ba2f9c50a08d3724c7f28a747d1486",
+                "sha256:5773183b6446b2c99bb77e77595dd486303b4faab2b086e7b17bc6bef28865f6",
+                "sha256:596106435fa6ad000c2991a98fa58eeb8656ef2325d7e158344fb33864ed87e3",
+                "sha256:6965a7bc3cf88e5a1c3bd2e0b5c22f8d677dc88a455344035f03399034eb3007",
+                "sha256:69b023b2b4daa7548bcfbd4aa3da05b3a74b772db9e23b982788168117739938",
+                "sha256:6c22bec3fbe2524cde73d7ada88f6566758a8f7227bfbf93a408a9d86bcc12a0",
+                "sha256:704219a11b772aea0d8ecd7058d0082713c3562b4e271b849ad7dc4a5c90c13c",
+                "sha256:7e07cbde391ba96ab58e532ff4803f79c4129397514e1413a7dc761ccd755735",
+                "sha256:81e0b275a9ecc9c0c0c07b4b90ba548307583c125f54d5b6946cfee6360c733d",
+                "sha256:855fb52b0dc35af121542a76b9a84f8d1cd886ea97c84703eaa6d88e37a2ad28",
+                "sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4",
+                "sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba",
+                "sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8",
+                "sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5",
+                "sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd",
+                "sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3",
+                "sha256:b786eecbdf8499b9ca1d697215862083bd6d2a99965554781d0d8d1ad31e13a0",
+                "sha256:ba336e390cd8e4d1739f42dfe9bb83a3cc2e80f567d8805e11b46f4a943f5515",
+                "sha256:baa90d3f661d43131ca170712d903e6295d1f7a0f595074f151c0aed377c9b9c",
+                "sha256:bc1bf2925a1ecd43da378f4db9e4f799775d6367bdb94671027b73b393a7c42c",
+                "sha256:bd4af7373a854424dabd882decdc5579653d7868b8fb26dc7d0e99f823aa5924",
+                "sha256:bf07ee2fef7014951eeb99f56f39c9bb4af143d8aa3c21b1677805985307da34",
+                "sha256:bfdf460b1736c775f2ba9f6a92bca30bc2095067b8a9d77876d1fad6cc3b4a43",
+                "sha256:c8098ddcc2a85b61647b2590f825f3db38891662cfc2fc776415143f599bb859",
+                "sha256:d2b04aac4d386b172d5b9692e2d2da8de7bfb6c387fa4f801fbf6fb2e6ba4673",
+                "sha256:d483d2cdf104e7c9fa60c544d92981f12ad66a457afae824d146093b8c294c54",
+                "sha256:d858aa552c999bc8a8d57426ed01e40bef403cd8ccdd0fc5f6f04a00414cac2a",
+                "sha256:e7d73685e87afe9f3b36c799222440d6cf362062f78be1013661b00c5c6f678b",
+                "sha256:f003ed9ad21d6a4713f0a9b5a7a0a79e08dd0f221aff4525a2be4c346ee60aab",
+                "sha256:f22ac1c3cac4dbc50079e965eba2c1058622631e526bd9afd45fedd49ba781fa",
+                "sha256:faca3bdcf85b2fc05d06ff3fbc1f83e1391b3e724afa3feba7d13eeab355484c",
+                "sha256:fca0e3a251908a499833aa292323f32437106001d436eca0e6e7833256674585",
+                "sha256:fd1592b3fdf65fff2ad0004b5e363300ef59ced41c2e6b3a99d4089fa8c5435d",
+                "sha256:fd66fc5d0da6d9815ba2cebeb4205f95818ff4b79c3ebe268e75d961704af52f"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==6.0"
+            "version": "==6.0.1"
         },
         "requests": {
             "hashes": [
@@ -386,28 +396,28 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:48e7fafa40319d358848e1bc6809b208340fafe2096f1725d05d67443d0483d1",
-                "sha256:bee28b5e56addb8226c96f7f13ac28cb4c301dd5ea8a6ca179c0b9835e032825"
+                "sha256:13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594",
+                "sha256:ef16afa8ba34a1f989db38e1dbbe0c302e4289a47856990d0682e374563ce35e"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.0.3"
+            "version": "==2.0.5"
         },
         "websocket-client": {
             "hashes": [
-                "sha256:c951af98631d24f8df89ab1019fc365f2227c0892f12fd150e935607c79dd0dd",
-                "sha256:f1f9f2ad5291f0225a49efad77abf9e700b6fef553900623060dad6e26503b9d"
+                "sha256:3aad25d31284266bcfcfd1fd8a743f63282305a364b8d0948a43bd606acc652f",
+                "sha256:6cfc30d051ebabb73a5fa246efdcc14c8fbebbd0330f8984ac3bb6d9edd2ad03"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==1.6.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.6.3"
         }
     },
     "develop": {
         "asttokens": {
             "hashes": [
-                "sha256:4622110b2a6f30b77e1473affaa97e711bc2f07d3f10848420ff1898edbe94f3",
-                "sha256:6b0ac9e93fb0335014d382b8fa9b3afa7df546984258005da0b9e7095b3deb1c"
+                "sha256:2e0171b991b2c959acc6c49318049236844a5da1d65ba2672c4880c1c894834e",
+                "sha256:cf8fc9e61a86461aa9fb161a14a0841a03c405fa829ac6b202670b3495d2ce69"
             ],
-            "version": "==2.2.1"
+            "version": "==2.4.0"
         },
         "backcall": {
             "hashes": [
@@ -474,19 +484,19 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:1d197b907b6ba441b692c48cf2a3a2de280dc0ac91a3405b39349a50272ca0a1",
-                "sha256:248aca623f5c99a6635bc3857677b7320b9b8039f99f070ee0d20a5ca5a8e6bf"
+                "sha256:2baeb5be6949eeebf532150f81746f8333e2ccce02de1c7eedde3f23ed5e9f1e",
+                "sha256:45a2c3a529296870a97b7de34eda4a31bee16bc7bf954e07d39abe49caf8f887"
             ],
             "markers": "python_version >= '3.11'",
-            "version": "==8.14.0"
+            "version": "==8.15.0"
         },
         "jedi": {
             "hashes": [
-                "sha256:203c1fd9d969ab8f2119ec0a3342e0b49910045abe6af0a3ae83a5764d54639e",
-                "sha256:bae794c30d07f6d910d32a7048af09b5a39ed740918da923c6b780790ebac612"
+                "sha256:bcf9894f1753969cbac8022a8c2eaee06bfa3724e4192470aaffe7eb6272b0c4",
+                "sha256:cb8ce23fbccff0025e9386b5cf85e892f94c9b822378f8da49970471335ac64e"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.18.2"
+            "version": "==0.19.0"
         },
         "matplotlib-inline": {
             "hashes": [
@@ -543,11 +553,11 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:8ace4d3c1dd481894b2005f560ead0f9f19ee64fe983366be1a21e171d12775c",
-                "sha256:db2db3deb4b4179f399a09054b023b6a586b76499d36965813c71aa8ed7b5fd1"
+                "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692",
+                "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.15.1"
+            "version": "==2.16.1"
         },
         "six": {
             "hashes": [
@@ -566,11 +576,11 @@
         },
         "traitlets": {
             "hashes": [
-                "sha256:9e6ec080259b9a5940c797d58b613b5e31441c2257b87c2e795c5228ae80d2d8",
-                "sha256:f6cde21a9c68cf756af02035f72d5a723bf607e862e7be33ece505abf4a3bad9"
+                "sha256:417745a96681fbb358e723d5346a547521f36e9bd0d50ba7ab368fff5d67aa54",
+                "sha256:f584ea209240466e66e91f3c81aa7d004ba4cf794990b0c775938a1544217cd1"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==5.9.0"
+            "markers": "python_version >= '3.8'",
+            "version": "==5.10.0"
         },
         "wcwidth": {
             "hashes": [

docker/test/tests/test_capi_whitelists.py

@@ -25,7 +25,7 @@ def test_capi_whitelists(crowdsec, tmp_path_factory, flavor,):
     with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs:
         cs.wait_for_log("*Starting processing data*")
         cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
-        res = cs.cont.exec_run(f'cscli config show-yaml')
+        res = cs.cont.exec_run('cscli config show-yaml')
         assert res.exit_code == 0
         stdout = res.output.decode()
         y = yaml.safe_load(stdout)

docker/test/tests/test_flavors.py

@@ -50,9 +50,11 @@ def test_flavor_content(crowdsec, flavor):
             assert 'notification-http' not in stdout
             assert 'notification-slack' not in stdout
             assert 'notification-splunk' not in stdout
+            assert 'notification-sentinel' not in stdout
         else:
             assert x.exit_code == 0
             assert 'notification-email' in stdout
             assert 'notification-http' in stdout
             assert 'notification-slack' in stdout
             assert 'notification-splunk' in stdout
+            assert 'notification-sentinel' in stdout

docker/test/tests/test_tls.py

@@ -4,7 +4,7 @@
 Test agent-lapi and cscli-lapi communication via TLS, on the same container.
 """
 
-import random
+import uuid
 
 from pytest_cs import Status
 
@@ -140,7 +140,7 @@ def test_tls_lapi_var(crowdsec, flavor, certs_dir):
 def test_tls_split_lapi_agent(crowdsec, flavor, certs_dir):
     """Server-only certificate, split containers"""
 
-    rand = random.randint(0, 10000)
+    rand = uuid.uuid1()
     lapiname = 'lapi-' + str(rand)
     agentname = 'agent-' + str(rand)
 
@@ -193,7 +193,7 @@ def test_tls_split_lapi_agent(crowdsec, flavor, certs_dir):
 def test_tls_mutual_split_lapi_agent(crowdsec, flavor, certs_dir):
     """Server and client certificates, split containers"""
 
-    rand = random.randint(0, 10000)
+    rand = uuid.uuid1()
     lapiname = 'lapi-' + str(rand)
     agentname = 'agent-' + str(rand)
 
@@ -244,7 +244,7 @@ def test_tls_mutual_split_lapi_agent(crowdsec, flavor, certs_dir):
 def test_tls_client_ou(crowdsec, certs_dir):
     """Check behavior of client certificate vs AGENTS_ALLOWED_OU"""
 
-    rand = random.randint(0, 10000)
+    rand = uuid.uuid1()
     lapiname = 'lapi-' + str(rand)
     agentname = 'agent-' + str(rand)
 
@@ -287,6 +287,19 @@ def test_tls_client_ou(crowdsec, certs_dir):
 
     lapi_env['AGENTS_ALLOWED_OU'] = 'custom-client-ou'
 
+    # change container names to avoid conflict
+    # recreate certificates because they need the new hostname
+
+    rand = uuid.uuid1()
+    lapiname = 'lapi-' + str(rand)
+    agentname = 'agent-' + str(rand)
+
+    agent_env['LOCAL_API_URL'] = f'https://{lapiname}:8080'
+
+    volumes = {
+        certs_dir(lapi_hostname=lapiname, agent_ou='custom-client-ou'): {'bind': '/etc/ssl/crowdsec', 'mode': 'ro'},
+    }
+
     cs_lapi = crowdsec(name=lapiname, environment=lapi_env, volumes=volumes)
     cs_agent = crowdsec(name=agentname, environment=agent_env, volumes=volumes)
 

go.mod

@@ -1,9 +1,13 @@
 module github.com/crowdsecurity/crowdsec
 
-go 1.20
+go 1.21
+
+// Don't use the toolchain directive to avoid uncontrolled downloads during
+// a build, especially in sandboxed environments (freebsd, gentoo...).
+// toolchain go1.21.1
 
 require (
-	entgo.io/ent v0.11.3
+	entgo.io/ent v0.12.4
 	github.com/AlecAivazis/survey/v2 v2.2.7
 	github.com/Masterminds/semver/v3 v3.1.1
 	github.com/Masterminds/sprig/v3 v3.2.2
@@ -21,7 +25,7 @@ require (
 	github.com/c-robinson/iplib v1.0.3
 	github.com/cespare/xxhash/v2 v2.2.0
 	github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
-	github.com/crowdsecurity/go-cs-lib v0.0.2
+	github.com/crowdsecurity/go-cs-lib v0.0.4
 	github.com/crowdsecurity/grokky v0.2.1
 	github.com/crowdsecurity/machineid v1.0.2
 	github.com/davecgh/go-spew v1.1.1
@@ -43,7 +47,7 @@ require (
 	github.com/golang-jwt/jwt/v4 v4.4.2
 	github.com/google/go-querystring v1.0.0
 	github.com/google/uuid v1.3.0
-	github.com/google/winops v0.0.0-20211216095627-f0e86eb1453b
+	github.com/google/winops v0.0.0-20230712152054-af9b550d0601
 	github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e
 	github.com/hashicorp/go-hclog v1.5.0
 	github.com/hashicorp/go-plugin v1.4.10
@@ -68,12 +72,13 @@ require (
 	github.com/segmentio/kafka-go v0.4.34
 	github.com/shirou/gopsutil/v3 v3.23.5
 	github.com/sirupsen/logrus v1.9.3
+	github.com/slack-go/slack v0.12.2
 	github.com/spf13/cobra v1.7.0
-	github.com/stretchr/testify v1.8.3
+	github.com/stretchr/testify v1.8.4
 	github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
 	github.com/wasilibs/go-re2 v1.3.0
+	github.com/xhit/go-simple-mail/v2 v2.16.0
 	golang.org/x/crypto v0.9.0
-	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
 	golang.org/x/mod v0.11.0
 	golang.org/x/sys v0.9.0
 	google.golang.org/grpc v1.56.1
@@ -86,7 +91,7 @@ require (
 )
 
 require (
-	ariga.io/atlas v0.7.2-0.20220927111110-867ee0cca56a // indirect
+	ariga.io/atlas v0.14.1-0.20230918065911-83ad451a4935 // indirect
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Microsoft/go-winio v0.6.1 // indirect
 	github.com/ahmetalpbalkan/dlog v0.0.0-20170105205344-4fb5f8204f26 // indirect
@@ -121,6 +126,7 @@ require (
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
+	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/hashicorp/hcl/v2 v2.13.0 // indirect
 	github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect
 	github.com/huandu/xstrings v1.3.2 // indirect
@@ -176,6 +182,7 @@ require (
 	github.com/tidwall/gjson v1.13.0 // indirect
 	github.com/tklauser/go-sysconf v0.3.11 // indirect
 	github.com/tklauser/numcpus v0.6.0 // indirect
+	github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
 	github.com/ugorji/go/codec v1.2.11 // indirect
 	github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
@@ -187,7 +194,8 @@ require (
 	golang.org/x/sync v0.1.0 // indirect
 	golang.org/x/term v0.8.0 // indirect
 	golang.org/x/text v0.9.0 // indirect
-	golang.org/x/tools v0.7.0 // indirect
+	golang.org/x/time v0.2.0 // indirect
+	golang.org/x/tools v0.8.1-0.20230428195545-5283a0178901 // indirect
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
 	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect

go.sum

@@ -1,5 +1,5 @@
-ariga.io/atlas v0.7.2-0.20220927111110-867ee0cca56a h1:6/nt4DODfgxzHTTg3tYy7YkVzruGQGZ/kRvXpA45KUo=
-ariga.io/atlas v0.7.2-0.20220927111110-867ee0cca56a/go.mod h1:ft47uSh5hWGDCmQC9DsztZg6Xk+KagM5Ts/mZYKb9JE=
+ariga.io/atlas v0.14.1-0.20230918065911-83ad451a4935 h1:JnYs/y8RJ3+MiIUp+3RgyyeO48VHLAZimqiaZYnMKk8=
+ariga.io/atlas v0.14.1-0.20230918065911-83ad451a4935/go.mod h1:isZrlzJ5cpoCoKFoY9knZug7Lq4pP1cm8g3XciLZ0Pw=
 bitbucket.org/creachadair/stringset v0.0.9 h1:L4vld9nzPt90UZNrXjNelTshD74ps4P5NGs3Iq6yN3o=
 bitbucket.org/creachadair/stringset v0.0.9/go.mod h1:t+4WcQ4+PXTa8aQdNKe40ZP6iwesoMFWAxPGd3UGjyY=
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
@@ -35,14 +35,16 @@ cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohl
 cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs=
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-entgo.io/ent v0.11.3 h1:F5FBGAWiDCGder7YT+lqMnyzXl6d0xU3xMBM/SO3CMc=
-entgo.io/ent v0.11.3/go.mod h1:mvDhvynOzAsOe7anH7ynPPtMjA/eeXP96kAfweevyxc=
+entgo.io/ent v0.12.4 h1:LddPnAyxls/O7DTXZvUGDj0NZIdGSu317+aoNLJWbD8=
+entgo.io/ent v0.12.4/go.mod h1:Y3JVAjtlIk8xVZYSn3t3mf8xlZIn5SAOXZQxD6kKI+Q=
 github.com/AlecAivazis/survey/v2 v2.2.7 h1:5NbxkF4RSKmpywYdcRgUmos1o+roJY8duCLZXbVjoig=
 github.com/AlecAivazis/survey/v2 v2.2.7/go.mod h1:9DYvHgXtiXm6nCn+jXnOXLKbH+Yo9u8fAS/SduGdoPk=
 github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8=
+github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60=
+github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM=
 github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI=
 github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU=
 github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
@@ -129,15 +131,14 @@ github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSV
 github.com/cpuguy83/go-md2man/v2 v2.0.1/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w=
 github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
-github.com/creachadair/staticfile v0.1.3/go.mod h1:a3qySzCIXEprDGxk6tSxSI+dBBdLzqeBOMhZ+o2d3pM=
 github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY=
 github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
 github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
-github.com/crowdsecurity/go-cs-lib v0.0.2 h1:+Tjmf/IclOXNzU9sxKVQvUl9CkMfbM60xQ0zA05NWps=
-github.com/crowdsecurity/go-cs-lib v0.0.2/go.mod h1:iznTJ19qLTYdZBcRb5RVDlcUdSlayBCivBkWsXlOY3g=
+github.com/crowdsecurity/go-cs-lib v0.0.4 h1:mH3iqz8H8iH9YpldqCdojyKHy9z3JDhas/k6I8M0ims=
+github.com/crowdsecurity/go-cs-lib v0.0.4/go.mod h1:8FMKNGsh3hMZi2SEv6P15PURhEJnZV431XjzzBSuf0k=
 github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
 github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
 github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=
@@ -285,6 +286,7 @@ github.com/go-openapi/validate v0.20.0 h1:pzutNCCBZGZlE+u8HD3JZyWdc/TVbtVwlWUp8/
 github.com/go-openapi/validate v0.20.0/go.mod h1:b60iJT+xNNLfaQJUqLI7946tYiFEOuE9E4k54HpKcJ0=
 github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
+github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4=
 github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8=
 github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
 github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
@@ -299,7 +301,8 @@ github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfC
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
+github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho=
+github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA=
 github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0=
 github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY=
 github.com/gobuffalo/depgen v0.1.0/go.mod h1:+ifsuy7fhi15RWncXQQKjWS9JPkdah5sZvtHc2RXGlg=
@@ -328,7 +331,6 @@ github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
 github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/goccy/go-yaml v1.11.0 h1:n7Z+zx8S9f9KgzG6KtQKf+kwqXZlLNR2F6018Dgau54=
 github.com/goccy/go-yaml v1.11.0/go.mod h1:H+mJrWtjPTJAHvRbV09MCK9xYwODM+wRTVFFTWckfng=
-github.com/godbus/dbus v4.1.0+incompatible/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw=
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
@@ -339,7 +341,6 @@ github.com/golang-jwt/jwt/v4 v4.2.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw
 github.com/golang-jwt/jwt/v4 v4.4.2 h1:rcc4lwaZgFMCZ5jxF9ABolDcIHdBytAFgqFPbSJQAYs=
 github.com/golang-jwt/jwt/v4 v4.4.2/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
-github.com/golang/glog v0.0.0-20210429001901-424d2337a529/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE=
 github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -384,6 +385,7 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
@@ -391,7 +393,6 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
 github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
-github.com/google/logger v1.1.1/go.mod h1:BkeJZ+1FhQ+/d087r4dzojEg1u2ZX+ZqG1jTUrLM+zQ=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
@@ -406,13 +407,15 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
 github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/google/winops v0.0.0-20211216095627-f0e86eb1453b h1:THwEE9J2wPxF3BZm7WjLCASMcM7ctFzqLpTsCGh7gDY=
-github.com/google/winops v0.0.0-20211216095627-f0e86eb1453b/go.mod h1:ShbX8v8clPm/3chw9zHVwtW3QhrFpL8mXOwNxClt4pg=
+github.com/google/winops v0.0.0-20230712152054-af9b550d0601 h1:XvlrmqZIuwxuRE88S9mkxX+FkV+YakqbiAC5Z4OzDnM=
+github.com/google/winops v0.0.0-20230712152054-af9b550d0601/go.mod h1:rT1mcjzuvcDDbRmUTsoH6kV0DG91AkFe9UCjASraK5I=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
 github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
 github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e h1:XmA6L9IPRdUr28a+SK/oMchGgQy159wvzXA5tJ7l+40=
 github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e/go.mod h1:AFIo+02s+12CEg8Gzz9kzhCbmbq6JcKNrhHffCGA9z4=
-github.com/groob/plist v0.0.0-20210519001750-9f754062e6d6/go.mod h1:itkABA+w2cw7x5nYUS/pLRef6ludkZKOigbROmCTaFw=
+github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c=
 github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
 github.com/hashicorp/go-plugin v1.4.10 h1:xUbmA4jC6Dq163/fWcp8P3JuHilrHHMLNRxzGQJ9hNk=
@@ -489,6 +492,7 @@ github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dv
 github.com/jarcoal/httpmock v1.1.0 h1:F47ChZj1Y2zFsCXxNkBPwNNKnAyOATcdQibk0qEdVCE=
 github.com/jarcoal/httpmock v1.1.0/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik=
 github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE=
+github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74=
 github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg=
 github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo=
 github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8=
@@ -530,6 +534,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
+github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.4/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
 github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA=
@@ -539,14 +544,15 @@ github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348 h1:MtvEpTB6LX3vkb4ax0b5D2DHbNAUsen0Gx5wZoq3lV4=
+github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k=
 github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII=
 github.com/leodido/go-urn v1.2.4 h1:XlAE/cm/ms7TE/VMVoduSpNBoyc2dOxHs5MZSwAN63Q=
 github.com/leodido/go-urn v1.2.4/go.mod h1:7ZrI8mTSeBSHl/UaRyKQW1qZeMgak41ANeCNaVckg+4=
 github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
+github.com/lib/pq v1.10.2 h1:AqzbZs4ZoCBp+GtejcpCpcxM3zlSMx29dXbUSeVtJb8=
 github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
-github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw=
 github.com/lithammer/dedent v1.1.0 h1:VNzHMVCBNG1j0fh3OrsFRkVUwStdDArbgBWoPAffktY=
 github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z9BP0jIOc=
 github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
@@ -692,6 +698,7 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
 github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
+github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
 github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
 github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU=
 github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc=
@@ -718,6 +725,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd
 github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88=
 github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ=
 github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
+github.com/slack-go/slack v0.12.2 h1:x3OppyMyGIbbiyFhsBmpf9pwkUzMhthJMRNmNlA4LaQ=
+github.com/slack-go/slack v0.12.2/go.mod h1:hlGi5oXA+Gt+yWTPP0plCdRKmjsDxecdHxYQdlMQKOw=
 github.com/spf13/cast v1.3.1 h1:nFm6S0SMdyzrzcmThSipiEubIDy8WEXKNZ0UOgiRpng=
 github.com/spf13/cast v1.3.1/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
 github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
@@ -745,8 +754,9 @@ github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1F
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY=
 github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
+github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/tetratelabs/wazero v1.2.1 h1:J4X2hrGzJvt+wqltuvcSjHQ7ujQxA9gb6PeMs4qlUWs=
 github.com/tetratelabs/wazero v1.2.1/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ=
 github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk=
@@ -761,6 +771,8 @@ github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+Kd
 github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI=
 github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms=
 github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ5UVIcaL4=
+github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208 h1:PM5hJF7HVfNWmCjMdEfbuOBNXSVF2cMFGgQTPdKCbwM=
+github.com/toorop/go-dkim v0.0.0-20201103131630-e1cd1a0a5208/go.mod h1:BzWtXXrXzZUvMacR0oF/fbDDgUPO8L36tDMmRAf14ns=
 github.com/twitchyliquid64/golang-asm v0.15.1 h1:SU5vSMR7hnwNxj24w34ZyCi/FmDZTkS4MhqMhdFk5YI=
 github.com/twitchyliquid64/golang-asm v0.15.1/go.mod h1:a1lVb/DtPvCB8fslRZhAngC2+aY1QWCk3Cedj/Gdt08=
 github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw=
@@ -771,6 +783,7 @@ github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 h1:UFHFmFfixpmf
 github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26/go.mod h1:IGhd0qMDsUa9acVjsbsT7bu3ktadtGOHI79+idTew/M=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
 github.com/vjeantet/grok v1.0.1 h1:2rhIR7J4gThTgcZ1m2JY4TrJZNgjn985U28kT2wQrJ4=
+github.com/vjeantet/grok v1.0.1/go.mod h1:ax1aAchzC6/QMXMcyzHQGZWaW1l195+uMYIkCWPCNIo=
 github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaUXK79GlxNBwueZn0xI=
 github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
 github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
@@ -778,6 +791,7 @@ github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgq
 github.com/wasilibs/go-re2 v1.3.0 h1:LFhBNzoStM3wMie6rN2slD1cuYH2CGiHpvNL3UtcsMw=
 github.com/wasilibs/go-re2 v1.3.0/go.mod h1:AafrCXVvGRJJOImMajgJ2M7rVmWyisVK7sFshbxnVrg=
 github.com/wasilibs/nottinygc v0.4.0 h1:h1TJMihMC4neN6Zq+WKpLxgd9xCFMw7O9ETLwY2exJQ=
+github.com/wasilibs/nottinygc v0.4.0/go.mod h1:oDcIotskuYNMpqMF23l7Z8uzD4TC0WXHK8jetlB3HIo=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
 github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
@@ -787,6 +801,8 @@ github.com/xdg/scram v1.0.5/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49
 github.com/xdg/stringprep v0.0.0-20180714160509-73f8eece6fdc/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
 github.com/xdg/stringprep v1.0.3 h1:cmL5Enob4W83ti/ZHuZLuKD/xqJfus4fVPwE+/BDm+4=
 github.com/xdg/stringprep v1.0.3/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
+github.com/xhit/go-simple-mail/v2 v2.16.0 h1:ouGy/Ww4kuaqu2E2UrDw7SvLaziWTB60ICLkIkNVccA=
+github.com/xhit/go-simple-mail/v2 v2.16.0/go.mod h1:b7P5ygho6SYE+VIqpxA6QkYfv4teeyG4MKqB3utRu98=
 github.com/youmark/pkcs8 v0.0.0-20181117223130-1be2e3e5546d/go.mod h1:rHwXgn7JulP+udvsHwJoVG1YGAP6VLg4y9I5dyZdqmA=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
@@ -855,8 +871,6 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
-golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
-golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
@@ -994,8 +1008,6 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210601080250-7ecdf8ef093b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -1032,7 +1044,8 @@ golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
-golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 h1:vVKdlvoWBphwdxWKrFZEuM0kGgGLxUOYcY4U/2Vjg44=
+golang.org/x/time v0.2.0 h1:52I/1L54xyEQAYdtcSuxtiT84KGYTBGXwayxmIpNJhE=
+golang.org/x/time v0.2.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -1088,8 +1101,8 @@ golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc
 golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA=
 golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
-golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4=
-golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s=
+golang.org/x/tools v0.8.1-0.20230428195545-5283a0178901 h1:0wxTF6pSjIIhNt7mo9GvjDfzyCOiWhmICgtO/Ah948s=
+golang.org/x/tools v0.8.1-0.20230428195545-5283a0178901/go.mod h1:JxBZ99ISMI5ViVkT1tr6tdNmXeTrcpVSD3vZ1RsRdN4=
 golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
@@ -1238,3 +1251,4 @@ sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h6
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3 h1:PRbqxJClWWYMNV1dhaG4NsibJbArud9kFxnAMREiWFE=
 sigs.k8s.io/structured-merge-diff/v4 v4.2.3/go.mod h1:qjx8mGObPmV2aSZepjQjbmb2ihdVs8cGKBraizNC69E=
 sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo=
+sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8=

pkg/acquisition/acquisition.go

@@ -15,7 +15,7 @@ import (
 	tomb "gopkg.in/tomb.v2"
 	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
+	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
 	cloudwatchacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/cloudwatch"

pkg/acquisition/acquisition_test.go

@@ -13,7 +13,7 @@ import (
 	tomb "gopkg.in/tomb.v2"
 	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/cstest"
+	"github.com/crowdsecurity/go-cs-lib/cstest"
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"

pkg/acquisition/modules/cloudwatch/cloudwatch.go

@@ -370,7 +370,7 @@ func (cw *CloudwatchSource) LogStreamManager(in chan LogStreamTailConfig, outCha
 			}
 
 			if cw.Config.StreamRegexp != nil {
-				match, err := regexp.Match(*cw.Config.StreamRegexp, []byte(newStream.StreamName))
+				match, err := regexp.MatchString(*cw.Config.StreamRegexp, newStream.StreamName)
 				if err != nil {
 					cw.logger.Warningf("invalid regexp : %s", err)
 				} else if !match {

pkg/acquisition/modules/cloudwatch/cloudwatch_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/cstest"
+	"github.com/crowdsecurity/go-cs-lib/cstest"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"

pkg/acquisition/modules/docker/docker.go

@@ -392,14 +392,14 @@ func (d *DockerSource) EvalContainer(container dockerTypes.Container) (*Containe
 	}
 
 	for _, cont := range d.compiledContainerID {
-		if matched := cont.Match([]byte(container.ID)); matched {
+		if matched := cont.MatchString(container.ID); matched {
 			return &ContainerConfig{ID: container.ID, Name: container.Names[0], Labels: d.Config.Labels, Tty: d.getContainerTTY(container.ID)}, true
 		}
 	}
 
 	for _, cont := range d.compiledContainerName {
 		for _, name := range container.Names {
-			if matched := cont.Match([]byte(name)); matched {
+			if matched := cont.MatchString(name); matched {
 				return &ContainerConfig{ID: container.ID, Name: name, Labels: d.Config.Labels, Tty: d.getContainerTTY(container.ID)}, true
 			}
 		}

pkg/acquisition/modules/docker/docker_test.go

@@ -11,7 +11,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/cstest"
+	"github.com/crowdsecurity/go-cs-lib/cstest"
 
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 	dockerTypes "github.com/docker/docker/api/types"
@@ -193,7 +193,7 @@ container_name_regexp:
 					actualLines++
 					ticker.Reset(1 * time.Second)
 				case <-ticker.C:
-					log.Infof("no more line to read")
+					log.Infof("no more lines to read")
 					dockerSource.t.Kill(nil)
 					return nil
 				}

pkg/acquisition/modules/file/file.go

@@ -21,7 +21,7 @@ import (
 	"gopkg.in/tomb.v2"
 	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
+	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
 	"github.com/crowdsecurity/crowdsec/pkg/types"

pkg/acquisition/modules/file/file_test.go

@@ -13,7 +13,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"gopkg.in/tomb.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/cstest"
+	"github.com/crowdsecurity/go-cs-lib/cstest"
 
 	fileacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/file"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
@@ -410,9 +410,7 @@ force_inotify: true`, testPattern),
 
 			if tc.expectedLines != 0 {
 				fd, err := os.Create("test_files/stream.log")
-				if err != nil {
-					t.Fatalf("could not create test file : %s", err)
-				}
+				require.NoError(t, err, "could not create test file")
 
 				for i := 0; i < 5; i++ {
 					_, err = fmt.Fprintf(fd, "%d\n", i)
@@ -424,7 +422,7 @@ force_inotify: true`, testPattern),
 
 				fd.Close()
 				// we sleep to make sure we detect the new file
-				time.Sleep(1 * time.Second)
+				time.Sleep(3 * time.Second)
 				os.Remove("test_files/stream.log")
 				assert.Equal(t, tc.expectedLines, actualLines)
 			}

pkg/acquisition/modules/journalctl/journalctl.go

@@ -14,7 +14,7 @@ import (
 	"gopkg.in/tomb.v2"
 	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
+	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
 	"github.com/crowdsecurity/crowdsec/pkg/types"

pkg/acquisition/modules/journalctl/journalctl_test.go

@@ -8,7 +8,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/cstest"
+	"github.com/crowdsecurity/go-cs-lib/cstest"
 
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 	log "github.com/sirupsen/logrus"

pkg/acquisition/modules/kafka/kafka.go

@@ -16,7 +16,7 @@ import (
 	"gopkg.in/tomb.v2"
 	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
+	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
@@ -149,7 +149,9 @@ func (k *KafkaSource) ReadMessage(out chan types.Event) error {
 				return nil
 			}
 			k.logger.Errorln(fmt.Errorf("while reading %s message: %w", dataSourceName, err))
+			continue
 		}
+		k.logger.Tracef("got message: %s", string(m.Value))
 		l := types.Line{
 			Raw:     string(m.Value),
 			Labels:  k.Config.Labels,
@@ -223,7 +225,6 @@ func (kc *KafkaConfiguration) NewTLSConfig() (*tls.Config, error) {
 	caCertPool.AppendCertsFromPEM(caCert)
 	tlsConfig.RootCAs = caCertPool
 
-	tlsConfig.BuildNameToCertificate()
 	return &tlsConfig, err
 }
 

pkg/acquisition/modules/kafka/kafka_test.go

@@ -13,7 +13,7 @@ import (
 	"github.com/stretchr/testify/require"
 	"gopkg.in/tomb.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/cstest"
+	"github.com/crowdsecurity/go-cs-lib/cstest"
 
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )

pkg/acquisition/modules/kinesis/kinesis.go

@@ -18,7 +18,7 @@ import (
 	"gopkg.in/tomb.v2"
 	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
+	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
 	"github.com/crowdsecurity/crowdsec/pkg/types"

pkg/acquisition/modules/kinesis/kinesis_test.go

@@ -12,7 +12,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/cstest"
+	"github.com/crowdsecurity/go-cs-lib/cstest"
 
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/session"

pkg/acquisition/modules/kubernetesaudit/k8s_audit.go

@@ -14,7 +14,7 @@ import (
 	"gopkg.in/yaml.v2"
 	"k8s.io/apiserver/pkg/apis/audit"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
+	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
 	"github.com/crowdsecurity/crowdsec/pkg/types"

pkg/acquisition/modules/syslog/internal/parser/rfc5424/parse_test.go

@@ -4,7 +4,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/cstest"
+	"github.com/crowdsecurity/go-cs-lib/cstest"
 
 	"github.com/stretchr/testify/require"
 )

pkg/acquisition/modules/syslog/syslog.go

@@ -11,7 +11,7 @@ import (
 	"gopkg.in/tomb.v2"
 	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
+	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog/internal/parser/rfc3164"

pkg/acquisition/modules/syslog/syslog_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 	"time"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/cstest"
+	"github.com/crowdsecurity/go-cs-lib/cstest"
 
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 	log "github.com/sirupsen/logrus"

pkg/acquisition/modules/wineventlog/wineventlog_windows.go

@@ -17,7 +17,7 @@ import (
 	"gopkg.in/tomb.v2"
 	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
+	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
 	"github.com/crowdsecurity/crowdsec/pkg/types"