Merge tag 'tags/v1.6.0' into internal-release

.github/governance.yml

@@ -81,7 +81,7 @@ pull_request:
             failure: Missing kind label to generate release automatically.
 
     - prefix: area
-      list: [ "agent", "local-api", "cscli", "security", "configuration"]
+      list: [ "agent", "local-api", "cscli", "security", "configuration", "appsec"]
       multiple: true
       needs:
         comment: |
@@ -89,6 +89,7 @@ pull_request:
           * `/area agent`
           * `/area local-api`
           * `/area cscli`
+          * `/area appsec`
           * `/area security`
           * `/area configuration`
 
@@ -98,4 +99,4 @@ pull_request:
       author_association:
         collaborator: true
         member: true
-        owner: true
+        owner: true

.github/workflows/bats-hub.yml

@@ -15,7 +15,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.21.3"]
+        test-file: ["hub-1.bats", "hub-2.bats", "hub-3.bats"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest
@@ -33,22 +33,24 @@ jobs:
         fetch-depth: 0
         submodules: true
 
-    - name: "Set up Go ${{ matrix.go-version }}"
+    - name: "Set up Go"
       uses: actions/setup-go@v4
       with:
-        go-version: ${{ matrix.go-version }}
+        go-version: "1.21.6"
 
     - name: "Install bats dependencies"
       env:
         GOBIN: /usr/local/bin
       run: |
-        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
+        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
 
     - name: "Build crowdsec and fixture"
       run: make bats-clean bats-build bats-fixture BUILD_STATIC=1
 
     - name: "Run hub tests"
-      run: make bats-test-hub
+      run: |
+          ./test/bin/generate-hub-tests
+          ./test/run-tests test/dyn-bats/${{ matrix.test-file }}
 
     - name: "Collect hub coverage"
       run: ./test/bin/collect-hub-coverage >> $GITHUB_ENV

.github/workflows/bats-mysql.yml

@@ -14,7 +14,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.21.3"]
+        go-version: ["1.21.6"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest
@@ -49,7 +49,7 @@ jobs:
       env:
         GOBIN: /usr/local/bin
       run: |
-        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
+        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
 
     - name: "Build crowdsec and fixture"
       run: |

.github/workflows/bats-postgres.yml

@@ -10,7 +10,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.21.3"]
+        go-version: ["1.21.6"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest
@@ -58,7 +58,7 @@ jobs:
       env:
         GOBIN: /usr/local/bin
       run: |
-        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
+        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
 
     - name: "Build crowdsec and fixture (DB_BACKEND: pgx)"
       run: |

.github/workflows/bats-sqlite-coverage.yml

@@ -11,7 +11,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.21.3"]
+        go-version: ["1.21.6"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest
@@ -39,7 +39,7 @@ jobs:
       env:
         GOBIN: /usr/local/bin
       run: |
-        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
+        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
 
     - name: "Build crowdsec and fixture"
       run: |

.github/workflows/bats.yml

@@ -31,7 +31,7 @@ jobs:
 
   # Jobs for Postgres (and sometimes MySQL) can have failing tests on GitHub
   # CI, but they pass when run on devs' machines or in the release checks. We
-  # disable them here by default. Remove the if..false to enable them.
+  # disable them here by default. Remove if...false to enable them.
 
   mariadb:
     uses: ./.github/workflows/bats-mysql.yml

.github/workflows/ci-windows-build-msi.yml

@@ -23,7 +23,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.21.3"]
+        go-version: ["1.21.6"]
 
     name: Build
     runs-on: windows-2019

.github/workflows/codeql-analysis.yml

@@ -74,7 +74,7 @@ jobs:
     - name: "Set up Go"
       uses: actions/setup-go@v4
       with:
-        go-version: "1.21.0"
+        go-version: "1.21.6"
         cache-dependency-path: "**/go.sum"
 
     - run: |

.github/workflows/go-tests-windows.yml

@@ -22,7 +22,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.21.3"]
+        go-version: ["1.21.6"]
 
     name: "Build + tests"
     runs-on: windows-2022
@@ -60,7 +60,7 @@ jobs:
     - name: golangci-lint
       uses: golangci/golangci-lint-action@v3
       with:
-        version: v1.54
+        version: v1.55
         args: --issues-exit-code=1 --timeout 10m
         only-new-issues: false
         # the cache is already managed above, enabling it here

.github/workflows/go-tests.yml

@@ -24,23 +24,18 @@ env:
   RICHGO_FORCE_COLOR: 1
   AWS_HOST: localstack
   # these are to mimic aws config
-  AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
-  AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+  AWS_ACCESS_KEY_ID: test
+  AWS_SECRET_ACCESS_KEY: test
   AWS_REGION: us-east-1
-  KINESIS_INITIALIZE_STREAMS: "stream-1-shard:1,stream-2-shards:2"
   CROWDSEC_FEATURE_DISABLE_HTTP_RETRY_BACKOFF: true
 
 jobs:
   build:
-    strategy:
-      matrix:
-        go-version: ["1.21.3"]
-
     name: "Build + tests"
     runs-on: ubuntu-latest
     services:
       localstack:
-        image: localstack/localstack:1.3.0
+        image: localstack/localstack:3.0
         ports:
         - 4566:4566  # Localstack exposes all services on the same port
         env:
@@ -49,7 +44,7 @@ jobs:
           KINESIS_ERROR_PROBABILITY: ""
           DOCKER_HOST: unix:///var/run/docker.sock
           KINESIS_INITIALIZE_STREAMS: ${{ env.KINESIS_INITIALIZE_STREAMS }}
-          HOSTNAME_EXTERNAL: ${{ env.AWS_HOST }}  # Required so that resource urls are provided properly
+          LOCALSTACK_HOST: ${{ env.AWS_HOST }}  # Required so that resource urls are provided properly
           # e.g sqs url will get localhost if we don't set this env to map our service
         options: >-
           --name=localstack
@@ -58,7 +53,7 @@ jobs:
           --health-timeout=5s
           --health-retries=3
       zoo1:
-        image: confluentinc/cp-zookeeper:7.3.0
+        image: confluentinc/cp-zookeeper:7.4.3
         ports:
           - "2181:2181"
         env:
@@ -108,6 +103,18 @@ jobs:
           --health-timeout 10s
           --health-retries 5
 
+      loki:
+        image: grafana/loki:2.9.1
+        ports:
+          - "3100:3100"
+        options: >-
+          --name=loki1
+          --health-cmd "wget -q -O - http://localhost:3100/ready | grep 'ready'"
+          --health-interval 30s
+          --health-timeout 10s
+          --health-retries 5
+          --health-start-period 30s
+
     steps:
 
     - name: Check out CrowdSec repository
@@ -116,10 +123,15 @@ jobs:
         fetch-depth: 0
         submodules: false
 
-    - name: "Set up Go ${{ matrix.go-version }}"
+    - name: "Set up Go"
       uses: actions/setup-go@v4
       with:
-        go-version: ${{ matrix.go-version }}
+        go-version: "1.21.6"
+
+    - name: Create localstack streams
+      run: |
+          aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-1-shard --shard-count 1
+          aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-2-shards --shard-count 2
 
     - name: Build and run tests, static
       run: |
@@ -128,12 +140,13 @@ jobs:
         go install github.com/kyoh86/richgo@v0.3.10
         set -o pipefail
         make build BUILD_STATIC=1
-        make go-acc | richgo testfilter
+        make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
 
     - name: Run tests again, dynamic
       run: |
         make clean build
-        make go-acc | richgo testfilter
+        set -o pipefail
+        make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
 
     - name: Upload unit coverage to Codecov
       uses: codecov/codecov-action@v3
@@ -144,7 +157,7 @@ jobs:
     - name: golangci-lint
       uses: golangci/golangci-lint-action@v3
       with:
-        version: v1.54
+        version: v1.55
         args: --issues-exit-code=1 --timeout 10m
         only-new-issues: false
         # the cache is already managed above, enabling it here

.github/workflows/publish_docker-image_on_master-debian.yml

@@ -19,6 +19,7 @@ jobs:
   push_to_registry:
     name: Push Debian Docker image to Docker Hub
     runs-on: ubuntu-latest
+    if: ${{ github.repository_owner == 'crowdsecurity' }}
     steps:
 
       - name: Check out the repo

.github/workflows/publish_docker-image_on_master.yml

@@ -19,6 +19,7 @@ jobs:
   push_to_registry:
     name: Push Docker image to Docker Hub
     runs-on: ubuntu-latest
+    if: ${{ github.repository_owner == 'crowdsecurity' }}
     steps:
 
       - name: Check out the repo

.github/workflows/release_publish-package.yml

@@ -14,7 +14,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.21.3"]
+        go-version: ["1.21.6"]
 
     name: Build and upload binary package
     runs-on: ubuntu-latest
@@ -41,4 +41,4 @@ jobs:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
         run: |
           tag_name="${GITHUB_REF##*/}"
-          hub release edit -a crowdsec-release.tgz -a vendor.tgz -a *-vendor.tar.xz -m "" "$tag_name"
+          gh release upload "$tag_name" crowdsec-release.tgz vendor.tgz *-vendor.tar.xz

.github/workflows/release_publish_docker-image.yml

@@ -29,10 +29,10 @@ jobs:
             VERSION=pr-${{ github.event.number }}
           fi
           TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}"
-          TAGS_SLIM="${DOCKER_IMAGE}:${VERSION}-slim"
+          TAGS_SLIM="${DOCKER_IMAGE}:${VERSION}-slim,${GHCR_IMAGE}:${VERSION}-slim"
           if [[ ${{ github.event.action }} == released ]]; then
             TAGS=$TAGS,${DOCKER_IMAGE}:latest,${GHCR_IMAGE}:latest
-            TAGS_SLIM=$TAGS_SLIM,${DOCKER_IMAGE}:slim
+            TAGS_SLIM=$TAGS_SLIM,${DOCKER_IMAGE}:slim,${GHCR_IMAGE}:slim
           fi
           echo "version=${VERSION}" >> $GITHUB_OUTPUT
           echo "tags=${TAGS}" >> $GITHUB_OUTPUT

.golangci.yml

@@ -9,8 +9,24 @@ run:
     - pkg/yamlpatch/merge_test.go
 
 linters-settings:
+  cyclop:
+    # lower this after refactoring
+    max-complexity: 66
+
+  gci:
+    sections:
+     - standard
+     - default
+     - prefix(github.com/crowdsecurity)
+     - prefix(github.com/crowdsecurity/crowdsec)
+
+  gocognit:
+    # lower this after refactoring
+    min-complexity: 145
+
   gocyclo:
-    min-complexity: 30
+    # lower this after refactoring
+    min-complexity: 64
 
   funlen:
     # Checks the number of lines in a function.
@@ -28,11 +44,21 @@ linters-settings:
   lll:
     line-length: 140
 
+  maintidx:
+    # raise this after refactoring
+    under: 9
+
   misspell:
     locale: US
 
+  nestif:
+    # lower this after refactoring
+    min-complexity: 27
+
+  nlreturn:
+    block-size: 4
+
   nolintlint:
-    allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
     allow-unused: false # report any unused nolint directives
     require-explanation: false # don't require an explanation for nolint directives
     require-specific: false # don't require nolint directives to be specific about which linter is being skipped
@@ -40,6 +66,13 @@ linters-settings:
   interfacebloat:
     max: 12
 
+  depguard:
+    rules:
+      main:
+        deny:
+          - pkg: "github.com/pkg/errors"
+            desc: "errors.Wrap() is deprecated in favor of fmt.Errorf()"
+
 linters:
   enable-all: true
   disable:
@@ -64,15 +97,21 @@ linters:
     # - asasalint           # check for pass []any as any in variadic func(...any)
     # - asciicheck          # Simple linter to check that your code does not contain non-ASCII identifiers
     # - bidichk             # Checks for dangerous unicode character sequences
+    # - bodyclose           # checks whether HTTP response body is closed successfully
+    # - cyclop              # checks function and package cyclomatic complexity
     # - decorder            # check declaration order and count of types, constants, variables and functions
+    # - depguard            # Go linter that checks if package imports are in a list of acceptable packages
     # - dupword             # checks for duplicate words in the source code
     # - durationcheck       # check for two durations multiplied together
     # - errcheck            # Errcheck is a program for checking for unchecked errors in go programs. These unchecked errors can be critical bugs in some cases
+    # - errorlint           # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13.
     # - exportloopref       # checks for pointers to enclosing loop variables
     # - funlen              # Tool for detection of long functions
     # - ginkgolinter        # enforces standards of using ginkgo and gomega
     # - gochecknoinits      # Checks that no init functions are present in Go code
+    # - gocognit            # Computes and checks the cognitive complexity of functions
     # - gocritic            # Provides diagnostics that check for bugs, performance and style issues.
+    # - gocyclo             # Computes and checks the cyclomatic complexity of functions
     # - goheader            # Checks is file header matches to pattern
     # - gomoddirectives     # Manage the use of 'replace', 'retract', and 'excludes' directives in go.mod.
     # - gomodguard          # Allow and block list linter for direct Go module dependencies. This is different from depguard where there are different block types for example version constraints and module recommendations.
@@ -84,10 +123,15 @@ linters:
     # - ineffassign         # Detects when assignments to existing variables are not used
     # - interfacebloat      # A linter that checks the number of methods inside an interface.
     # - logrlint            # Check logr arguments.
+    # - maintidx            # maintidx measures the maintainability index of each function.
     # - makezero            # Finds slice declarations with non-zero initial length
     # - misspell            # Finds commonly misspelled English words in comments
+    # - nakedret            # Finds naked returns in functions greater than a specified function length
+    # - nestif              # Reports deeply nested if statements
     # - nilerr              # Finds the code that returns nil even if it checks that the error is not nil.
     # - nolintlint          # Reports ill-formed or insufficient nolint directives
+    # - nonamedreturns      # Reports all named returns
+    # - nosprintfhostport   # Checks for misuse of Sprintf to construct a host with port in a URL.
     # - predeclared         # find code that shadows one of Go's predeclared identifiers
     # - reassign            # Checks that package variables are not reassigned
     # - rowserrcheck        # checks whether Err of rows is checked successfully
@@ -100,38 +144,34 @@ linters:
     # - unconvert           # Remove unnecessary type conversions
     # - unused              # (megacheck): Checks Go code for unused constants, variables, functions and types
     # - usestdlibvars       # A linter that detect the possibility to use variables/constants from the Go standard library.
+    # - wastedassign        # wastedassign finds wasted assignment statements.
 
     #
     # Recommended? (easy)
     #
 
-    - depguard              # Go linter that checks if package imports are in a list of acceptable packages
     - dogsled               # Checks assignments with too many blank identifiers (e.g. x, _, _, _, := f())
     - errchkjson            # Checks types passed to the json encoding functions. Reports unsupported types and optionally reports occations, where the check for the returned error can be omitted.
-    - errorlint             # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13.
     - exhaustive            # check exhaustiveness of enum switch statements
     - gci                   # Gci control golang package import order and make it always deterministic.
     - godot                 # Check if comments end in a period
     - gofmt                 # Gofmt checks whether code was gofmt-ed. By default this tool runs with -s option to check for code simplification
     - goimports             # In addition to fixing imports, goimports also formats your code in the same style as gofmt.
     - gosec                 # (gas): Inspects source code for security problems
+    - inamedparam           # reports interfaces with unnamed method parameters
     - lll                   # Reports long lines
     - musttag               # enforce field tags in (un)marshaled structs
-    - nakedret              # Finds naked returns in functions greater than a specified function length
-    - nonamedreturns        # Reports all named returns
-    - nosprintfhostport     # Checks for misuse of Sprintf to construct a host with port in a URL.
     - promlinter            # Check Prometheus metrics naming via promlint
+    - protogetter           # Reports direct reads from proto message fields when getters should be used
     - revive                # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint.
-    - tagalign              # check that struct tags are well aligned [fast: true, auto-fix: true]
+    - tagalign              # check that struct tags are well aligned
     - thelper               # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers
-    - wastedassign          # wastedassign finds wasted assignment statements.
     - wrapcheck             # Checks that errors returned from external packages are wrapped
 
     #
     # Recommended? (requires some work)
     #
 
-    - bodyclose             # checks whether HTTP response body is closed successfully
     - containedctx          # containedctx is a linter that detects struct contained context.Context field
     - contextcheck          # check the function whether use a non-inherited context
     - errname               # Checks that sentinel errors are prefixed with the `Err` and error types are suffixed with the `Error`.
@@ -153,15 +193,10 @@ linters:
     #
     # Well intended, but not ready for this
     #
-    - cyclop                # checks function and package cyclomatic complexity
     - dupl                  # Tool for code clone detection
     - forcetypeassert       # finds forced type assertions
-    - gocognit              # Computes and checks the cognitive complexity of functions
-    - gocyclo               # Computes and checks the cyclomatic complexity of functions
     - godox                 # Tool for detection of FIXME, TODO and other comment keywords
     - goerr113              # Golang linter to check the errors handling expressions
-    - maintidx              # maintidx measures the maintainability index of each function.
-    - nestif                # Reports deeply nested if statements
     - paralleltest          # paralleltest detects missing usage of t.Parallel() method in your Go test
     - testpackage           # linter that makes you use a separate _test package
 
@@ -189,8 +224,11 @@ issues:
   # break ‘em.” ― Terry Pratchett
 
   max-issues-per-linter: 0
-  max-same-issues: 10
+  max-same-issues: 0
   exclude-rules:
+
+    # Won't fix:
+
     - path: go.mod
       text: "replacement are not allowed: golang.org/x/time/rate"
 
@@ -199,30 +237,10 @@ issues:
         - govet
       text: "shadow: declaration of \"err\" shadows declaration"
 
-    #
-    # typecheck
-    #
-
-    - linters:
-        - typecheck
-      text: "undefined: min"
-
-    - linters:
-        - typecheck
-      text: "undefined: max"
-
-    #
-    # errcheck
-    #
-
     - linters:
         - errcheck
       text: "Error return value of `.*` is not checked"
 
-    #
-    # gocritic
-    #
-
     - linters:
         - gocritic
       text: "ifElseChain: rewrite if-else to switch statement"
@@ -239,6 +257,74 @@ issues:
         - gocritic
       text: "commentFormatting: put a space between `//` and comment text"
 
+    # Will fix, trivial - just beware of merge conflicts
+
+    - linters:
+        - perfsprint
+      text: "fmt.Sprintf can be replaced .*"
+
+    #
+    # Will fix, easy but some neurons required
+    #
+
+    - linters:
+        - errorlint
+      text: "non-wrapping format verb for fmt.Errorf. Use `%w` to format errors"
+
+    - linters:
+        - errorlint
+      text: "type assertion on error will fail on wrapped errors. Use errors.As to check for specific errors"
+
+    - linters:
+        - errorlint
+      text: "type switch on error will fail on wrapped errors. Use errors.As to check for specific errors"
+
+    - linters:
+        - errorlint
+      text: "type assertion on error will fail on wrapped errors. Use errors.Is to check for specific errors"
+
+    - linters:
+        - errorlint
+      text: "comparing with .* will fail on wrapped errors. Use errors.Is to check for a specific error"
+
+    - linters:
+        - errorlint
+      text: "switch on an error will fail on wrapped errors. Use errors.Is to check for specific errors"
+
+    - linters:
+        - nosprintfhostport
+      text: "host:port in url should be constructed with net.JoinHostPort and not directly with fmt.Sprintf"
+
+    - linters:
+        - wastedassign
+      text: "assigned to .*, but reassigned without using the value"
+
+    # https://github.com/timakin/bodyclose
+    - linters:
+        - bodyclose
+      text: "response body must be closed"
+
+    # named/naked returns are evil, with a single exception
+    # https://go.dev/wiki/CodeReviewComments#named-result-parameters
+    - linters:
+        - nonamedreturns
+      text: "named return .* with type .* found"
+
+    # https://github.com/alexkohler/nakedret#purpose
+    - linters:
+        - nakedret
+      text: "naked return in func .* with .* lines of code"
+
+    #
+    # Will fix,  might be trickier
+    #
+
     - linters:
         - staticcheck
       text: "x509.ParseCRL has been deprecated since Go 1.19: Use ParseRevocationList instead"
+
+    # https://github.com/pkg/errors/issues/245
+    - linters:
+        - depguard
+      text: "import 'github.com/pkg/errors' is not allowed .*"
+

Dockerfile

@@ -1,12 +1,14 @@
 # vim: set ft=dockerfile:
-ARG GOVERSION=1.21.3
+ARG GOVERSION=1.21.6
+ARG BUILD_VERSION
 
-FROM golang:${GOVERSION}-alpine AS build
+FROM golang:${GOVERSION}-alpine3.18 AS build
 
 WORKDIR /go/src/crowdsec
 
 # We like to choose the release of re2 to use, and Alpine does not ship a static version anyway.
 ENV RE2_VERSION=2023-03-01
+ENV BUILD_VERSION=${BUILD_VERSION}
 
 # wizard.sh requires GNU coreutils
 RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils pkgconfig && \
@@ -15,7 +17,7 @@ RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold core
     cd re2-${RE2_VERSION} && \
     make install && \
     echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
-    go install github.com/mikefarah/yq/v4@v4.34.1
+    go install github.com/mikefarah/yq/v4@v4.40.4
 
 COPY . .
 
@@ -32,7 +34,7 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
 
 FROM alpine:latest as slim
 
-RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata bash && \
+RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata bash rsync && \
     mkdir -p /staging/etc/crowdsec && \
     mkdir -p /staging/etc/crowdsec/acquis.d && \
     mkdir -p /staging/var/lib/crowdsec && \
@@ -46,7 +48,7 @@ COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
 COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
 RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml
 
-ENTRYPOINT /bin/bash docker_start.sh
+ENTRYPOINT /bin/bash /docker_start.sh
 
 FROM slim as plugins
 

Dockerfile.debian

@@ -1,5 +1,6 @@
 # vim: set ft=dockerfile:
-ARG GOVERSION=1.21.3
+ARG GOVERSION=1.21.6
+ARG BUILD_VERSION
 
 FROM golang:${GOVERSION}-bookworm AS build
 
@@ -10,6 +11,7 @@ ENV DEBCONF_NOWARNINGS="yes"
 
 # We like to choose the release of re2 to use, the debian version is usually older.
 ENV RE2_VERSION=2023-03-01
+ENV BUILD_VERSION=${BUILD_VERSION}
 
 # wizard.sh requires GNU coreutils
 RUN apt-get update && \
@@ -20,7 +22,7 @@ RUN apt-get update && \
     make && \
     make install && \
     echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
-    go install github.com/mikefarah/yq/v4@v4.34.1
+    go install github.com/mikefarah/yq/v4@v4.40.4
 
 COPY . .
 
@@ -47,7 +49,8 @@ RUN apt-get update && \
     iproute2 \
     ca-certificates \
     bash \
-    tzdata && \
+    tzdata \
+    rsync && \
     mkdir -p /staging/etc/crowdsec && \
     mkdir -p /staging/etc/crowdsec/acquis.d && \
     mkdir -p /staging/var/lib/crowdsec && \

Makefile

@@ -128,11 +128,10 @@ endif
 #--------------------------------------
 
 .PHONY: build
-build: pre-build goversion crowdsec cscli plugins
+build: pre-build goversion crowdsec cscli plugins  ## Build crowdsec, cscli and plugins
 
-# Sanity checks and build information
 .PHONY: pre-build
-pre-build:
+pre-build:  ## Sanity checks and build information
 	$(info Building $(BUILD_VERSION) ($(BUILD_TAG)) $(BUILD_TYPE) for $(GOOS)/$(GOARCH))
 
 ifneq (,$(RE2_FAIL))
@@ -153,14 +152,14 @@ ifeq ($(call bool,$(TEST_COVERAGE)),1)
 	$(info Test coverage collection enabled)
 endif
 
+# intentional, empty line
 	$(info )
 
-
 .PHONY: all
-all: clean test build
+all: clean test build  ## Clean, test and build (requires localstack)
 
 .PHONY: plugins
-plugins:
+plugins:  ## Build notification plugins
 	@$(foreach plugin,$(PLUGINS), \
 		$(MAKE) -C $(PLUGINS_DIR_PREFIX)$(plugin) build $(MAKE_FLAGS); \
 	)
@@ -184,7 +183,7 @@ clean-rpm:
 	@$(RM) -r rpm/SRPMS
 
 .PHONY: clean
-clean: clean-debian clean-rpm testclean
+clean: clean-debian clean-rpm testclean  ## Remove build artifacts
 	@$(MAKE) -C $(CROWDSEC_FOLDER) clean $(MAKE_FLAGS)
 	@$(MAKE) -C $(CSCLI_FOLDER) clean $(MAKE_FLAGS)
 	@$(RM) $(CROWDSEC_BIN) $(WIN_IGNORE_ERR)
@@ -196,21 +195,16 @@ clean: clean-debian clean-rpm testclean
 	)
 
 .PHONY: cscli
-cscli: goversion
+cscli: goversion  ## Build cscli
 	@$(MAKE) -C $(CSCLI_FOLDER) build $(MAKE_FLAGS)
 
 .PHONY: crowdsec
-crowdsec: goversion
+crowdsec: goversion  ## Build crowdsec
 	@$(MAKE) -C $(CROWDSEC_FOLDER) build $(MAKE_FLAGS)
 
-.PHONY: notification-email
-notification-email: goversion
-	@$(MAKE) -C cmd/notification-email build $(MAKE_FLAGS)
-
-
 
 .PHONY: testclean
-testclean: bats-clean
+testclean: bats-clean  ## Remove test artifacts
 	@$(RM) pkg/apiserver/ent $(WIN_IGNORE_ERR)
 	@$(RM) pkg/cwhub/hubdir $(WIN_IGNORE_ERR)
 	@$(RM) pkg/cwhub/install $(WIN_IGNORE_ERR)
@@ -218,42 +212,39 @@ testclean: bats-clean
 
 # for the tests with localstack
 export AWS_ENDPOINT_FORCE=http://localhost:4566
-export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
-export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+export AWS_ACCESS_KEY_ID=test
+export AWS_SECRET_ACCESS_KEY=test
 
 testenv:
 	@echo 'NOTE: You need Docker, docker-compose and run "make localstack" in a separate shell ("make localstack-stop" to terminate it)'
 
-# run the tests with localstack
 .PHONY: test
-test: testenv goversion
+test: testenv goversion  ## Run unit tests with localstack
 	$(GOTEST) $(LD_OPTS) ./...
 
-# run the tests with localstack and coverage
 .PHONY: go-acc
-go-acc: testenv goversion
-	go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS) | \
-		sed 's/ *coverage:.*of statements in.*//'
+go-acc: testenv goversion  ## Run unit tests with localstack + coverage
+	go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS)
 
 # mock AWS services
 .PHONY: localstack
-localstack:
+localstack:  ## Run localstack containers (required for unit testing)
 	docker-compose -f test/localstack/docker-compose.yml up
 
 .PHONY: localstack-stop
-localstack-stop:
+localstack-stop:  ## Stop localstack containers
 	docker-compose -f test/localstack/docker-compose.yml down
 
 # build vendor.tgz to be distributed with the release
 .PHONY: vendor
-vendor: vendor-remove
+vendor: vendor-remove  ## CI only - vendor dependencies and archive them for packaging
 	$(GO) mod vendor
 	tar czf vendor.tgz vendor
 	tar --create --auto-compress --file=$(RELDIR)-vendor.tar.xz vendor
 
 # remove vendor directories and vendor.tgz
 .PHONY: vendor-remove
-vendor-remove:
+vendor-remove:  ## Remove vendor dependencies and archives
 	$(RM) vendor vendor.tgz *-vendor.tar.xz
 
 .PHONY: package
@@ -285,18 +276,15 @@ else
 	@if (Test-Path -Path $(RELDIR)) { echo "$(RELDIR) already exists, abort" ;  exit 1 ; }
 endif
 
-# build a release tarball
 .PHONY: release
-release: check_release build package
+release: check_release build package  ## Build a release tarball
 
-# build the windows installer
 .PHONY: windows_installer
-windows_installer: build
+windows_installer: build  ## Windows - build the installer
 	@.\make_installer.ps1 -version $(BUILD_VERSION)
 
-# build the chocolatey package
 .PHONY: chocolatey
-chocolatey: windows_installer
+chocolatey: windows_installer  ## Windows - build the chocolatey package
 	@.\make_chocolatey.ps1 -version $(BUILD_VERSION)
 
 # Include test/bats.mk only if it exists
@@ -309,3 +297,4 @@ include test/bats.mk
 endif
 
 include mk/goversion.mk
+include mk/help.mk

azure-pipelines.yml

@@ -25,9 +25,9 @@ stages:
               custom: 'tool'
               arguments: 'install --global SignClient --version 1.3.155'
           - task: GoTool@0
-            displayName: "Install Go 1.20"
+            displayName: "Install Go"
             inputs:
-                version: '1.21.3'
+                version: '1.21.6'
 
           - pwsh: |
               choco install -y make

cmd/crowdsec-cli/alerts.go

@@ -21,12 +21,11 @@ import (
 
 	"github.com/crowdsecurity/go-cs-lib/version"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/models"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
 func DecisionsFromAlert(alert *models.Alert) string {
@@ -208,8 +207,14 @@ func DisplayOneAlert(alert *models.Alert, withDetail bool) error {
 	return nil
 }
 
-func NewAlertsCmd() *cobra.Command {
-	var cmdAlerts = &cobra.Command{
+type cliAlerts struct{}
+
+func NewCLIAlerts() *cliAlerts {
+	return &cliAlerts{}
+}
+
+func (cli cliAlerts) NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "alerts [action]",
 		Short:             "Manage alerts",
 		Args:              cobra.MinimumNArgs(1),
@@ -239,15 +244,15 @@ func NewAlertsCmd() *cobra.Command {
 		},
 	}
 
-	cmdAlerts.AddCommand(NewAlertsListCmd())
-	cmdAlerts.AddCommand(NewAlertsInspectCmd())
-	cmdAlerts.AddCommand(NewAlertsFlushCmd())
-	cmdAlerts.AddCommand(NewAlertsDeleteCmd())
+	cmd.AddCommand(cli.NewListCmd())
+	cmd.AddCommand(cli.NewInspectCmd())
+	cmd.AddCommand(cli.NewFlushCmd())
+	cmd.AddCommand(cli.NewDeleteCmd())
 
-	return cmdAlerts
+	return cmd
 }
 
-func NewAlertsListCmd() *cobra.Command {
+func (cli cliAlerts) NewListCmd() *cobra.Command {
 	var alertListFilter = apiclient.AlertsListOpts{
 		ScopeEquals:    new(string),
 		ValueEquals:    new(string),
@@ -260,10 +265,11 @@ func NewAlertsListCmd() *cobra.Command {
 		IncludeCAPI:    new(bool),
 		OriginEquals:   new(string),
 	}
-	var limit = new(int)
+	limit := new(int)
 	contained := new(bool)
 	var printMachine bool
-	var cmdAlertsList = &cobra.Command{
+
+	cmd := &cobra.Command{
 		Use:   "list [filters]",
 		Short: "List alerts",
 		Example: `cscli alerts list
@@ -353,25 +359,25 @@ cscli alerts list --type ban`,
 			return nil
 		},
 	}
-	cmdAlertsList.Flags().SortFlags = false
-	cmdAlertsList.Flags().BoolVarP(alertListFilter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
-	cmdAlertsList.Flags().StringVar(alertListFilter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
-	cmdAlertsList.Flags().StringVar(alertListFilter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
-	cmdAlertsList.Flags().StringVarP(alertListFilter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
-	cmdAlertsList.Flags().StringVarP(alertListFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
-	cmdAlertsList.Flags().StringVarP(alertListFilter.RangeEquals, "range", "r", "", "restrict to alerts from this range (shorthand for --scope range --value <RANGE/X>)")
-	cmdAlertsList.Flags().StringVar(alertListFilter.TypeEquals, "type", "", "restrict to alerts with given decision type (ie. ban, captcha)")
-	cmdAlertsList.Flags().StringVar(alertListFilter.ScopeEquals, "scope", "", "restrict to alerts of this scope (ie. ip,range)")
-	cmdAlertsList.Flags().StringVarP(alertListFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
-	cmdAlertsList.Flags().StringVar(alertListFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
-	cmdAlertsList.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
-	cmdAlertsList.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that sent alerts")
-	cmdAlertsList.Flags().IntVarP(limit, "limit", "l", 50, "limit size of alerts list table (0 to view all alerts)")
+	cmd.Flags().SortFlags = false
+	cmd.Flags().BoolVarP(alertListFilter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
+	cmd.Flags().StringVar(alertListFilter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
+	cmd.Flags().StringVar(alertListFilter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
+	cmd.Flags().StringVarP(alertListFilter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
+	cmd.Flags().StringVarP(alertListFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
+	cmd.Flags().StringVarP(alertListFilter.RangeEquals, "range", "r", "", "restrict to alerts from this range (shorthand for --scope range --value <RANGE/X>)")
+	cmd.Flags().StringVar(alertListFilter.TypeEquals, "type", "", "restrict to alerts with given decision type (ie. ban, captcha)")
+	cmd.Flags().StringVar(alertListFilter.ScopeEquals, "scope", "", "restrict to alerts of this scope (ie. ip,range)")
+	cmd.Flags().StringVarP(alertListFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
+	cmd.Flags().StringVar(alertListFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
+	cmd.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
+	cmd.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that sent alerts")
+	cmd.Flags().IntVarP(limit, "limit", "l", 50, "limit size of alerts list table (0 to view all alerts)")
 
-	return cmdAlertsList
+	return cmd
 }
 
-func NewAlertsDeleteCmd() *cobra.Command {
+func (cli cliAlerts) NewDeleteCmd() *cobra.Command {
 	var ActiveDecision *bool
 	var AlertDeleteAll bool
 	var delAlertByID string
@@ -383,7 +389,7 @@ func NewAlertsDeleteCmd() *cobra.Command {
 		IPEquals:       new(string),
 		RangeEquals:    new(string),
 	}
-	var cmdAlertsDelete = &cobra.Command{
+	cmd := &cobra.Command{
 		Use: "delete [filters] [--all]",
 		Short: `Delete alerts
 /!\ This command can be use only on the same machine than the local API.`,
@@ -461,21 +467,21 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`,
 			return nil
 		},
 	}
-	cmdAlertsDelete.Flags().SortFlags = false
-	cmdAlertsDelete.Flags().StringVar(alertDeleteFilter.ScopeEquals, "scope", "", "the scope (ie. ip,range)")
-	cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
-	cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
-	cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
-	cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
-	cmdAlertsDelete.Flags().StringVar(&delAlertByID, "id", "", "alert ID")
-	cmdAlertsDelete.Flags().BoolVarP(&AlertDeleteAll, "all", "a", false, "delete all alerts")
-	cmdAlertsDelete.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
-	return cmdAlertsDelete
+	cmd.Flags().SortFlags = false
+	cmd.Flags().StringVar(alertDeleteFilter.ScopeEquals, "scope", "", "the scope (ie. ip,range)")
+	cmd.Flags().StringVarP(alertDeleteFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
+	cmd.Flags().StringVarP(alertDeleteFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
+	cmd.Flags().StringVarP(alertDeleteFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
+	cmd.Flags().StringVarP(alertDeleteFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
+	cmd.Flags().StringVar(&delAlertByID, "id", "", "alert ID")
+	cmd.Flags().BoolVarP(&AlertDeleteAll, "all", "a", false, "delete all alerts")
+	cmd.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
+	return cmd
 }
 
-func NewAlertsInspectCmd() *cobra.Command {
+func (cli cliAlerts) NewInspectCmd() *cobra.Command {
 	var details bool
-	var cmdAlertsInspect = &cobra.Command{
+	cmd := &cobra.Command{
 		Use:               `inspect "alert_id"`,
 		Short:             `Show info about an alert`,
 		Example:           `cscli alerts inspect 123`,
@@ -517,16 +523,16 @@ func NewAlertsInspectCmd() *cobra.Command {
 			return nil
 		},
 	}
-	cmdAlertsInspect.Flags().SortFlags = false
-	cmdAlertsInspect.Flags().BoolVarP(&details, "details", "d", false, "show alerts with events")
+	cmd.Flags().SortFlags = false
+	cmd.Flags().BoolVarP(&details, "details", "d", false, "show alerts with events")
 
-	return cmdAlertsInspect
+	return cmd
 }
 
-func NewAlertsFlushCmd() *cobra.Command {
+func (cli cliAlerts) NewFlushCmd() *cobra.Command {
 	var maxItems int
 	var maxAge string
-	var cmdAlertsFlush = &cobra.Command{
+	cmd := &cobra.Command{
 		Use: `flush`,
 		Short: `Flush alerts
 /!\ This command can be used only on the same machine than the local API`,
@@ -552,9 +558,9 @@ func NewAlertsFlushCmd() *cobra.Command {
 		},
 	}
 
-	cmdAlertsFlush.Flags().SortFlags = false
-	cmdAlertsFlush.Flags().IntVar(&maxItems, "max-items", 5000, "Maximum number of alert items to keep in the database")
-	cmdAlertsFlush.Flags().StringVar(&maxAge, "max-age", "7d", "Maximum age of alert items to keep in the database")
+	cmd.Flags().SortFlags = false
+	cmd.Flags().IntVar(&maxItems, "max-items", 5000, "Maximum number of alert items to keep in the database")
+	cmd.Flags().StringVar(&maxAge, "max-age", "7d", "Maximum age of alert items to keep in the database")
 
-	return cmdAlertsFlush
+	return cmd
 }

cmd/crowdsec-cli/bouncers.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"slices"
 	"strings"
 	"time"
 
@@ -13,12 +12,12 @@ import (
 	"github.com/fatih/color"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"slices"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
 func getBouncers(out io.Writer, dbClient *database.Client) error {
@@ -26,16 +25,18 @@ func getBouncers(out io.Writer, dbClient *database.Client) error {
 	if err != nil {
 		return fmt.Errorf("unable to list bouncers: %s", err)
 	}
-	if csConfig.Cscli.Output == "human" {
+
+	switch csConfig.Cscli.Output {
+	case "human":
 		getBouncersTable(out, bouncers)
-	} else if csConfig.Cscli.Output == "json" {
+	case "json":
 		enc := json.NewEncoder(out)
 		enc.SetIndent("", "  ")
 		if err := enc.Encode(bouncers); err != nil {
 			return fmt.Errorf("failed to unmarshal: %w", err)
 		}
 		return nil
-	} else if csConfig.Cscli.Output == "raw" {
+	case "raw":
 		csvwriter := csv.NewWriter(out)
 		err := csvwriter.Write([]string{"name", "ip", "revoked", "last_pull", "type", "version", "auth_type"})
 		if err != nil {
@@ -55,11 +56,50 @@ func getBouncers(out io.Writer, dbClient *database.Client) error {
 		}
 		csvwriter.Flush()
 	}
+
 	return nil
 }
 
-func NewBouncersListCmd() *cobra.Command {
-	cmdBouncersList := &cobra.Command{
+type cliBouncers struct {}
+
+func NewCLIBouncers() *cliBouncers {
+	return &cliBouncers{}
+}
+
+func (cli cliBouncers) NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "bouncers [action]",
+		Short: "Manage bouncers [requires local API]",
+		Long: `To list/add/delete/prune bouncers.
+Note: This command requires database direct access, so is intended to be run on Local API/master.
+`,
+		Args:              cobra.MinimumNArgs(1),
+		Aliases:           []string{"bouncer"},
+		DisableAutoGenTag: true,
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			var err error
+			if err = require.LAPI(csConfig); err != nil {
+				return err
+			}
+
+			dbClient, err = database.NewClient(csConfig.DbConfig)
+			if err != nil {
+				return fmt.Errorf("unable to create new database client: %s", err)
+			}
+			return nil
+		},
+	}
+
+	cmd.AddCommand(cli.NewListCmd())
+	cmd.AddCommand(cli.NewAddCmd())
+	cmd.AddCommand(cli.NewDeleteCmd())
+	cmd.AddCommand(cli.NewPruneCmd())
+
+	return cmd
+}
+
+func (cli cliBouncers) NewListCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "list",
 		Short:             "list all bouncers within the database",
 		Example:           `cscli bouncers list`,
@@ -74,16 +114,13 @@ func NewBouncersListCmd() *cobra.Command {
 		},
 	}
 
-	return cmdBouncersList
+	return cmd
 }
 
-func runBouncersAdd(cmd *cobra.Command, args []string) error {
-	flags := cmd.Flags()
+func (cli cliBouncers) add(cmd *cobra.Command, args []string) error {
+	keyLength := 32
 
-	keyLength, err := flags.GetInt("length")
-	if err != nil {
-		return err
-	}
+	flags := cmd.Flags()
 
 	key, err := flags.GetString("key")
 	if err != nil {
@@ -108,13 +145,14 @@ func runBouncersAdd(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("unable to create bouncer: %s", err)
 	}
 
-	if csConfig.Cscli.Output == "human" {
+	switch csConfig.Cscli.Output {
+	case "human":
 		fmt.Printf("API key for '%s':\n\n", keyName)
 		fmt.Printf("   %s\n\n", apiKey)
 		fmt.Print("Please keep this key since you will not be able to retrieve it!\n")
-	} else if csConfig.Cscli.Output == "raw" {
+	case "raw":
 		fmt.Printf("%s", apiKey)
-	} else if csConfig.Cscli.Output == "json" {
+	case "json":
 		j, err := json.Marshal(apiKey)
 		if err != nil {
 			return fmt.Errorf("unable to marshal api key")
@@ -125,27 +163,26 @@ func runBouncersAdd(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-func NewBouncersAddCmd() *cobra.Command {
-	cmdBouncersAdd := &cobra.Command{
-		Use:   "add MyBouncerName [--length 16]",
+func (cli cliBouncers) NewAddCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "add MyBouncerName",
 		Short: "add a single bouncer to the database",
 		Example: `cscli bouncers add MyBouncerName
-cscli bouncers add MyBouncerName -l 24
-cscli bouncers add MyBouncerName -k <random-key>`,
+cscli bouncers add MyBouncerName --key <random-key>`,
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
-		RunE:              runBouncersAdd,
+		RunE:              cli.add,
 	}
 
-	flags := cmdBouncersAdd.Flags()
-
-	flags.IntP("length", "l", 16, "length of the api key")
+	flags := cmd.Flags()
+	flags.StringP("length", "l", "", "length of the api key")
+	flags.MarkDeprecated("length", "use --key instead")
 	flags.StringP("key", "k", "", "api key for the bouncer")
 
-	return cmdBouncersAdd
+	return cmd
 }
 
-func runBouncersDelete(cmd *cobra.Command, args []string) error {
+func (cli cliBouncers) delete(cmd *cobra.Command, args []string) error {
 	for _, bouncerID := range args {
 		err := dbClient.DeleteBouncer(bouncerID)
 		if err != nil {
@@ -157,8 +194,8 @@ func runBouncersDelete(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-func NewBouncersDeleteCmd() *cobra.Command {
-	cmdBouncersDelete := &cobra.Command{
+func (cli cliBouncers) NewDeleteCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "delete MyBouncerName",
 		Short:             "delete bouncer(s) from the database",
 		Args:              cobra.MinimumNArgs(1),
@@ -183,15 +220,15 @@ func NewBouncersDeleteCmd() *cobra.Command {
 			}
 			return ret, cobra.ShellCompDirectiveNoFileComp
 		},
-		RunE: runBouncersDelete,
+		RunE: cli.delete,
 	}
 
-	return cmdBouncersDelete
+	return cmd
 }
 
-func NewBouncersPruneCmd() *cobra.Command {
+func (cli cliBouncers) NewPruneCmd() *cobra.Command {
 	var parsedDuration time.Duration
-	cmdBouncersPrune := &cobra.Command{
+	cmd := &cobra.Command{
 		Use:               "prune",
 		Short:             "prune multiple bouncers from the database",
 		Args:              cobra.NoArgs,
@@ -254,39 +291,7 @@ cscli bouncers prune -d 60m --force`,
 			return nil
 		},
 	}
-	cmdBouncersPrune.Flags().StringP("duration", "d", "60m", "duration of time since last pull")
-	cmdBouncersPrune.Flags().Bool("force", false, "force prune without asking for confirmation")
-	return cmdBouncersPrune
-}
-
-func NewBouncersCmd() *cobra.Command {
-	var cmdBouncers = &cobra.Command{
-		Use:   "bouncers [action]",
-		Short: "Manage bouncers [requires local API]",
-		Long: `To list/add/delete/prune bouncers.
-Note: This command requires database direct access, so is intended to be run on Local API/master.
-`,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"bouncer"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			var err error
-			if err = require.LAPI(csConfig); err != nil {
-				return err
-			}
-
-			dbClient, err = database.NewClient(csConfig.DbConfig)
-			if err != nil {
-				return fmt.Errorf("unable to create new database client: %s", err)
-			}
-			return nil
-		},
-	}
-
-	cmdBouncers.AddCommand(NewBouncersListCmd())
-	cmdBouncers.AddCommand(NewBouncersAddCmd())
-	cmdBouncers.AddCommand(NewBouncersDeleteCmd())
-	cmdBouncers.AddCommand(NewBouncersPruneCmd())
-
-	return cmdBouncers
+	cmd.Flags().StringP("duration", "d", "60m", "duration of time since last pull")
+	cmd.Flags().Bool("force", false, "force prune without asking for confirmation")
+	return cmd
 }

cmd/crowdsec-cli/capi.go

@@ -13,26 +13,32 @@ import (
 
 	"github.com/crowdsecurity/go-cs-lib/version"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-	"github.com/crowdsecurity/crowdsec/pkg/fflag"
 	"github.com/crowdsecurity/crowdsec/pkg/models"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
-const CAPIBaseURL string = "https://api.crowdsec.net/"
-const CAPIURLPrefix = "v3"
+const (
+	CAPIBaseURL   = "https://api.crowdsec.net/"
+	CAPIURLPrefix = "v3"
+)
 
-func NewCapiCmd() *cobra.Command {
-	var cmdCapi = &cobra.Command{
+type cliCapi struct{}
+
+func NewCLICapi() *cliCapi {
+	return &cliCapi{}
+}
+
+func (cli cliCapi) NewCommand() *cobra.Command {
+	var cmd = &cobra.Command{
 		Use:               "capi [action]",
 		Short:             "Manage interaction with Central API (CAPI)",
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
 			if err := require.LAPI(csConfig); err != nil {
 				return err
 			}
@@ -45,22 +51,24 @@ func NewCapiCmd() *cobra.Command {
 		},
 	}
 
-	cmdCapi.AddCommand(NewCapiRegisterCmd())
-	cmdCapi.AddCommand(NewCapiStatusCmd())
+	cmd.AddCommand(cli.NewRegisterCmd())
+	cmd.AddCommand(cli.NewStatusCmd())
 
-	return cmdCapi
+	return cmd
 }
 
-func NewCapiRegisterCmd() *cobra.Command {
-	var capiUserPrefix string
-	var outputFile string
+func (cli cliCapi) NewRegisterCmd() *cobra.Command {
+	var (
+		capiUserPrefix string
+		outputFile string
+	)
 
-	var cmdCapiRegister = &cobra.Command{
+	var cmd = &cobra.Command{
 		Use:               "register",
 		Short:             "Register to Central API (CAPI)",
 		Args:              cobra.MinimumNArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(_ *cobra.Command, _ []string) error {
 			var err error
 			capiUser, err := generateID(capiUserPrefix)
 			if err != nil {
@@ -98,21 +106,18 @@ func NewCapiRegisterCmd() *cobra.Command {
 				Password: password.String(),
 				URL:      types.CAPIBaseURL,
 			}
-			if fflag.PapiClient.IsEnabled() {
-				apiCfg.PapiURL = types.PAPIBaseURL
-			}
 			apiConfigDump, err := yaml.Marshal(apiCfg)
 			if err != nil {
 				return fmt.Errorf("unable to marshal api credentials: %w", err)
 			}
 			if dumpFile != "" {
-				err = os.WriteFile(dumpFile, apiConfigDump, 0600)
+				err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
 				if err != nil {
 					return fmt.Errorf("write api credentials in '%s' failed: %w", dumpFile, err)
 				}
-				log.Printf("Central API credentials dumped to '%s'", dumpFile)
+				log.Printf("Central API credentials written to '%s'", dumpFile)
 			} else {
-				fmt.Printf("%s\n", string(apiConfigDump))
+				fmt.Println(string(apiConfigDump))
 			}
 
 			log.Warning(ReloadMessage())
@@ -120,28 +125,26 @@ func NewCapiRegisterCmd() *cobra.Command {
 			return nil
 		},
 	}
-	cmdCapiRegister.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
-	cmdCapiRegister.Flags().StringVar(&capiUserPrefix, "schmilblick", "", "set a schmilblick (use in tests only)")
-	if err := cmdCapiRegister.Flags().MarkHidden("schmilblick"); err != nil {
+
+	cmd.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
+	cmd.Flags().StringVar(&capiUserPrefix, "schmilblick", "", "set a schmilblick (use in tests only)")
+
+	if err := cmd.Flags().MarkHidden("schmilblick"); err != nil {
 		log.Fatalf("failed to hide flag: %s", err)
 	}
 
-	return cmdCapiRegister
+	return cmd
 }
 
-func NewCapiStatusCmd() *cobra.Command {
-	var cmdCapiStatus = &cobra.Command{
+func (cli cliCapi) NewStatusCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "status",
 		Short:             "Check status with the Central API (CAPI)",
 		Args:              cobra.MinimumNArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if csConfig.API.Server.OnlineClient == nil {
-				return fmt.Errorf("please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
-			}
-
-			if csConfig.API.Server.OnlineClient.Credentials == nil {
-				return fmt.Errorf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
+		RunE: func(_ *cobra.Command, _ []string) error {
+			if err := require.CAPIRegistered(csConfig); err != nil {
+				return err
 			}
 
 			password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
@@ -151,11 +154,12 @@ func NewCapiStatusCmd() *cobra.Command {
 				return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err)
 			}
 
-			if err := require.Hub(csConfig); err != nil {
+			hub, err := require.Hub(csConfig, nil, nil)
+			if err != nil {
 				return err
 			}
 
-			scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
+			scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
 			if err != nil {
 				return fmt.Errorf("failed to get scenarios: %w", err)
 			}
@@ -188,5 +192,5 @@ func NewCapiStatusCmd() *cobra.Command {
 		},
 	}
 
-	return cmdCapiStatus
+	return cmd
 }

cmd/crowdsec-cli/collections.go

@@ -1,176 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/fatih/color"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewCollectionsCmd() *cobra.Command {
-	var cmdCollections = &cobra.Command{
-		Use:   "collections [action]",
-		Short: "Manage collections from hub",
-		Long:  `Install/Remove/Upgrade/Inspect collections from the CrowdSec Hub.`,
-		/*TBD fix help*/
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"collection"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := require.Hub(csConfig); err != nil {
-				return err
-			}
-
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			if cmd.Name() == "inspect" || cmd.Name() == "list" {
-				return
-			}
-			log.Infof(ReloadMessage())
-		},
-	}
-
-	var ignoreError bool
-
-	var cmdCollectionsInstall = &cobra.Command{
-		Use:     "install collection",
-		Short:   "Install given collection(s)",
-		Long:    `Fetch and install given collection(s) from hub`,
-		Example: `cscli collections install crowdsec/xxx crowdsec/xyz`,
-		Args:    cobra.MinimumNArgs(1),
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(cwhub.COLLECTIONS, args, toComplete)
-		},
-		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			for _, name := range args {
-				t := cwhub.GetItem(cwhub.COLLECTIONS, name)
-				if t == nil {
-					nearestItem, score := GetDistance(cwhub.COLLECTIONS, name)
-					Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError)
-					continue
-				}
-				if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
-					if !ignoreError {
-						return fmt.Errorf("error while installing '%s': %w", name, err)
-					}
-					log.Errorf("Error while installing '%s': %s", name, err)
-				}
-			}
-			return nil
-		},
-	}
-	cmdCollectionsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
-	cmdCollectionsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
-	cmdCollectionsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple collections")
-	cmdCollections.AddCommand(cmdCollectionsInstall)
-
-	var cmdCollectionsRemove = &cobra.Command{
-		Use:               "remove collection",
-		Short:             "Remove given collection(s)",
-		Long:              `Remove given collection(s) from hub`,
-		Example:           `cscli collections remove crowdsec/xxx crowdsec/xyz`,
-		Aliases:           []string{"delete"},
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, forceAction)
-				return nil
-			}
-
-			if len(args) == 0 {
-				return fmt.Errorf("specify at least one collection to remove or '--all'")
-			}
-
-			for _, name := range args {
-				if !forceAction {
-					item := cwhub.GetItem(cwhub.COLLECTIONS, name)
-					if item == nil {
-						return fmt.Errorf("unable to retrieve: %s", name)
-					}
-					if len(item.BelongsToCollections) > 0 {
-						log.Warningf("%s belongs to other collections :\n%s\n", name, item.BelongsToCollections)
-						log.Printf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection\n", name)
-						continue
-					}
-				}
-				cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, forceAction)
-			}
-			return nil
-		},
-	}
-	cmdCollectionsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
-	cmdCollectionsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
-	cmdCollectionsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the collections")
-	cmdCollections.AddCommand(cmdCollectionsRemove)
-
-	var cmdCollectionsUpgrade = &cobra.Command{
-		Use:               "upgrade collection",
-		Short:             "Upgrade given collection(s)",
-		Long:              `Fetch and upgrade given collection(s) from hub`,
-		Example:           `cscli collections upgrade crowdsec/xxx crowdsec/xyz`,
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
-			} else {
-				if len(args) == 0 {
-					return fmt.Errorf("specify at least one collection to upgrade or '--all'")
-				}
-				for _, name := range args {
-					cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, forceAction)
-				}
-			}
-			return nil
-		},
-	}
-	cmdCollectionsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the collections")
-	cmdCollectionsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
-	cmdCollections.AddCommand(cmdCollectionsUpgrade)
-
-	var cmdCollectionsInspect = &cobra.Command{
-		Use:               "inspect collection",
-		Short:             "Inspect given collection",
-		Long:              `Inspect given collection`,
-		Example:           `cscli collections inspect crowdsec/xxx crowdsec/xyz`,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
-		},
-		Run: func(cmd *cobra.Command, args []string) {
-			for _, name := range args {
-				InspectItem(name, cwhub.COLLECTIONS)
-			}
-		},
-	}
-	cmdCollectionsInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
-	cmdCollections.AddCommand(cmdCollectionsInspect)
-
-	var cmdCollectionsList = &cobra.Command{
-		Use:               "list collection [-a]",
-		Short:             "List all collections",
-		Long:              `List all collections`,
-		Example:           `cscli collections list`,
-		Args:              cobra.ExactArgs(0),
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all)
-		},
-	}
-	cmdCollectionsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
-	cmdCollections.AddCommand(cmdCollectionsList)
-
-	return cmdCollections
-}

cmd/crowdsec-cli/completion.go

@@ -7,8 +7,7 @@ import (
 )
 
 func NewCompletionCmd() *cobra.Command {
-
-	var completionCmd = &cobra.Command{
+	completionCmd := &cobra.Command{
 		Use:   "completion [bash|zsh|powershell|fish]",
 		Short: "Generate completion script",
 		Long: `To load completions:
@@ -82,5 +81,6 @@ func NewCompletionCmd() *cobra.Command {
 			}
 		},
 	}
+
 	return completionCmd
 }

cmd/crowdsec-cli/config_backup.go

@@ -9,69 +9,82 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
 func backupHub(dirPath string) error {
-	var err error
-	var itemDirectory string
-	var upstreamParsers []string
+	hub, err := require.Hub(csConfig, nil, nil)
+	if err != nil {
+		return err
+	}
 
 	for _, itemType := range cwhub.ItemTypes {
 		clog := log.WithFields(log.Fields{
 			"type": itemType,
 		})
-		itemMap := cwhub.GetItemMap(itemType)
+
+		itemMap := hub.GetItemMap(itemType)
 		if itemMap == nil {
 			clog.Infof("No %s to backup.", itemType)
 			continue
 		}
-		itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
-		if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
+
+		itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itemType)
+		if err = os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
 			return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
 		}
-		upstreamParsers = []string{}
+
+		upstreamParsers := []string{}
+
 		for k, v := range itemMap {
 			clog = clog.WithFields(log.Fields{
 				"file": v.Name,
 			})
-			if !v.Installed { //only backup installed ones
+			if !v.State.Installed { //only backup installed ones
 				clog.Debugf("[%s] : not installed", k)
 				continue
 			}
 
-			//for the local/tainted ones, we backup the full file
-			if v.Tainted || v.Local || !v.UpToDate {
+			//for the local/tainted ones, we back up the full file
+			if v.State.Tainted || v.State.IsLocal() || !v.State.UpToDate {
 				//we need to backup stages for parsers
-				if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
+				if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS {
 					fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
-					if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
+					if err = os.MkdirAll(fstagedir, os.ModePerm); err != nil {
 						return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
 					}
 				}
-				clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
+
+				clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.State.IsLocal(), v.State.UpToDate)
+
 				tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
-				if err = CopyFile(v.LocalPath, tfile); err != nil {
-					return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
+				if err = CopyFile(v.State.LocalPath, tfile); err != nil {
+					return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.State.LocalPath, tfile, err)
 				}
-				clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
+
+				clog.Infof("local/tainted saved %s to %s", v.State.LocalPath, tfile)
+
 				continue
 			}
-			clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
-			clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
+
+			clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.State.UpToDate)
+			clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.State.UpToDate)
 			upstreamParsers = append(upstreamParsers, v.Name)
 		}
 		//write the upstream items
 		upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
+
 		upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
 		if err != nil {
 			return fmt.Errorf("failed marshaling upstream parsers : %s", err)
 		}
-		err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
+
+		err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0o644)
 		if err != nil {
 			return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
 		}
+
 		clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
 	}
 
@@ -100,7 +113,7 @@ func backupConfigToDirectory(dirPath string) error {
 
 	/*if parent directory doesn't exist, bail out. create final dir with Mkdir*/
 	parentDir := filepath.Dir(dirPath)
-	if _, err := os.Stat(parentDir); err != nil {
+	if _, err = os.Stat(parentDir); err != nil {
 		return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err)
 	}
 
@@ -197,10 +210,6 @@ func backupConfigToDirectory(dirPath string) error {
 }
 
 func runConfigBackup(cmd *cobra.Command, args []string) error {
-	if err := require.Hub(csConfig); err != nil {
-		return err
-	}
-
 	if err := backupConfigToDirectory(args[0]); err != nil {
 		return fmt.Errorf("failed to backup config: %w", err)
 	}

cmd/crowdsec-cli/config_feature_flags.go

@@ -44,6 +44,7 @@ func runConfigFeatureFlags(cmd *cobra.Command, args []string) error {
 		if feat.State == fflag.RetiredState {
 			fmt.Printf("\n  %s %s", magenta("RETIRED"), feat.DeprecationMsg)
 		}
+
 		fmt.Println()
 	}
 
@@ -58,10 +59,12 @@ func runConfigFeatureFlags(cmd *cobra.Command, args []string) error {
 			retired = append(retired, feat)
 			continue
 		}
+
 		if feat.IsEnabled() {
 			enabled = append(enabled, feat)
 			continue
 		}
+
 		disabled = append(disabled, feat)
 	}
 

cmd/crowdsec-cli/config_restore.go

@@ -21,45 +21,12 @@ type OldAPICfg struct {
 	Password  string `json:"password"`
 }
 
-// it's a rip of the cli version, but in silent-mode
-func silentInstallItem(name string, obtype string) (string, error) {
-	var item = cwhub.GetItem(obtype, name)
-	if item == nil {
-		return "", fmt.Errorf("error retrieving item")
-	}
-	if downloadOnly && item.Downloaded && item.UpToDate {
-		return fmt.Sprintf("%s is already downloaded and up-to-date", item.Name), nil
-	}
-	err := cwhub.DownloadLatest(csConfig.Hub, item, forceAction, false)
-	if err != nil {
-		return "", fmt.Errorf("error while downloading %s : %v", item.Name, err)
-	}
-	if err := cwhub.AddItem(obtype, *item); err != nil {
-		return "", err
-	}
-
-	if downloadOnly {
-		return fmt.Sprintf("Downloaded %s to %s", item.Name, csConfig.Cscli.HubDir+"/"+item.RemotePath), nil
-	}
-	err = cwhub.EnableItem(csConfig.Hub, item)
-	if err != nil {
-		return "", fmt.Errorf("error while enabling %s : %v", item.Name, err)
-	}
-	if err := cwhub.AddItem(obtype, *item); err != nil {
-		return "", err
-	}
-	return fmt.Sprintf("Enabled %s", item.Name), nil
-}
-
 func restoreHub(dirPath string) error {
-	var err error
-
-	if err := csConfig.LoadHub(); err != nil {
+	hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), nil)
+	if err != nil {
 		return err
 	}
 
-	cwhub.SetHubBranch()
-
 	for _, itype := range cwhub.ItemTypes {
 		itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
 		if _, err = os.Stat(itemDirectory); err != nil {
@@ -68,23 +35,29 @@ func restoreHub(dirPath string) error {
 		}
 		/*restore the upstream items*/
 		upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
+
 		file, err := os.ReadFile(upstreamListFN)
 		if err != nil {
 			return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
 		}
+
 		var upstreamList []string
+
 		err = json.Unmarshal(file, &upstreamList)
 		if err != nil {
 			return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
 		}
+
 		for _, toinstall := range upstreamList {
-			label, err := silentInstallItem(toinstall, itype)
+			item := hub.GetItem(itype, toinstall)
+			if item == nil {
+				log.Errorf("Item %s/%s not found in hub", itype, toinstall)
+				continue
+			}
+
+			err := item.Install(false, false)
 			if err != nil {
 				log.Errorf("Error while installing %s : %s", toinstall, err)
-			} else if label != "" {
-				log.Infof("Installed %s : %s", toinstall, label)
-			} else {
-				log.Printf("Installed %s : ok", toinstall)
 			}
 		}
 
@@ -93,23 +66,28 @@ func restoreHub(dirPath string) error {
 		if err != nil {
 			return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
 		}
+
 		for _, file := range files {
 			//this was the upstream data
 			if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
 				continue
 			}
-			if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
+
+			if itype == cwhub.PARSERS || itype == cwhub.POSTOVERFLOWS {
 				//we expect a stage here
 				if !file.IsDir() {
 					continue
 				}
+
 				stage := file.Name()
 				stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
 				log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
+
 				if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
 					return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
 				}
-				/*find items*/
+
+				// find items
 				ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
 				if err != nil {
 					return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
@@ -118,10 +96,12 @@ func restoreHub(dirPath string) error {
 				for _, tfile := range ifiles {
 					log.Infof("Going to restore local/tainted [%s]", tfile.Name())
 					sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
+
 					destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
 					if err = CopyFile(sourceFile, destinationFile); err != nil {
 						return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
 					}
+
 					log.Infof("restored %s to %s", sourceFile, destinationFile)
 				}
 			} else {
@@ -133,9 +113,9 @@ func restoreHub(dirPath string) error {
 				}
 				log.Infof("restored %s to %s", sourceFile, destinationFile)
 			}
-
 		}
 	}
+
 	return nil
 }
 
@@ -215,7 +195,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
 			if csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" {
 				apiConfigDumpFile = csConfig.API.Server.OnlineClient.CredentialsFilePath
 			}
-			err = os.WriteFile(apiConfigDumpFile, apiConfigDump, 0o644)
+			err = os.WriteFile(apiConfigDumpFile, apiConfigDump, 0o600)
 			if err != nil {
 				return fmt.Errorf("write api credentials in '%s' failed: %s", apiConfigDumpFile, err)
 			}
@@ -302,10 +282,6 @@ func runConfigRestore(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	if err := require.Hub(csConfig); err != nil {
-		return err
-	}
-
 	if err := restoreConfigFromDirectory(args[0], oldBackup); err != nil {
 		return fmt.Errorf("failed to restore config from %s: %w", args[0], err)
 	}

cmd/crowdsec-cli/config_show.go

@@ -7,6 +7,7 @@ import (
 	"text/template"
 
 	"github.com/antonmedv/expr"
+	"github.com/sanity-io/litter"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
@@ -23,6 +24,7 @@ func showConfigKey(key string) error {
 	opts := []expr.Option{}
 	opts = append(opts, exprhelpers.GetExprOptions(map[string]interface{}{})...)
 	opts = append(opts, expr.Env(Env{}))
+
 	program, err := expr.Compile(key, opts...)
 	if err != nil {
 		return err
@@ -35,13 +37,13 @@ func showConfigKey(key string) error {
 
 	switch csConfig.Cscli.Output {
 	case "human", "raw":
+		// Don't use litter for strings, it adds quotes
+		// that we didn't have before
 		switch output.(type) {
 		case string:
-			fmt.Printf("%s\n", output)
-		case int:
-			fmt.Printf("%d\n", output)
+			fmt.Println(output)
 		default:
-			fmt.Printf("%v\n", output)
+			litter.Dump(output)
 		}
 	case "json":
 		data, err := json.MarshalIndent(output, "", "  ")
@@ -51,6 +53,7 @@ func showConfigKey(key string) error {
 
 		fmt.Printf("%s\n", string(data))
 	}
+
 	return nil
 }
 
@@ -82,7 +85,6 @@ Crowdsec{{if and .Crowdsec.Enable (not (ValueBool .Crowdsec.Enable))}} (disabled
 cscli:
   - Output                  : {{.Cscli.Output}}
   - Hub Branch              : {{.Cscli.HubBranch}}
-  - Hub Folder              : {{.Cscli.HubDir}}
 {{- end }}
 
 {{- if .API }}
@@ -211,6 +213,7 @@ func runConfigShow(cmd *cobra.Command, args []string) error {
 		if err != nil {
 			return err
 		}
+
 		err = tmp.Execute(os.Stdout, csConfig)
 		if err != nil {
 			return err
@@ -230,6 +233,7 @@ func runConfigShow(cmd *cobra.Command, args []string) error {
 
 		fmt.Printf("%s\n", string(data))
 	}
+
 	return nil
 }
 

cmd/crowdsec-cli/console.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"net/url"
 	"os"
+	"strings"
 
 	"github.com/fatih/color"
 	"github.com/go-openapi/strfmt"
@@ -17,13 +18,11 @@ import (
 	"github.com/crowdsecurity/go-cs-lib/ptr"
 	"github.com/crowdsecurity/go-cs-lib/version"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-	"github.com/crowdsecurity/crowdsec/pkg/fflag"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
 func NewConsoleCmd() *cobra.Command {
@@ -49,6 +48,7 @@ func NewConsoleCmd() *cobra.Command {
 	name := ""
 	overwrite := false
 	tags := []string{}
+	opts := []string{}
 
 	cmdEnroll := &cobra.Command{
 		Use:   "enroll [enroll-key]",
@@ -58,10 +58,12 @@ Enroll this instance to https://app.crowdsec.net
 		
 You can get your enrollment key by creating an account on https://app.crowdsec.net.
 After running this command your will need to validate the enrollment in the webapp.`,
-		Example: `cscli console enroll YOUR-ENROLL-KEY
+		Example: fmt.Sprintf(`cscli console enroll YOUR-ENROLL-KEY
 		cscli console enroll --name [instance_name] YOUR-ENROLL-KEY
 		cscli console enroll --name [instance_name] --tags [tag_1] --tags [tag_2] YOUR-ENROLL-KEY
-`,
+		cscli console enroll --enable context,manual YOUR-ENROLL-KEY
+
+		valid options are : %s,all (see 'cscli console status' for details)`, strings.Join(csconfig.CONSOLE_CONFIGS, ",")),
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -71,11 +73,12 @@ After running this command your will need to validate the enrollment in the weba
 				return fmt.Errorf("could not parse CAPI URL: %s", err)
 			}
 
-			if err := require.Hub(csConfig); err != nil {
+			hub, err := require.Hub(csConfig, nil, nil)
+			if err != nil {
 				return err
 			}
 
-			scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
+			scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
 			if err != nil {
 				return fmt.Errorf("failed to get installed scenarios: %s", err)
 			}
@@ -84,6 +87,37 @@ After running this command your will need to validate the enrollment in the weba
 				scenarios = make([]string, 0)
 			}
 
+			enable_opts := []string{csconfig.SEND_MANUAL_SCENARIOS, csconfig.SEND_TAINTED_SCENARIOS}
+			if len(opts) != 0 {
+				for _, opt := range opts {
+					valid := false
+					if opt == "all" {
+						enable_opts = csconfig.CONSOLE_CONFIGS
+						break
+					}
+					for _, available_opt := range csconfig.CONSOLE_CONFIGS {
+						if opt == available_opt {
+							valid = true
+							enable := true
+							for _, enabled_opt := range enable_opts {
+								if opt == enabled_opt {
+									enable = false
+									continue
+								}
+							}
+							if enable {
+								enable_opts = append(enable_opts, opt)
+							}
+							break
+						}
+					}
+					if !valid {
+						return fmt.Errorf("option %s doesn't exist", opt)
+
+					}
+				}
+			}
+
 			c, _ := apiclient.NewClient(&apiclient.Config{
 				MachineID:     csConfig.API.Server.OnlineClient.Credentials.Login,
 				Password:      password,
@@ -101,11 +135,13 @@ After running this command your will need to validate the enrollment in the weba
 				return nil
 			}
 
-			if err := SetConsoleOpts([]string{csconfig.SEND_MANUAL_SCENARIOS, csconfig.SEND_TAINTED_SCENARIOS}, true); err != nil {
+			if err := SetConsoleOpts(enable_opts, true); err != nil {
 				return err
 			}
 
-			log.Info("Enabled tainted&manual alerts sharing, see 'cscli console status'.")
+			for _, opt := range enable_opts {
+				log.Infof("Enabled %s : %s", opt, csconfig.CONSOLE_CONFIGS_HELP[opt])
+			}
 			log.Info("Watcher successfully enrolled. Visit https://app.crowdsec.net to accept it.")
 			log.Info("Please restart crowdsec after accepting the enrollment.")
 			return nil
@@ -114,6 +150,7 @@ After running this command your will need to validate the enrollment in the weba
 	cmdEnroll.Flags().StringVarP(&name, "name", "n", "", "Name to display in the console")
 	cmdEnroll.Flags().BoolVarP(&overwrite, "overwrite", "", false, "Force enroll the instance")
 	cmdEnroll.Flags().StringSliceVarP(&tags, "tags", "t", tags, "Tags to display in the console")
+	cmdEnroll.Flags().StringSliceVarP(&opts, "enable", "e", opts, "Enable console options")
 	cmdConsole.AddCommand(cmdEnroll)
 
 	var enableAll, disableAll bool
@@ -188,11 +225,11 @@ Disable given information push to the central API.`,
 			case "json":
 				c := csConfig.API.Server.ConsoleConfig
 				out := map[string](*bool){
-					csconfig.SEND_MANUAL_SCENARIOS: c.ShareManualDecisions,
-					csconfig.SEND_CUSTOM_SCENARIOS: c.ShareCustomScenarios,
+					csconfig.SEND_MANUAL_SCENARIOS:  c.ShareManualDecisions,
+					csconfig.SEND_CUSTOM_SCENARIOS:  c.ShareCustomScenarios,
 					csconfig.SEND_TAINTED_SCENARIOS: c.ShareTaintedScenarios,
-					csconfig.SEND_CONTEXT: c.ShareContext,
-					csconfig.CONSOLE_MANAGEMENT: c.ConsoleManagement,
+					csconfig.SEND_CONTEXT:           c.ShareContext,
+					csconfig.CONSOLE_MANAGEMENT:     c.ConsoleManagement,
 				}
 				data, err := json.MarshalIndent(out, "", "  ")
 				if err != nil {
@@ -240,7 +277,7 @@ func dumpConsoleConfig(c *csconfig.LocalApiServerCfg) error {
 		log.Debugf("Empty console_path, defaulting to %s", c.ConsoleConfigPath)
 	}
 
-	if err := os.WriteFile(c.ConsoleConfigPath, out, 0600); err != nil {
+	if err := os.WriteFile(c.ConsoleConfigPath, out, 0o600); err != nil {
 		return fmt.Errorf("while dumping console config to %s: %w", c.ConsoleConfigPath, err)
 	}
 
@@ -251,9 +288,6 @@ func SetConsoleOpts(args []string, wanted bool) error {
 	for _, arg := range args {
 		switch arg {
 		case csconfig.CONSOLE_MANAGEMENT:
-			if !fflag.PapiClient.IsEnabled() {
-				continue
-			}
 			/*for each flag check if it's already set before setting it*/
 			if csConfig.API.Server.ConsoleConfig.ConsoleManagement != nil {
 				if *csConfig.API.Server.ConsoleConfig.ConsoleManagement == wanted {
@@ -266,6 +300,7 @@ func SetConsoleOpts(args []string, wanted bool) error {
 				log.Infof("%s set to %t", csconfig.CONSOLE_MANAGEMENT, wanted)
 				csConfig.API.Server.ConsoleConfig.ConsoleManagement = ptr.Of(wanted)
 			}
+
 			if csConfig.API.Server.OnlineClient.Credentials != nil {
 				changed := false
 				if wanted && csConfig.API.Server.OnlineClient.Credentials.PapiURL == "" {
@@ -275,13 +310,16 @@ func SetConsoleOpts(args []string, wanted bool) error {
 					changed = true
 					csConfig.API.Server.OnlineClient.Credentials.PapiURL = ""
 				}
+
 				if changed {
 					fileContent, err := yaml.Marshal(csConfig.API.Server.OnlineClient.Credentials)
 					if err != nil {
 						return fmt.Errorf("cannot marshal credentials: %s", err)
 					}
+
 					log.Infof("Updating credentials file: %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
-					err = os.WriteFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, fileContent, 0600)
+
+					err = os.WriteFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, fileContent, 0o600)
 					if err != nil {
 						return fmt.Errorf("cannot write credentials file: %s", err)
 					}

cmd/crowdsec-cli/console_table.go

@@ -17,43 +17,30 @@ func cmdConsoleStatusTable(out io.Writer, csConfig csconfig.Config) {
 	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	for _, option := range csconfig.CONSOLE_CONFIGS {
+		activated := string(emoji.CrossMark)
 		switch option {
 		case csconfig.SEND_CUSTOM_SCENARIOS:
-			activated := string(emoji.CrossMark)
 			if *csConfig.API.Server.ConsoleConfig.ShareCustomScenarios {
 				activated = string(emoji.CheckMarkButton)
 			}
-
-			t.AddRow(option, activated, "Send alerts from custom scenarios to the console")
-
 		case csconfig.SEND_MANUAL_SCENARIOS:
-			activated := string(emoji.CrossMark)
 			if *csConfig.API.Server.ConsoleConfig.ShareManualDecisions {
 				activated = string(emoji.CheckMarkButton)
 			}
-
-			t.AddRow(option, activated, "Send manual decisions to the console")
-
 		case csconfig.SEND_TAINTED_SCENARIOS:
-			activated := string(emoji.CrossMark)
 			if *csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios {
 				activated = string(emoji.CheckMarkButton)
 			}
-
-			t.AddRow(option, activated, "Send alerts from tainted scenarios to the console")
 		case csconfig.SEND_CONTEXT:
-			activated := string(emoji.CrossMark)
 			if *csConfig.API.Server.ConsoleConfig.ShareContext {
 				activated = string(emoji.CheckMarkButton)
 			}
-			t.AddRow(option, activated, "Send context with alerts to the console")
 		case csconfig.CONSOLE_MANAGEMENT:
-			activated := string(emoji.CrossMark)
 			if *csConfig.API.Server.ConsoleConfig.ConsoleManagement {
 				activated = string(emoji.CheckMarkButton)
 			}
-			t.AddRow(option, activated, "Receive decisions from console")
 		}
+		t.AddRow(option, activated, csconfig.CONSOLE_CONFIGS_HELP[option])
 	}
 
 	t.Render()

cmd/crowdsec-cli/copyfile.go

@@ -18,20 +18,25 @@ func copyFileContents(src, dst string) (err error) {
 		return
 	}
 	defer in.Close()
+
 	out, err := os.Create(dst)
 	if err != nil {
 		return
 	}
+
 	defer func() {
 		cerr := out.Close()
 		if err == nil {
 			err = cerr
 		}
 	}()
+
 	if _, err = io.Copy(out, in); err != nil {
 		return
 	}
+
 	err = out.Sync()
+
 	return
 }
 
@@ -40,6 +45,7 @@ func CopyFile(sourceSymLink, destinationFile string) (err error) {
 	sourceFile, err := filepath.EvalSymlinks(sourceSymLink)
 	if err != nil {
 		log.Infof("Not a symlink : %s", err)
+
 		sourceFile = sourceSymLink
 	}
 
@@ -47,11 +53,13 @@ func CopyFile(sourceSymLink, destinationFile string) (err error) {
 	if err != nil {
 		return
 	}
+
 	if !sourceFileStat.Mode().IsRegular() {
 		// cannot copy non-regular files (e.g., directories,
 		// symlinks, devices, etc.)
 		return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String())
 	}
+
 	destinationFileStat, err := os.Stat(destinationFile)
 	if err != nil {
 		if !os.IsNotExist(err) {
@@ -65,9 +73,11 @@ func CopyFile(sourceSymLink, destinationFile string) (err error) {
 			return
 		}
 	}
+
 	if err = os.Link(sourceFile, destinationFile); err != nil {
 		err = copyFileContents(sourceFile, destinationFile)
 	}
+
 	return
 }
 

cmd/crowdsec-cli/dashboard.go

@@ -1,3 +1,5 @@
+//go:build linux
+
 package main
 
 import (
@@ -9,6 +11,7 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
+	"syscall"
 	"unicode"
 
 	"github.com/AlecAivazis/survey/v2"
@@ -37,12 +40,18 @@ var (
 
 	forceYes bool
 
-	/*informations needed to setup a random password on user's behalf*/
+	// information needed to set up a random password on user's behalf
 )
 
-func NewDashboardCmd() *cobra.Command {
+type cliDashboard struct{}
+
+func NewCLIDashboard() *cliDashboard {
+	return &cliDashboard{}
+}
+
+func (cli cliDashboard) NewCommand() *cobra.Command {
 	/* ---- UPDATE COMMAND */
-	var cmdDashboard = &cobra.Command{
+	cmd := &cobra.Command{
 		Use:   "dashboard [command]",
 		Short: "Manage your metabase dashboard container [requires local API]",
 		Long: `Install/Start/Stop/Remove a metabase container exposing dashboard and metrics.
@@ -90,19 +99,19 @@ cscli dashboard remove
 		},
 	}
 
-	cmdDashboard.AddCommand(NewDashboardSetupCmd())
-	cmdDashboard.AddCommand(NewDashboardStartCmd())
-	cmdDashboard.AddCommand(NewDashboardStopCmd())
-	cmdDashboard.AddCommand(NewDashboardShowPasswordCmd())
-	cmdDashboard.AddCommand(NewDashboardRemoveCmd())
+	cmd.AddCommand(cli.NewSetupCmd())
+	cmd.AddCommand(cli.NewStartCmd())
+	cmd.AddCommand(cli.NewStopCmd())
+	cmd.AddCommand(cli.NewShowPasswordCmd())
+	cmd.AddCommand(cli.NewRemoveCmd())
 
-	return cmdDashboard
+	return cmd
 }
 
-func NewDashboardSetupCmd() *cobra.Command {
+func (cli cliDashboard) NewSetupCmd() *cobra.Command {
 	var force bool
 
-	var cmdDashSetup = &cobra.Command{
+	cmd := &cobra.Command{
 		Use:               "setup",
 		Short:             "Setup a metabase container.",
 		Long:              `Perform a metabase docker setup, download standard dashboards, create a fresh user and start the container`,
@@ -136,6 +145,9 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
 			if err != nil {
 				return err
 			}
+			if err = chownDatabase(dockerGroup.Gid); err != nil {
+				return err
+			}
 			mb, err := metabase.SetupMetabase(csConfig.API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDbPath, dockerGroup.Gid, metabaseContainerID, metabaseImage)
 			if err != nil {
 				return err
@@ -152,20 +164,20 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
 			return nil
 		},
 	}
-	cmdDashSetup.Flags().BoolVarP(&force, "force", "f", false, "Force setup : override existing files")
-	cmdDashSetup.Flags().StringVarP(&metabaseDbPath, "dir", "d", "", "Shared directory with metabase container")
-	cmdDashSetup.Flags().StringVarP(&metabaseListenAddress, "listen", "l", metabaseListenAddress, "Listen address of container")
-	cmdDashSetup.Flags().StringVar(&metabaseImage, "metabase-image", metabaseImage, "Metabase image to use")
-	cmdDashSetup.Flags().StringVarP(&metabaseListenPort, "port", "p", metabaseListenPort, "Listen port of container")
-	cmdDashSetup.Flags().BoolVarP(&forceYes, "yes", "y", false, "force  yes")
-	//cmdDashSetup.Flags().StringVarP(&metabaseUser, "user", "u", "crowdsec@crowdsec.net", "metabase user")
-	cmdDashSetup.Flags().StringVar(&metabasePassword, "password", "", "metabase password")
+	cmd.Flags().BoolVarP(&force, "force", "f", false, "Force setup : override existing files")
+	cmd.Flags().StringVarP(&metabaseDbPath, "dir", "d", "", "Shared directory with metabase container")
+	cmd.Flags().StringVarP(&metabaseListenAddress, "listen", "l", metabaseListenAddress, "Listen address of container")
+	cmd.Flags().StringVar(&metabaseImage, "metabase-image", metabaseImage, "Metabase image to use")
+	cmd.Flags().StringVarP(&metabaseListenPort, "port", "p", metabaseListenPort, "Listen port of container")
+	cmd.Flags().BoolVarP(&forceYes, "yes", "y", false, "force  yes")
+	//cmd.Flags().StringVarP(&metabaseUser, "user", "u", "crowdsec@crowdsec.net", "metabase user")
+	cmd.Flags().StringVar(&metabasePassword, "password", "", "metabase password")
 
-	return cmdDashSetup
+	return cmd
 }
 
-func NewDashboardStartCmd() *cobra.Command {
-	var cmdDashStart = &cobra.Command{
+func (cli cliDashboard) NewStartCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "start",
 		Short:             "Start the metabase container.",
 		Long:              `Stats the metabase container using docker.`,
@@ -188,12 +200,13 @@ func NewDashboardStartCmd() *cobra.Command {
 			return nil
 		},
 	}
-	cmdDashStart.Flags().BoolVarP(&forceYes, "yes", "y", false, "force  yes")
-	return cmdDashStart
+	cmd.Flags().BoolVarP(&forceYes, "yes", "y", false, "force  yes")
+
+	return cmd
 }
 
-func NewDashboardStopCmd() *cobra.Command {
-	var cmdDashStop = &cobra.Command{
+func (cli cliDashboard) NewStopCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "stop",
 		Short:             "Stops the metabase container.",
 		Long:              `Stops the metabase container using docker.`,
@@ -206,11 +219,12 @@ func NewDashboardStopCmd() *cobra.Command {
 			return nil
 		},
 	}
-	return cmdDashStop
+
+	return cmd
 }
 
-func NewDashboardShowPasswordCmd() *cobra.Command {
-	var cmdDashShowPassword = &cobra.Command{Use: "show-password",
+func (cli cliDashboard) NewShowPasswordCmd() *cobra.Command {
+	cmd := &cobra.Command{Use: "show-password",
 		Short:             "displays password of metabase.",
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
@@ -223,13 +237,14 @@ func NewDashboardShowPasswordCmd() *cobra.Command {
 			return nil
 		},
 	}
-	return cmdDashShowPassword
+
+	return cmd
 }
 
-func NewDashboardRemoveCmd() *cobra.Command {
+func (cli cliDashboard) NewRemoveCmd() *cobra.Command {
 	var force bool
 
-	var cmdDashRemove = &cobra.Command{
+	cmd := &cobra.Command{
 		Use:               "remove",
 		Short:             "removes the metabase container.",
 		Long:              `removes the metabase container using docker.`,
@@ -294,17 +309,19 @@ cscli dashboard remove --force
 			return nil
 		},
 	}
-	cmdDashRemove.Flags().BoolVarP(&force, "force", "f", false, "Remove also the metabase image")
-	cmdDashRemove.Flags().BoolVarP(&forceYes, "yes", "y", false, "force  yes")
+	cmd.Flags().BoolVarP(&force, "force", "f", false, "Remove also the metabase image")
+	cmd.Flags().BoolVarP(&forceYes, "yes", "y", false, "force  yes")
 
-	return cmdDashRemove
+	return cmd
 }
 
 func passwordIsValid(password string) bool {
 	hasDigit := false
+
 	for _, j := range password {
 		if unicode.IsDigit(j) {
 			hasDigit = true
+
 			break
 		}
 	}
@@ -312,8 +329,8 @@ func passwordIsValid(password string) bool {
 	if !hasDigit || len(password) < 6 {
 		return false
 	}
-	return true
 
+	return true
 }
 
 func checkSystemMemory(forceYes *bool) error {
@@ -321,8 +338,10 @@ func checkSystemMemory(forceYes *bool) error {
 	if totMem >= uint64(math.Pow(2, 30)) {
 		return nil
 	}
+
 	if !*forceYes {
 		var answer bool
+
 		prompt := &survey.Confirm{
 			Message: "Metabase requires 1-2GB of RAM, your system is below this requirement continue ?",
 			Default: true,
@@ -330,12 +349,16 @@ func checkSystemMemory(forceYes *bool) error {
 		if err := survey.AskOne(prompt, &answer); err != nil {
 			return fmt.Errorf("unable to ask about RAM check: %s", err)
 		}
+
 		if !answer {
 			return fmt.Errorf("user stated no to continue")
 		}
+
 		return nil
 	}
+
 	log.Warn("Metabase requires 1-2GB of RAM, your system is below this requirement")
+
 	return nil
 }
 
@@ -343,68 +366,95 @@ func warnIfNotLoopback(addr string) {
 	if addr == "127.0.0.1" || addr == "::1" {
 		return
 	}
+
 	log.Warnf("You are potentially exposing your metabase port to the internet (addr: %s), please consider using a reverse proxy", addr)
 }
 
 func disclaimer(forceYes *bool) error {
 	if !*forceYes {
 		var answer bool
+
 		prompt := &survey.Confirm{
 			Message: "CrowdSec takes no responsibility for the security of your metabase instance. Do you accept these responsibilities ?",
 			Default: true,
 		}
+
 		if err := survey.AskOne(prompt, &answer); err != nil {
 			return fmt.Errorf("unable to ask to question: %s", err)
 		}
+
 		if !answer {
 			return fmt.Errorf("user stated no to responsibilities")
 		}
+
 		return nil
 	}
+
 	log.Warn("CrowdSec takes no responsibility for the security of your metabase instance. You used force yes, so you accept this disclaimer")
+
 	return nil
 }
 
 func checkGroups(forceYes *bool) (*user.Group, error) {
-	groupExist := false
 	dockerGroup, err := user.LookupGroup(crowdsecGroup)
 	if err == nil {
-		groupExist = true
+		return dockerGroup, nil
 	}
-	if !groupExist {
-		if !*forceYes {
-			var answer bool
-			prompt := &survey.Confirm{
-				Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup),
-				Default: true,
-			}
-			if err := survey.AskOne(prompt, &answer); err != nil {
-				return dockerGroup, fmt.Errorf("unable to ask to question: %s", err)
-			}
-			if !answer {
-				return dockerGroup, fmt.Errorf("unable to continue without creating '%s' group", crowdsecGroup)
-			}
-		}
-		groupAddCmd, err := exec.LookPath("groupadd")
-		if err != nil {
-			return dockerGroup, fmt.Errorf("unable to find 'groupadd' command, can't continue")
+
+	if !*forceYes {
+		var answer bool
+
+		prompt := &survey.Confirm{
+			Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup),
+			Default: true,
 		}
 
-		groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}}
-		if err := groupAdd.Run(); err != nil {
-			return dockerGroup, fmt.Errorf("unable to add group '%s': %s", dockerGroup, err)
+		if err := survey.AskOne(prompt, &answer); err != nil {
+			return dockerGroup, fmt.Errorf("unable to ask to question: %s", err)
 		}
-		dockerGroup, err = user.LookupGroup(crowdsecGroup)
-		if err != nil {
-			return dockerGroup, fmt.Errorf("unable to lookup '%s' group: %+v", dockerGroup, err)
+
+		if !answer {
+			return dockerGroup, fmt.Errorf("unable to continue without creating '%s' group", crowdsecGroup)
 		}
 	}
-	intID, err := strconv.Atoi(dockerGroup.Gid)
+
+	groupAddCmd, err := exec.LookPath("groupadd")
 	if err != nil {
-		return dockerGroup, fmt.Errorf("unable to convert group ID to int: %s", err)
+		return dockerGroup, fmt.Errorf("unable to find 'groupadd' command, can't continue")
 	}
-	if err := os.Chown(csConfig.DbConfig.DbPath, 0, intID); err != nil {
-		return dockerGroup, fmt.Errorf("unable to chown sqlite db file '%s': %s", csConfig.DbConfig.DbPath, err)
+
+	groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}}
+	if err := groupAdd.Run(); err != nil {
+		return dockerGroup, fmt.Errorf("unable to add group '%s': %s", dockerGroup, err)
 	}
-	return dockerGroup, nil
+
+	return user.LookupGroup(crowdsecGroup)
+}
+
+func chownDatabase(gid string) error {
+	intID, err := strconv.Atoi(gid)
+	if err != nil {
+		return fmt.Errorf("unable to convert group ID to int: %s", err)
+	}
+
+	if stat, err := os.Stat(csConfig.DbConfig.DbPath); !os.IsNotExist(err) {
+		info := stat.Sys()
+		if err := os.Chown(csConfig.DbConfig.DbPath, int(info.(*syscall.Stat_t).Uid), intID); err != nil {
+			return fmt.Errorf("unable to chown sqlite db file '%s': %s", csConfig.DbConfig.DbPath, err)
+		}
+	}
+
+	if csConfig.DbConfig.Type == "sqlite" && csConfig.DbConfig.UseWal != nil && *csConfig.DbConfig.UseWal {
+		for _, ext := range []string{"-wal", "-shm"} {
+			file := csConfig.DbConfig.DbPath + ext
+			if stat, err := os.Stat(file); !os.IsNotExist(err) {
+				info := stat.Sys()
+				if err := os.Chown(file, int(info.(*syscall.Stat_t).Uid), intID); err != nil {
+					return fmt.Errorf("unable to chown sqlite db file '%s': %s", file, err)
+				}
+			}
+		}
+	}
+
+	return nil
 }

cmd/crowdsec-cli/dashboard_unsupported.go

@@ -0,0 +1,28 @@
+//go:build !linux
+
+package main
+
+import (
+	"runtime"
+
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+)
+
+type cliDashboard struct{}
+
+func NewCLIDashboard() *cliDashboard {
+	return &cliDashboard{}
+}
+
+func (cli cliDashboard) NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               "dashboard",
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			log.Infof("Dashboard command is disabled on %s", runtime.GOOS)
+		},
+	}
+
+	return cmd
+}

cmd/crowdsec-cli/decisions.go

@@ -33,27 +33,35 @@ func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error
 	for aIdx := 0; aIdx < len(*alerts); aIdx++ {
 		alertItem := (*alerts)[aIdx]
 		newDecisions := make([]*models.Decision, 0)
+
 		for _, decisionItem := range alertItem.Decisions {
 			spamKey := fmt.Sprintf("%t:%s:%s:%s", *decisionItem.Simulated, *decisionItem.Type, *decisionItem.Scope, *decisionItem.Value)
 			if _, ok := spamLimit[spamKey]; ok {
 				skipped++
 				continue
 			}
+
 			spamLimit[spamKey] = true
+
 			newDecisions = append(newDecisions, decisionItem)
 		}
+
 		alertItem.Decisions = newDecisions
 	}
+
 	if csConfig.Cscli.Output == "raw" {
 		csvwriter := csv.NewWriter(os.Stdout)
 		header := []string{"id", "source", "ip", "reason", "action", "country", "as", "events_count", "expiration", "simulated", "alert_id"}
+
 		if printMachine {
 			header = append(header, "machine")
 		}
+
 		err := csvwriter.Write(header)
 		if err != nil {
 			return err
 		}
+
 		for _, alertItem := range *alerts {
 			for _, decisionItem := range alertItem.Decisions {
 				raw := []string{
@@ -79,6 +87,7 @@ func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error
 				}
 			}
 		}
+
 		csvwriter.Flush()
 	} else if csConfig.Cscli.Output == "json" {
 		if *alerts == nil {
@@ -99,11 +108,19 @@ func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error
 			fmt.Printf("%d duplicated entries skipped\n", skipped)
 		}
 	}
+
 	return nil
 }
 
-func NewDecisionsCmd() *cobra.Command {
-	var cmdDecisions = &cobra.Command{
+
+type cliDecisions struct {}
+
+func NewCLIDecisions() *cliDecisions {
+	return &cliDecisions{}
+}
+
+func (cli cliDecisions) NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:     "decisions [action]",
 		Short:   "Manage decisions",
 		Long:    `Add/List/Delete/Import decisions from LAPI`,
@@ -112,7 +129,7 @@ func NewDecisionsCmd() *cobra.Command {
 		/*TBD example*/
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
 			if err := csConfig.LoadAPIClient(); err != nil {
 				return fmt.Errorf("loading api client: %w", err)
 			}
@@ -135,15 +152,15 @@ func NewDecisionsCmd() *cobra.Command {
 		},
 	}
 
-	cmdDecisions.AddCommand(NewDecisionsListCmd())
-	cmdDecisions.AddCommand(NewDecisionsAddCmd())
-	cmdDecisions.AddCommand(NewDecisionsDeleteCmd())
-	cmdDecisions.AddCommand(NewDecisionsImportCmd())
+	cmd.AddCommand(cli.NewListCmd())
+	cmd.AddCommand(cli.NewAddCmd())
+	cmd.AddCommand(cli.NewDeleteCmd())
+	cmd.AddCommand(cli.NewImportCmd())
 
-	return cmdDecisions
+	return cmd
 }
 
-func NewDecisionsListCmd() *cobra.Command {
+func (cli cliDecisions) NewListCmd() *cobra.Command {
 	var filter = apiclient.AlertsListOpts{
 		ValueEquals:    new(string),
 		ScopeEquals:    new(string),
@@ -157,11 +174,13 @@ func NewDecisionsListCmd() *cobra.Command {
 		IncludeCAPI:    new(bool),
 		Limit:          new(int),
 	}
+
 	NoSimu := new(bool)
 	contained := new(bool)
+
 	var printMachine bool
 
-	var cmdDecisionsList = &cobra.Command{
+	cmd := &cobra.Command{
 		Use:   "list [options]",
 		Short: "List decisions from LAPI",
 		Example: `cscli decisions list -i 1.2.3.4
@@ -171,7 +190,7 @@ cscli decisions list -t ban
 `,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, _ []string) error {
 			var err error
 			/*take care of shorthand options*/
 			if err = manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil {
@@ -251,26 +270,26 @@ cscli decisions list -t ban
 			return nil
 		},
 	}
-	cmdDecisionsList.Flags().SortFlags = false
-	cmdDecisionsList.Flags().BoolVarP(filter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
-	cmdDecisionsList.Flags().StringVar(filter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
-	cmdDecisionsList.Flags().StringVar(filter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
-	cmdDecisionsList.Flags().StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)")
-	cmdDecisionsList.Flags().StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)")
-	cmdDecisionsList.Flags().StringVar(filter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
-	cmdDecisionsList.Flags().StringVarP(filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)")
-	cmdDecisionsList.Flags().StringVarP(filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)")
-	cmdDecisionsList.Flags().StringVarP(filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
-	cmdDecisionsList.Flags().StringVarP(filter.RangeEquals, "range", "r", "", "restrict to alerts from this source range (shorthand for --scope range --value <RANGE>)")
-	cmdDecisionsList.Flags().IntVarP(filter.Limit, "limit", "l", 100, "number of alerts to get (use 0 to remove the limit)")
-	cmdDecisionsList.Flags().BoolVar(NoSimu, "no-simu", false, "exclude decisions in simulation mode")
-	cmdDecisionsList.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that triggered decisions")
-	cmdDecisionsList.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
+	cmd.Flags().SortFlags = false
+	cmd.Flags().BoolVarP(filter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
+	cmd.Flags().StringVar(filter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
+	cmd.Flags().StringVar(filter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
+	cmd.Flags().StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)")
+	cmd.Flags().StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)")
+	cmd.Flags().StringVar(filter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
+	cmd.Flags().StringVarP(filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)")
+	cmd.Flags().StringVarP(filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)")
+	cmd.Flags().StringVarP(filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
+	cmd.Flags().StringVarP(filter.RangeEquals, "range", "r", "", "restrict to alerts from this source range (shorthand for --scope range --value <RANGE>)")
+	cmd.Flags().IntVarP(filter.Limit, "limit", "l", 100, "number of alerts to get (use 0 to remove the limit)")
+	cmd.Flags().BoolVar(NoSimu, "no-simu", false, "exclude decisions in simulation mode")
+	cmd.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that triggered decisions")
+	cmd.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
 
-	return cmdDecisionsList
+	return cmd
 }
 
-func NewDecisionsAddCmd() *cobra.Command {
+func (cli cliDecisions) NewAddCmd() *cobra.Command {
 	var (
 		addIP       string
 		addRange    string
@@ -281,7 +300,7 @@ func NewDecisionsAddCmd() *cobra.Command {
 		addType     string
 	)
 
-	var cmdDecisionsAdd = &cobra.Command{
+	cmd := &cobra.Command{
 		Use:   "add [options]",
 		Short: "Add decision to LAPI",
 		Example: `cscli decisions add --ip 1.2.3.4
@@ -292,7 +311,7 @@ cscli decisions add --scope username --value foobar
 		/*TBD : fix long and example*/
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, _ []string) error {
 			var err error
 			alerts := models.AddAlertsRequest{}
 			origin := types.CscliOrigin
@@ -318,7 +337,7 @@ cscli decisions add --scope username --value foobar
 				addScope = types.Range
 			} else if addValue == "" {
 				printHelp(cmd)
-				return fmt.Errorf("Missing arguments, a value is required (--ip, --range or --scope and --value)")
+				return fmt.Errorf("missing arguments, a value is required (--ip, --range or --scope and --value)")
 			}
 
 			if addReason == "" {
@@ -369,19 +388,19 @@ cscli decisions add --scope username --value foobar
 		},
 	}
 
-	cmdDecisionsAdd.Flags().SortFlags = false
-	cmdDecisionsAdd.Flags().StringVarP(&addIP, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
-	cmdDecisionsAdd.Flags().StringVarP(&addRange, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
-	cmdDecisionsAdd.Flags().StringVarP(&addDuration, "duration", "d", "4h", "Decision duration (ie. 1h,4h,30m)")
-	cmdDecisionsAdd.Flags().StringVarP(&addValue, "value", "v", "", "The value (ie. --scope username --value foobar)")
-	cmdDecisionsAdd.Flags().StringVar(&addScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)")
-	cmdDecisionsAdd.Flags().StringVarP(&addReason, "reason", "R", "", "Decision reason (ie. scenario-name)")
-	cmdDecisionsAdd.Flags().StringVarP(&addType, "type", "t", "ban", "Decision type (ie. ban,captcha,throttle)")
+	cmd.Flags().SortFlags = false
+	cmd.Flags().StringVarP(&addIP, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
+	cmd.Flags().StringVarP(&addRange, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
+	cmd.Flags().StringVarP(&addDuration, "duration", "d", "4h", "Decision duration (ie. 1h,4h,30m)")
+	cmd.Flags().StringVarP(&addValue, "value", "v", "", "The value (ie. --scope username --value foobar)")
+	cmd.Flags().StringVar(&addScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)")
+	cmd.Flags().StringVarP(&addReason, "reason", "R", "", "Decision reason (ie. scenario-name)")
+	cmd.Flags().StringVarP(&addType, "type", "t", "ban", "Decision type (ie. ban,captcha,throttle)")
 
-	return cmdDecisionsAdd
+	return cmd
 }
 
-func NewDecisionsDeleteCmd() *cobra.Command {
+func (cli cliDecisions) NewDeleteCmd() *cobra.Command {
 	var delFilter = apiclient.DecisionsDeleteOpts{
 		ScopeEquals:    new(string),
 		ValueEquals:    new(string),
@@ -391,11 +410,14 @@ func NewDecisionsDeleteCmd() *cobra.Command {
 		ScenarioEquals: new(string),
 		OriginEquals:   new(string),
 	}
-	var delDecisionId string
+
+	var delDecisionID string
+
 	var delDecisionAll bool
+
 	contained := new(bool)
 
-	var cmdDecisionsDelete = &cobra.Command{
+	cmd := &cobra.Command{
 		Use:               "delete [options]",
 		Short:             "Delete decisions",
 		DisableAutoGenTag: true,
@@ -406,21 +428,21 @@ cscli decisions delete --id 42
 cscli decisions delete --type captcha
 `,
 		/*TBD : refaire le Long/Example*/
-		PreRunE: func(cmd *cobra.Command, args []string) error {
+		PreRunE: func(cmd *cobra.Command, _ []string) error {
 			if delDecisionAll {
 				return nil
 			}
 			if *delFilter.ScopeEquals == "" && *delFilter.ValueEquals == "" &&
 				*delFilter.TypeEquals == "" && *delFilter.IPEquals == "" &&
 				*delFilter.RangeEquals == "" && *delFilter.ScenarioEquals == "" &&
-				*delFilter.OriginEquals == "" && delDecisionId == "" {
+				*delFilter.OriginEquals == "" && delDecisionID == "" {
 				cmd.Usage()
 				return fmt.Errorf("at least one filter or --all must be specified")
 			}
 
 			return nil
 		},
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(_ *cobra.Command, _ []string) error {
 			var err error
 			var decisions *models.DeleteDecisionResponse
 
@@ -453,18 +475,18 @@ cscli decisions delete --type captcha
 				delFilter.Contains = new(bool)
 			}
 
-			if delDecisionId == "" {
+			if delDecisionID == "" {
 				decisions, _, err = Client.Decisions.Delete(context.Background(), delFilter)
 				if err != nil {
-					return fmt.Errorf("Unable to delete decisions: %v", err)
+					return fmt.Errorf("unable to delete decisions: %v", err)
 				}
 			} else {
-				if _, err = strconv.Atoi(delDecisionId); err != nil {
-					return fmt.Errorf("id '%s' is not an integer: %v", delDecisionId, err)
+				if _, err = strconv.Atoi(delDecisionID); err != nil {
+					return fmt.Errorf("id '%s' is not an integer: %v", delDecisionID, err)
 				}
-				decisions, _, err = Client.Decisions.DeleteOne(context.Background(), delDecisionId)
+				decisions, _, err = Client.Decisions.DeleteOne(context.Background(), delDecisionID)
 				if err != nil {
-					return fmt.Errorf("Unable to delete decision: %v", err)
+					return fmt.Errorf("unable to delete decision: %v", err)
 				}
 			}
 			log.Infof("%s decision(s) deleted", decisions.NbDeleted)
@@ -472,17 +494,17 @@ cscli decisions delete --type captcha
 		},
 	}
 
-	cmdDecisionsDelete.Flags().SortFlags = false
-	cmdDecisionsDelete.Flags().StringVarP(delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
-	cmdDecisionsDelete.Flags().StringVarP(delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
-	cmdDecisionsDelete.Flags().StringVarP(delFilter.TypeEquals, "type", "t", "", "the decision type (ie. ban,captcha)")
-	cmdDecisionsDelete.Flags().StringVarP(delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
-	cmdDecisionsDelete.Flags().StringVarP(delFilter.ScenarioEquals, "scenario", "s", "", "the scenario name (ie. crowdsecurity/ssh-bf)")
-	cmdDecisionsDelete.Flags().StringVar(delFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
+	cmd.Flags().SortFlags = false
+	cmd.Flags().StringVarP(delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
+	cmd.Flags().StringVarP(delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
+	cmd.Flags().StringVarP(delFilter.TypeEquals, "type", "t", "", "the decision type (ie. ban,captcha)")
+	cmd.Flags().StringVarP(delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
+	cmd.Flags().StringVarP(delFilter.ScenarioEquals, "scenario", "s", "", "the scenario name (ie. crowdsecurity/ssh-bf)")
+	cmd.Flags().StringVar(delFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
 
-	cmdDecisionsDelete.Flags().StringVar(&delDecisionId, "id", "", "decision id")
-	cmdDecisionsDelete.Flags().BoolVar(&delDecisionAll, "all", false, "delete all decisions")
-	cmdDecisionsDelete.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
+	cmd.Flags().StringVar(&delDecisionID, "id", "", "decision id")
+	cmd.Flags().BoolVar(&delDecisionAll, "all", false, "delete all decisions")
+	cmd.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
 
-	return cmdDecisionsDelete
+	return cmd
 }

cmd/crowdsec-cli/decisions_import.go

@@ -37,21 +37,25 @@ func parseDecisionList(content []byte, format string) ([]decisionRaw, error) {
 	switch format {
 	case "values":
 		log.Infof("Parsing values")
+
 		scanner := bufio.NewScanner(bytes.NewReader(content))
 		for scanner.Scan() {
 			value := strings.TrimSpace(scanner.Text())
 			ret = append(ret, decisionRaw{Value: value})
 		}
+
 		if err := scanner.Err(); err != nil {
 			return nil, fmt.Errorf("unable to parse values: '%s'", err)
 		}
 	case "json":
 		log.Infof("Parsing json")
+
 		if err := json.Unmarshal(content, &ret); err != nil {
 			return nil, err
 		}
 	case "csv":
 		log.Infof("Parsing csv")
+
 		if err := csvutil.Unmarshal(content, &ret); err != nil {
 			return nil, fmt.Errorf("unable to parse csv: '%s'", err)
 		}
@@ -63,7 +67,7 @@ func parseDecisionList(content []byte, format string) ([]decisionRaw, error) {
 }
 
 
-func runDecisionsImport(cmd *cobra.Command, args []string) error  {
+func (cli cliDecisions) runImport(cmd *cobra.Command, args []string) error  {
 	flags := cmd.Flags()
 
 	input, err := flags.GetString("input")
@@ -75,6 +79,7 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error  {
 	if err != nil {
 		return err
 	}
+
 	if defaultDuration == "" {
 		return fmt.Errorf("--duration cannot be empty")
 	}
@@ -83,6 +88,7 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error  {
 	if err != nil {
 		return err
 	}
+
 	if defaultScope == "" {
 		return fmt.Errorf("--scope cannot be empty")
 	}
@@ -91,6 +97,7 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error  {
 	if err != nil {
 		return err
 	}
+
 	if defaultReason == "" {
 		return fmt.Errorf("--reason cannot be empty")
 	}
@@ -99,6 +106,7 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error  {
 	if err != nil {
 		return err
 	}
+
 	if defaultType == "" {
 		return fmt.Errorf("--type cannot be empty")
 	}
@@ -152,6 +160,7 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error  {
 	}
 
 	decisions := make([]*models.Decision, len(decisionsListRaw))
+
 	for i, d := range decisionsListRaw {
 		if d.Value == "" {
 			return fmt.Errorf("item %d: missing 'value'", i)
@@ -222,17 +231,19 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error  {
 	}
 
 	log.Infof("Imported %d decisions", len(decisions))
+
 	return nil
 }
 
 
-func NewDecisionsImportCmd() *cobra.Command {
-	var cmdDecisionsImport = &cobra.Command{
+func (cli cliDecisions) NewImportCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:   "import [options]",
 		Short: "Import decisions from a file or pipe",
 		Long: "expected format:\n" +
 			"csv  : any of duration,reason,scope,type,value, with a header line\n" +
-			`json : {"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`,
+			"json :" + "`{" + `"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"` + "}`",
+		Args:	 cobra.NoArgs,
 		DisableAutoGenTag: true,
 		Example: `decisions.csv:
 duration,scope,value
@@ -250,10 +261,10 @@ Raw values, standard input:
 
 $ echo "1.2.3.4" | cscli decisions import -i - --format values
 `,
-		RunE: runDecisionsImport,
+		RunE: cli.runImport,
 	}
 
-	flags := cmdDecisionsImport.Flags()
+	flags := cmd.Flags()
 	flags.SortFlags = false
 	flags.StringP("input", "i", "", "Input file")
 	flags.StringP("duration", "d", "4h", "Decision duration: 1h,4h,30m")
@@ -263,7 +274,7 @@ $ echo "1.2.3.4" | cscli decisions import -i - --format values
 	flags.Int("batch", 0, "Split import in batches of N decisions")
 	flags.String("format", "", "Input format: 'json', 'csv' or 'values' (each line is a value, no headers)")
 
-	cmdDecisionsImport.MarkFlagRequired("input")
+	cmd.MarkFlagRequired("input")
 
-	return cmdDecisionsImport
+	return cmd
 }

cmd/crowdsec-cli/doc.go

@@ -0,0 +1,49 @@
+package main
+
+import (
+	"fmt"
+	"path/filepath"
+	"strings"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/cobra/doc"
+)
+
+type cliDoc struct{}
+
+func NewCLIDoc() *cliDoc {
+	return &cliDoc{}
+}
+
+func (cli cliDoc) NewCommand(rootCmd *cobra.Command) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               "doc",
+		Short:             "Generate the documentation in `./doc/`. Directory must exist.",
+		Args:              cobra.ExactArgs(0),
+		Hidden:            true,
+		DisableAutoGenTag: true,
+		RunE: func(_ *cobra.Command, _ []string) error {
+			if err := doc.GenMarkdownTreeCustom(rootCmd, "./doc/", cli.filePrepender, cli.linkHandler); err != nil {
+				return fmt.Errorf("failed to generate cobra doc: %s", err)
+			}
+			return nil
+		},
+	}
+
+	return cmd
+}
+
+func (cli cliDoc) filePrepender(filename string) string {
+	const header = `---
+id: %s
+title: %s
+---
+`
+	name := filepath.Base(filename)
+	base := strings.TrimSuffix(name, filepath.Ext(name))
+	return fmt.Sprintf(header, base, strings.ReplaceAll(base, "_", " "))
+}
+
+func (cli cliDoc) linkHandler(name string) string {
+	return fmt.Sprintf("/cscli/%s", name)
+}

cmd/crowdsec-cli/explain.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"bufio"
+	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -11,6 +12,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
+	"github.com/crowdsecurity/crowdsec/pkg/dumps"
 	"github.com/crowdsecurity/crowdsec/pkg/hubtest"
 )
 
@@ -21,14 +23,99 @@ func GetLineCountForFile(filepath string) (int, error) {
 	}
 	defer f.Close()
 	lc := 0
-	fs := bufio.NewScanner(f)
-	for fs.Scan() {
-		lc++
+	fs := bufio.NewReader(f)
+	for {
+		input, err := fs.ReadBytes('\n')
+		if len(input) > 1 {
+			lc++
+		}
+		if err != nil && err == io.EOF {
+			break
+		}
 	}
 	return lc, nil
 }
 
-func runExplain(cmd *cobra.Command, args []string) error {
+type cliExplain struct{}
+
+func NewCLIExplain() *cliExplain {
+	return &cliExplain{}
+}
+
+func (cli cliExplain) NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "explain",
+		Short: "Explain log pipeline",
+		Long: `
+Explain log pipeline 
+		`,
+		Example: `
+cscli explain --file ./myfile.log --type nginx 
+cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog
+cscli explain --dsn "file://myfile.log" --type nginx
+tail -n 5 myfile.log | cscli explain --type nginx -f -
+		`,
+		Args:              cobra.ExactArgs(0),
+		DisableAutoGenTag: true,
+		RunE:              cli.run,
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			flags := cmd.Flags()
+
+			logFile, err := flags.GetString("file")
+			if err != nil {
+				return err
+			}
+
+			dsn, err := flags.GetString("dsn")
+			if err != nil {
+				return err
+			}
+
+			logLine, err := flags.GetString("log")
+			if err != nil {
+				return err
+			}
+
+			logType, err := flags.GetString("type")
+			if err != nil {
+				return err
+			}
+
+			if logLine == "" && logFile == "" && dsn == "" {
+				printHelp(cmd)
+				fmt.Println()
+				return fmt.Errorf("please provide --log, --file or --dsn flag")
+			}
+			if logType == "" {
+				printHelp(cmd)
+				fmt.Println()
+				return fmt.Errorf("please provide --type flag")
+			}
+			fileInfo, _ := os.Stdin.Stat()
+			if logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) {
+				return fmt.Errorf("the option -f - is intended to work with pipes")
+			}
+			return nil
+		},
+	}
+
+	flags := cmd.Flags()
+
+	flags.StringP("file", "f", "", "Log file to test")
+	flags.StringP("dsn", "d", "", "DSN to test")
+	flags.StringP("log", "l", "", "Log line to test")
+	flags.StringP("type", "t", "", "Type of the acquisition to test")
+	flags.String("labels", "", "Additional labels to add to the acquisition format (key:value,key2:value2)")
+	flags.BoolP("verbose", "v", false, "Display individual changes")
+	flags.Bool("failures", false, "Only show failed lines")
+	flags.Bool("only-successful-parsers", false, "Only show successful parsers")
+	flags.String("crowdsec", "crowdsec", "Path to crowdsec")
+	flags.Bool("no-clean", false, "Don't clean runtime environment after tests")
+
+	return cmd
+}
+
+func (cli cliExplain) run(cmd *cobra.Command, args []string) error {
 	flags := cmd.Flags()
 
 	logFile, err := flags.GetString("file")
@@ -51,13 +138,18 @@ func runExplain(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	opts := hubtest.DumpOpts{}
+	opts := dumps.DumpOpts{}
 
 	opts.Details, err = flags.GetBool("verbose")
 	if err != nil {
 		return err
 	}
 
+	no_clean, err := flags.GetBool("no-clean")
+	if err != nil {
+		return err
+	}
+
 	opts.SkipOk, err = flags.GetBool("failures")
 	if err != nil {
 		return err
@@ -79,19 +171,6 @@ func runExplain(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	fileInfo, _ := os.Stdin.Stat()
-
-	if logType == "" || (logLine == "" && logFile == "" && dsn == "") {
-		printHelp(cmd)
-		fmt.Println()
-		fmt.Printf("Please provide --type flag\n")
-		os.Exit(1)
-	}
-
-	if logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) {
-		return fmt.Errorf("the option -f - is intended to work with pipes")
-	}
-
 	var f *os.File
 
 	// using empty string fallback to /tmp
@@ -99,6 +178,16 @@ func runExplain(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %s", err)
 	}
+	defer func() {
+		if no_clean {
+			return
+		}
+		if _, err := os.Stat(dir); !os.IsNotExist(err) {
+			if err := os.RemoveAll(dir); err != nil {
+				log.Errorf("unable to delete temporary directory '%s': %s", dir, err)
+			}
+		}
+	}()
 	tmpFile := ""
 	// we create a  temporary log file if a log line/stdin has been provided
 	if logLine != "" || logFile == "-" {
@@ -118,16 +207,18 @@ func runExplain(cmd *cobra.Command, args []string) error {
 			errCount := 0
 			for {
 				input, err := reader.ReadBytes('\n')
-				if err != nil && err == io.EOF {
+				if err != nil && errors.Is(err, io.EOF) {
 					break
 				}
-				_, err = f.Write(input)
-				if err != nil {
+				if len(input) > 1 {
+					_, err = f.Write(input)
+				}
+				if err != nil || len(input) <= 1 {
 					errCount++
 				}
 			}
 			if errCount > 0 {
-				log.Warnf("Failed to write %d lines to tmp file", errCount)
+				log.Warnf("Failed to write %d lines to %s", errCount, tmpFile)
 			}
 		}
 		f.Close()
@@ -145,8 +236,12 @@ func runExplain(cmd *cobra.Command, args []string) error {
 		if err != nil {
 			return err
 		}
+		log.Debugf("file %s has %d lines", absolutePath, lineCount)
+		if lineCount == 0 {
+			return fmt.Errorf("the log file is empty: %s", absolutePath)
+		}
 		if lineCount > 100 {
-			log.Warnf("The log file contains %d lines. This may take a lot of resources.", lineCount)
+			log.Warnf("%s contains %d lines. This may take a lot of resources.", absolutePath, lineCount)
 		}
 	}
 
@@ -166,63 +261,20 @@ func runExplain(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("fail to run crowdsec for test: %v", err)
 	}
 
-	// rm the temporary log file if only a log line/stdin was provided
-	if tmpFile != "" {
-		if err := os.Remove(tmpFile); err != nil {
-			return fmt.Errorf("unable to remove tmp log file '%s': %+v", tmpFile, err)
-		}
-	}
 	parserDumpFile := filepath.Join(dir, hubtest.ParserResultFileName)
 	bucketStateDumpFile := filepath.Join(dir, hubtest.BucketPourResultFileName)
 
-	parserDump, err := hubtest.LoadParserDump(parserDumpFile)
+	parserDump, err := dumps.LoadParserDump(parserDumpFile)
 	if err != nil {
 		return fmt.Errorf("unable to load parser dump result: %s", err)
 	}
 
-	bucketStateDump, err := hubtest.LoadBucketPourDump(bucketStateDumpFile)
+	bucketStateDump, err := dumps.LoadBucketPourDump(bucketStateDumpFile)
 	if err != nil {
 		return fmt.Errorf("unable to load bucket dump result: %s", err)
 	}
 
-	hubtest.DumpTree(*parserDump, *bucketStateDump, opts)
-
-	if err := os.RemoveAll(dir); err != nil {
-		return fmt.Errorf("unable to delete temporary directory '%s': %s", dir, err)
-	}
+	dumps.DumpTree(*parserDump, *bucketStateDump, opts)
 
 	return nil
 }
-
-func NewExplainCmd() *cobra.Command {
-	cmdExplain := &cobra.Command{
-		Use:   "explain",
-		Short: "Explain log pipeline",
-		Long: `
-Explain log pipeline 
-		`,
-		Example: `
-cscli explain --file ./myfile.log --type nginx 
-cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog
-cscli explain --dsn "file://myfile.log" --type nginx
-tail -n 5 myfile.log | cscli explain --type nginx -f -
-		`,
-		Args:              cobra.ExactArgs(0),
-		DisableAutoGenTag: true,
-		RunE:              runExplain,
-	}
-
-	flags := cmdExplain.Flags()
-
-	flags.StringP("file", "f", "", "Log file to test")
-	flags.StringP("dsn", "d", "", "DSN to test")
-	flags.StringP("log", "l", "", "Log line to test")
-	flags.StringP("type", "t", "", "Type of the acquisition to test")
-	flags.String("labels", "", "Additional labels to add to the acquisition format (key:value,key2:value2)")
-	flags.BoolP("verbose", "v", false, "Display individual changes")
-	flags.Bool("failures", false, "Only show failed lines")
-	flags.Bool("only-successful-parsers", false, "Only show successful parsers")
-	flags.String("crowdsec", "crowdsec", "Path to crowdsec")
-
-	return cmdExplain
-}

cmd/crowdsec-cli/hub.go

@@ -1,160 +1,226 @@
 package main
 
 import (
-	"errors"
+	"encoding/json"
 	"fmt"
 
 	"github.com/fatih/color"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v3"
 
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-func NewHubCmd() *cobra.Command {
-	var cmdHub = &cobra.Command{
+type cliHub struct{}
+
+func NewCLIHub() *cliHub {
+	return &cliHub{}
+}
+
+func (cli cliHub) NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:   "hub [action]",
-		Short: "Manage Hub",
-		Long: `
-Hub management
+		Short: "Manage hub index",
+		Long: `Hub management
 
 List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net).
-The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.
-		`,
-		Example: `
-cscli hub list   # List all installed configurations
-cscli hub update # Download list of available configurations from the hub
-		`,
+The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.`,
+		Example: `cscli hub list
+cscli hub update
+cscli hub upgrade`,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if csConfig.Cscli == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			return nil
-		},
 	}
-	cmdHub.PersistentFlags().StringVarP(&cwhub.HubBranch, "branch", "b", "", "Use given branch from hub")
 
-	cmdHub.AddCommand(NewHubListCmd())
-	cmdHub.AddCommand(NewHubUpdateCmd())
-	cmdHub.AddCommand(NewHubUpgradeCmd())
+	cmd.AddCommand(cli.NewListCmd())
+	cmd.AddCommand(cli.NewUpdateCmd())
+	cmd.AddCommand(cli.NewUpgradeCmd())
+	cmd.AddCommand(cli.NewTypesCmd())
 
-	return cmdHub
+	return cmd
 }
 
-func NewHubListCmd() *cobra.Command {
-	var cmdHubList = &cobra.Command{
+func (cli cliHub) list(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	hub, err := require.Hub(csConfig, nil, log.StandardLogger())
+	if err != nil {
+		return err
+	}
+
+	for _, v := range hub.Warnings {
+		log.Info(v)
+	}
+
+	for _, line := range hub.ItemStats() {
+		log.Info(line)
+	}
+
+	items := make(map[string][]*cwhub.Item)
+
+	for _, itemType := range cwhub.ItemTypes {
+		items[itemType], err = selectItems(hub, itemType, nil, !all)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = listItems(color.Output, cwhub.ItemTypes, items, true)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (cli cliHub) NewListCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "list [-a]",
-		Short:             "List installed configs",
+		Short:             "List all installed configurations",
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := require.Hub(csConfig); err != nil {
-				return err
-			}
-
-			// use LocalSync to get warnings about tainted / outdated items
-			warn, _ := cwhub.LocalSync(csConfig.Hub)
-			for _, v := range warn {
-				log.Info(v)
-			}
-			cwhub.DisplaySummary()
-			ListItems(color.Output, []string{
-				cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW,
-			}, args, true, false, all)
-
-			return nil
-		},
+		RunE:              cli.list,
 	}
-	cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
 
-	return cmdHubList
+	flags := cmd.Flags()
+	flags.BoolP("all", "a", false, "List disabled items as well")
+
+	return cmd
 }
 
-func NewHubUpdateCmd() *cobra.Command {
-	var cmdHubUpdate = &cobra.Command{
+func (cli cliHub) update(cmd *cobra.Command, args []string) error {
+	local := csConfig.Hub
+	remote := require.RemoteHub(csConfig)
+
+	// don't use require.Hub because if there is no index file, it would fail
+	hub, err := cwhub.NewHub(local, remote, true, log.StandardLogger())
+	if err != nil {
+		return fmt.Errorf("failed to update hub: %w", err)
+	}
+
+	for _, v := range hub.Warnings {
+		log.Info(v)
+	}
+
+	return nil
+}
+
+func (cli cliHub) NewUpdateCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:   "update",
-		Short: "Fetch available configs from hub",
+		Short: "Download the latest index (catalog of available configurations)",
 		Long: `
-Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs.
+Fetches the .index.json file from the hub, containing the list of available configs.
 `,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if csConfig.Cscli == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			cwhub.SetHubBranch()
-
-			return nil
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := csConfig.LoadHub(); err != nil {
-				return err
-			}
-			if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
-				if !errors.Is(err, cwhub.ErrIndexNotFound) {
-					return fmt.Errorf("failed to get Hub index : %w", err)
-				}
-				log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch)
-				cwhub.HubBranch = "master"
-				if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
-					return fmt.Errorf("failed to get Hub index after retry: %w", err)
-				}
-			}
-			// use LocalSync to get warnings about tainted / outdated items
-			warn, _ := cwhub.LocalSync(csConfig.Hub)
-			for _, v := range warn {
-				log.Info(v)
-			}
-
-			return nil
-		},
+		RunE:              cli.update,
 	}
 
-	return cmdHubUpdate
+	return cmd
 }
 
-func NewHubUpgradeCmd() *cobra.Command {
-	var cmdHubUpgrade = &cobra.Command{
+func (cli cliHub) upgrade(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), log.StandardLogger())
+	if err != nil {
+		return err
+	}
+
+	for _, itemType := range cwhub.ItemTypes {
+		items, err := hub.GetInstalledItems(itemType)
+		if err != nil {
+			return err
+		}
+
+		updated := 0
+
+		log.Infof("Upgrading %s", itemType)
+
+		for _, item := range items {
+			didUpdate, err := item.Upgrade(force)
+			if err != nil {
+				return err
+			}
+
+			if didUpdate {
+				updated++
+			}
+		}
+
+		log.Infof("Upgraded %d %s", updated, itemType)
+	}
+
+	return nil
+}
+
+func (cli cliHub) NewUpgradeCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:   "upgrade",
-		Short: "Upgrade all configs installed from hub",
+		Short: "Upgrade all configurations to their latest version",
 		Long: `
 Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available.
 `,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if csConfig.Cscli == nil {
-				return fmt.Errorf("you must configure cli before interacting with hub")
-			}
-
-			cwhub.SetHubBranch()
-
-			return nil
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := require.Hub(csConfig); err != nil {
-				return err
-			}
-
-			log.Infof("Upgrading collections")
-			cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
-			log.Infof("Upgrading parsers")
-			cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
-			log.Infof("Upgrading scenarios")
-			cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
-			log.Infof("Upgrading postoverflows")
-			cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
-
-			return nil
-		},
+		RunE:              cli.upgrade,
 	}
-	cmdHubUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
 
-	return cmdHubUpgrade
+	flags := cmd.Flags()
+	flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
+
+	return cmd
+}
+
+func (cli cliHub) types(cmd *cobra.Command, args []string) error {
+	switch csConfig.Cscli.Output {
+	case "human":
+		s, err := yaml.Marshal(cwhub.ItemTypes)
+		if err != nil {
+			return err
+		}
+
+		fmt.Print(string(s))
+	case "json":
+		jsonStr, err := json.Marshal(cwhub.ItemTypes)
+		if err != nil {
+			return err
+		}
+
+		fmt.Println(string(jsonStr))
+	case "raw":
+		for _, itemType := range cwhub.ItemTypes {
+			fmt.Println(itemType)
+		}
+	}
+
+	return nil
+}
+
+func (cli cliHub) NewTypesCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "types",
+		Short: "List supported item types",
+		Long: `
+List the types of supported hub items.
+`,
+		Args:              cobra.ExactArgs(0),
+		DisableAutoGenTag: true,
+		RunE:              cli.types,
+	}
+
+	return cmd
 }

cmd/crowdsec-cli/hubappsec.go

@@ -0,0 +1,121 @@
+package main
+
+import (
+	"fmt"
+	"os"
+
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
+	"gopkg.in/yaml.v3"
+
+	"github.com/crowdsecurity/crowdsec/pkg/appsec"
+	"github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func NewCLIAppsecConfig() *cliItem {
+	return &cliItem{
+		name:      cwhub.APPSEC_CONFIGS,
+		singular:  "appsec-config",
+		oneOrMore: "appsec-config(s)",
+		help: cliHelp{
+			example: `cscli appsec-configs list -a
+cscli appsec-configs install crowdsecurity/vpatch
+cscli appsec-configs inspect crowdsecurity/vpatch
+cscli appsec-configs upgrade crowdsecurity/vpatch
+cscli appsec-configs remove crowdsecurity/vpatch
+`,
+		},
+		installHelp: cliHelp{
+			example: `cscli appsec-configs install crowdsecurity/vpatch`,
+		},
+		removeHelp: cliHelp{
+			example: `cscli appsec-configs remove crowdsecurity/vpatch`,
+		},
+		upgradeHelp: cliHelp{
+			example: `cscli appsec-configs upgrade crowdsecurity/vpatch`,
+		},
+		inspectHelp: cliHelp{
+			example: `cscli appsec-configs inspect crowdsecurity/vpatch`,
+		},
+		listHelp: cliHelp{
+			example: `cscli appsec-configs list
+cscli appsec-configs list -a
+cscli appsec-configs list crowdsecurity/vpatch`,
+		},
+	}
+}
+
+func NewCLIAppsecRule() *cliItem {
+	inspectDetail := func(item *cwhub.Item) error {
+		// Only show the converted rules in human mode
+		if csConfig.Cscli.Output != "human" {
+			return nil
+		}
+
+		appsecRule := appsec.AppsecCollectionConfig{}
+
+		yamlContent, err := os.ReadFile(item.State.LocalPath)
+		if err != nil {
+			return fmt.Errorf("unable to read file %s : %s", item.State.LocalPath, err)
+		}
+
+		if err := yaml.Unmarshal(yamlContent, &appsecRule); err != nil {
+			return fmt.Errorf("unable to unmarshal yaml file %s : %s", item.State.LocalPath, err)
+		}
+
+		for _, ruleType := range appsec_rule.SupportedTypes() {
+			fmt.Printf("\n%s format:\n", cases.Title(language.Und, cases.NoLower).String(ruleType))
+
+			for _, rule := range appsecRule.Rules {
+				convertedRule, _, err := rule.Convert(ruleType, appsecRule.Name)
+				if err != nil {
+					return fmt.Errorf("unable to convert rule %s : %s", rule.Name, err)
+				}
+
+				fmt.Println(convertedRule)
+			}
+
+			switch ruleType { //nolint:gocritic
+			case appsec_rule.ModsecurityRuleType:
+				for _, rule := range appsecRule.SecLangRules {
+					fmt.Println(rule)
+				}
+			}
+		}
+
+		return nil
+	}
+
+	return &cliItem{
+		name:      "appsec-rules",
+		singular:  "appsec-rule",
+		oneOrMore: "appsec-rule(s)",
+		help: cliHelp{
+			example: `cscli appsec-rules list -a
+cscli appsec-rules install crowdsecurity/crs
+cscli appsec-rules inspect crowdsecurity/crs
+cscli appsec-rules upgrade crowdsecurity/crs
+cscli appsec-rules remove crowdsecurity/crs
+`,
+		},
+		installHelp: cliHelp{
+			example: `cscli appsec-rules install crowdsecurity/crs`,
+		},
+		removeHelp: cliHelp{
+			example: `cscli appsec-rules remove crowdsecurity/crs`,
+		},
+		upgradeHelp: cliHelp{
+			example: `cscli appsec-rules upgrade crowdsecurity/crs`,
+		},
+		inspectHelp: cliHelp{
+			example: `cscli appsec-rules inspect crowdsecurity/crs`,
+		},
+		inspectDetail: inspectDetail,
+		listHelp: cliHelp{
+			example: `cscli appsec-rules list
+cscli appsec-rules list -a
+cscli appsec-rules list crowdsecurity/crs`,
+		},
+	}
+}

cmd/crowdsec-cli/hubcollection.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func NewCLICollection() *cliItem {
+	return &cliItem{
+		name:      cwhub.COLLECTIONS,
+		singular:  "collection",
+		oneOrMore: "collection(s)",
+		help: cliHelp{
+			example: `cscli collections list -a
+cscli collections install crowdsecurity/http-cve crowdsecurity/iptables
+cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables
+cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables
+cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
+`,
+		},
+		installHelp: cliHelp{
+			example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		removeHelp: cliHelp{
+			example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		upgradeHelp: cliHelp{
+			example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		inspectHelp: cliHelp{
+			example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`,
+		},
+		listHelp: cliHelp{
+			example: `cscli collections list
+cscli collections list -a
+cscli collections list crowdsecurity/http-cve crowdsecurity/iptables
+
+List only enabled collections unless "-a" or names are specified.`,
+		},
+	}
+}

cmd/crowdsec-cli/hubcontext.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func NewCLIContext() *cliItem {
+	return &cliItem{
+		name:      cwhub.CONTEXTS,
+		singular:  "context",
+		oneOrMore: "context(s)",
+		help: cliHelp{
+			example: `cscli contexts list -a
+cscli contexts install crowdsecurity/yyy crowdsecurity/zzz
+cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz
+cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz
+cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz
+`,
+		},
+		installHelp: cliHelp{
+			example: `cscli contexts install crowdsecurity/yyy crowdsecurity/zzz`,
+		},
+		removeHelp: cliHelp{
+			example: `cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz`,
+		},
+		upgradeHelp: cliHelp{
+			example: `cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz`,
+		},
+		inspectHelp: cliHelp{
+			example: `cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz`,
+		},
+		listHelp: cliHelp{
+			example: `cscli contexts list
+cscli contexts list -a
+cscli contexts list crowdsecurity/yyy crowdsecurity/zzz
+
+List only enabled contexts unless "-a" or names are specified.`,
+		},
+	}
+}

cmd/crowdsec-cli/hubparser.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func NewCLIParser() *cliItem {
+	return &cliItem{
+		name:      cwhub.PARSERS,
+		singular:  "parser",
+		oneOrMore: "parser(s)",
+		help: cliHelp{
+			example: `cscli parsers list -a
+cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+`,
+		},
+		installHelp: cliHelp{
+			example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
+		},
+		removeHelp: cliHelp{
+			example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
+		},
+		upgradeHelp: cliHelp{
+			example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
+		},
+		inspectHelp: cliHelp{
+			example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`,
+		},
+		listHelp: cliHelp{
+			example: `cscli parsers list
+cscli parsers list -a
+cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs
+
+List only enabled parsers unless "-a" or names are specified.`,
+		},
+	}
+}

cmd/crowdsec-cli/hubpostoverflow.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func NewCLIPostOverflow() *cliItem {
+	return &cliItem{
+		name:      cwhub.POSTOVERFLOWS,
+		singular:  "postoverflow",
+		oneOrMore: "postoverflow(s)",
+		help: cliHelp{
+			example: `cscli postoverflows list -a
+cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns
+cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns
+cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns
+cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
+`,
+		},
+		installHelp: cliHelp{
+			example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		removeHelp: cliHelp{
+			example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		upgradeHelp: cliHelp{
+			example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		inspectHelp: cliHelp{
+			example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
+		},
+		listHelp: cliHelp{
+			example: `cscli postoverflows list
+cscli postoverflows list -a
+cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns
+
+List only enabled postoverflows unless "-a" or names are specified.`,
+		},
+	}
+}

cmd/crowdsec-cli/hubscenario.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func NewCLIScenario() *cliItem {
+	return &cliItem{
+		name:      cwhub.SCENARIOS,
+		singular:  "scenario",
+		oneOrMore: "scenario(s)",
+		help: cliHelp{
+			example: `cscli scenarios list -a
+cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing
+cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing
+cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing
+cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
+`,
+		},
+		installHelp: cliHelp{
+			example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		removeHelp: cliHelp{
+			example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		upgradeHelp: cliHelp{
+			example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		inspectHelp: cliHelp{
+			example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`,
+		},
+		listHelp: cliHelp{
+			example: `cscli scenarios list
+cscli scenarios list -a
+cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing
+
+List only enabled scenarios unless "-a" or names are specified.`,
+		},
+	}
+}

cmd/crowdsec-cli/hubtest.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"text/template"
 
 	"github.com/AlecAivazis/survey/v2"
 	"github.com/enescakir/emoji"
@@ -15,52 +16,70 @@ import (
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
 
+	"github.com/crowdsecurity/crowdsec/pkg/dumps"
 	"github.com/crowdsecurity/crowdsec/pkg/hubtest"
 )
 
-var (
-	HubTest hubtest.HubTest
-)
+var HubTest hubtest.HubTest
+var HubAppsecTests hubtest.HubTest
+var hubPtr *hubtest.HubTest
+var isAppsecTest bool
 
-func NewHubTestCmd() *cobra.Command {
+type cliHubTest struct{}
+
+func NewCLIHubTest() *cliHubTest {
+	return &cliHubTest{}
+}
+
+func (cli cliHubTest) NewCommand() *cobra.Command {
 	var hubPath string
 	var crowdsecPath string
 	var cscliPath string
 
-	var cmdHubTest = &cobra.Command{
+	cmd := &cobra.Command{
 		Use:               "hubtest",
 		Short:             "Run functional tests on hub configurations",
 		Long:              "Run functional tests on hub configurations (parsers, scenarios, collections...)",
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
 			var err error
-			HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath)
+			HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, false)
 			if err != nil {
 				return fmt.Errorf("unable to load hubtest: %+v", err)
 			}
 
+			HubAppsecTests, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, true)
+			if err != nil {
+				return fmt.Errorf("unable to load appsec specific hubtest: %+v", err)
+			}
+			/*commands will use the hubPtr, will point to the default hubTest object, or the one dedicated to appsec tests*/
+			hubPtr = &HubTest
+			if isAppsecTest {
+				hubPtr = &HubAppsecTests
+			}
 			return nil
 		},
 	}
-	cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
-	cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
-	cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
 
-	cmdHubTest.AddCommand(NewHubTestCreateCmd())
-	cmdHubTest.AddCommand(NewHubTestRunCmd())
-	cmdHubTest.AddCommand(NewHubTestCleanCmd())
-	cmdHubTest.AddCommand(NewHubTestInfoCmd())
-	cmdHubTest.AddCommand(NewHubTestListCmd())
-	cmdHubTest.AddCommand(NewHubTestCoverageCmd())
-	cmdHubTest.AddCommand(NewHubTestEvalCmd())
-	cmdHubTest.AddCommand(NewHubTestExplainCmd())
+	cmd.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
+	cmd.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
+	cmd.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
+	cmd.PersistentFlags().BoolVar(&isAppsecTest, "appsec", false, "Command relates to appsec tests")
 
-	return cmdHubTest
+	cmd.AddCommand(cli.NewCreateCmd())
+	cmd.AddCommand(cli.NewRunCmd())
+	cmd.AddCommand(cli.NewCleanCmd())
+	cmd.AddCommand(cli.NewInfoCmd())
+	cmd.AddCommand(cli.NewListCmd())
+	cmd.AddCommand(cli.NewCoverageCmd())
+	cmd.AddCommand(cli.NewEvalCmd())
+	cmd.AddCommand(cli.NewExplainCmd())
+
+	return cmd
 }
 
-
-func NewHubTestCreateCmd() *cobra.Command {
+func (cli cliHubTest) NewCreateCmd() *cobra.Command {
 	parsers := []string{}
 	postoverflows := []string{}
 	scenarios := []string{}
@@ -68,7 +87,7 @@ func NewHubTestCreateCmd() *cobra.Command {
 	var labels map[string]string
 	var logType string
 
-	var cmdHubTestCreate = &cobra.Command{
+	cmd := &cobra.Command{
 		Use:   "create",
 		Short: "create [test_name]",
 		Example: `cscli hubtest create my-awesome-test --type syslog
@@ -76,13 +95,17 @@ cscli hubtest create my-nginx-custom-test --type nginx
 cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios crowdsecurity/http-probing`,
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(_ *cobra.Command, args []string) error {
 			testName := args[0]
-			testPath := filepath.Join(HubTest.HubTestPath, testName)
+			testPath := filepath.Join(hubPtr.HubTestPath, testName)
 			if _, err := os.Stat(testPath); os.IsExist(err) {
 				return fmt.Errorf("test '%s' already exists in '%s', exiting", testName, testPath)
 			}
 
+			if isAppsecTest {
+				logType = "appsec"
+			}
+
 			if logType == "" {
 				return fmt.Errorf("please provide a type (--type) for the test")
 			}
@@ -91,54 +114,84 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
 				return fmt.Errorf("unable to create folder '%s': %+v", testPath, err)
 			}
 
-			// create empty log file
-			logFileName := fmt.Sprintf("%s.log", testName)
-			logFilePath := filepath.Join(testPath, logFileName)
-			logFile, err := os.Create(logFilePath)
-			if err != nil {
-				return err
-			}
-			logFile.Close()
-
-			// create empty parser assertion file
-			parserAssertFilePath := filepath.Join(testPath, hubtest.ParserAssertFileName)
-			parserAssertFile, err := os.Create(parserAssertFilePath)
-			if err != nil {
-				return err
-			}
-			parserAssertFile.Close()
-
-			// create empty scenario assertion file
-			scenarioAssertFilePath := filepath.Join(testPath, hubtest.ScenarioAssertFileName)
-			scenarioAssertFile, err := os.Create(scenarioAssertFilePath)
-			if err != nil {
-				return err
-			}
-			scenarioAssertFile.Close()
-
-			parsers = append(parsers, "crowdsecurity/syslog-logs")
-			parsers = append(parsers, "crowdsecurity/dateparse-enrich")
-
-			if len(scenarios) == 0 {
-				scenarios = append(scenarios, "")
-			}
-
-			if len(postoverflows) == 0 {
-				postoverflows = append(postoverflows, "")
-			}
-
-			configFileData := &hubtest.HubTestItemConfig{
-				Parsers:       parsers,
-				Scenarios:     scenarios,
-				PostOVerflows: postoverflows,
-				LogFile:       logFileName,
-				LogType:       logType,
-				IgnoreParsers: ignoreParsers,
-				Labels:        labels,
-			}
-
 			configFilePath := filepath.Join(testPath, "config.yaml")
-			fd, err := os.OpenFile(configFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
+
+			configFileData := &hubtest.HubTestItemConfig{}
+			if logType == "appsec" {
+				//create empty nuclei template file
+				nucleiFileName := fmt.Sprintf("%s.yaml", testName)
+				nucleiFilePath := filepath.Join(testPath, nucleiFileName)
+				nucleiFile, err := os.OpenFile(nucleiFilePath, os.O_RDWR|os.O_CREATE, 0755)
+				if err != nil {
+					return err
+				}
+
+				ntpl := template.Must(template.New("nuclei").Parse(hubtest.TemplateNucleiFile))
+				if ntpl == nil {
+					return fmt.Errorf("unable to parse nuclei template")
+				}
+				ntpl.ExecuteTemplate(nucleiFile, "nuclei", struct{ TestName string }{TestName: testName})
+				nucleiFile.Close()
+				configFileData.AppsecRules = []string{"./appsec-rules/<author>/your_rule_here.yaml"}
+				configFileData.NucleiTemplate = nucleiFileName
+				fmt.Println()
+				fmt.Printf("  Test name                   :  %s\n", testName)
+				fmt.Printf("  Test path                   :  %s\n", testPath)
+				fmt.Printf("  Config File                 :  %s\n", configFilePath)
+				fmt.Printf("  Nuclei Template             :  %s\n", nucleiFilePath)
+			} else {
+				// create empty log file
+				logFileName := fmt.Sprintf("%s.log", testName)
+				logFilePath := filepath.Join(testPath, logFileName)
+				logFile, err := os.Create(logFilePath)
+				if err != nil {
+					return err
+				}
+				logFile.Close()
+
+				// create empty parser assertion file
+				parserAssertFilePath := filepath.Join(testPath, hubtest.ParserAssertFileName)
+				parserAssertFile, err := os.Create(parserAssertFilePath)
+				if err != nil {
+					return err
+				}
+				parserAssertFile.Close()
+				// create empty scenario assertion file
+				scenarioAssertFilePath := filepath.Join(testPath, hubtest.ScenarioAssertFileName)
+				scenarioAssertFile, err := os.Create(scenarioAssertFilePath)
+				if err != nil {
+					return err
+				}
+				scenarioAssertFile.Close()
+
+				parsers = append(parsers, "crowdsecurity/syslog-logs")
+				parsers = append(parsers, "crowdsecurity/dateparse-enrich")
+
+				if len(scenarios) == 0 {
+					scenarios = append(scenarios, "")
+				}
+
+				if len(postoverflows) == 0 {
+					postoverflows = append(postoverflows, "")
+				}
+				configFileData.Parsers = parsers
+				configFileData.Scenarios = scenarios
+				configFileData.PostOverflows = postoverflows
+				configFileData.LogFile = logFileName
+				configFileData.LogType = logType
+				configFileData.IgnoreParsers = ignoreParsers
+				configFileData.Labels = labels
+				fmt.Println()
+				fmt.Printf("  Test name                   :  %s\n", testName)
+				fmt.Printf("  Test path                   :  %s\n", testPath)
+				fmt.Printf("  Log file                    :  %s (please fill it with logs)\n", logFilePath)
+				fmt.Printf("  Parser assertion file       :  %s (please fill it with assertion)\n", parserAssertFilePath)
+				fmt.Printf("  Scenario assertion file     :  %s (please fill it with assertion)\n", scenarioAssertFilePath)
+				fmt.Printf("  Configuration File          :  %s (please fill it with parsers, scenarios...)\n", configFilePath)
+
+			}
+
+			fd, err := os.Create(configFilePath)
 			if err != nil {
 				return fmt.Errorf("open: %s", err)
 			}
@@ -153,56 +206,52 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
 			if err := fd.Close(); err != nil {
 				return fmt.Errorf("close: %s", err)
 			}
-			fmt.Println()
-			fmt.Printf("  Test name                   :  %s\n", testName)
-			fmt.Printf("  Test path                   :  %s\n", testPath)
-			fmt.Printf("  Log file                    :  %s (please fill it with logs)\n", logFilePath)
-			fmt.Printf("  Parser assertion file       :  %s (please fill it with assertion)\n", parserAssertFilePath)
-			fmt.Printf("  Scenario assertion file     :  %s (please fill it with assertion)\n", scenarioAssertFilePath)
-			fmt.Printf("  Configuration File          :  %s (please fill it with parsers, scenarios...)\n", configFilePath)
-
 			return nil
 		},
 	}
-	cmdHubTestCreate.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test")
-	cmdHubTestCreate.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test")
-	cmdHubTestCreate.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test")
-	cmdHubTestCreate.Flags().StringSliceVarP(&scenarios, "scenarios", "s", scenarios, "Scenarios to add to test")
-	cmdHubTestCreate.PersistentFlags().BoolVar(&ignoreParsers, "ignore-parsers", false, "Don't run test on parsers")
 
-	return cmdHubTestCreate
+	cmd.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test")
+	cmd.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test")
+	cmd.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test")
+	cmd.Flags().StringSliceVarP(&scenarios, "scenarios", "s", scenarios, "Scenarios to add to test")
+	cmd.PersistentFlags().BoolVar(&ignoreParsers, "ignore-parsers", false, "Don't run test on parsers")
+
+	return cmd
 }
 
-
-func NewHubTestRunCmd() *cobra.Command {
+func (cli cliHubTest) NewRunCmd() *cobra.Command {
 	var noClean bool
 	var runAll bool
 	var forceClean bool
-
-	var cmdHubTestRun = &cobra.Command{
+	var NucleiTargetHost string
+	var AppSecHost string
+	var cmd = &cobra.Command{
 		Use:               "run",
 		Short:             "run [test_name]",
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if !runAll && len(args) == 0 {
 				printHelp(cmd)
-				return fmt.Errorf("Please provide test to run or --all flag")
+				return fmt.Errorf("please provide test to run or --all flag")
 			}
-
+			hubPtr.NucleiTargetHost = NucleiTargetHost
+			hubPtr.AppSecHost = AppSecHost
 			if runAll {
-				if err := HubTest.LoadAllTests(); err != nil {
+				if err := hubPtr.LoadAllTests(); err != nil {
 					return fmt.Errorf("unable to load all tests: %+v", err)
 				}
 			} else {
 				for _, testName := range args {
-					_, err := HubTest.LoadTestItem(testName)
+					_, err := hubPtr.LoadTestItem(testName)
 					if err != nil {
 						return fmt.Errorf("unable to load test '%s': %s", testName, err)
 					}
 				}
 			}
 
-			for _, test := range HubTest.Tests {
+			// set timezone to avoid DST issues
+			os.Setenv("TZ", "UTC")
+			for _, test := range hubPtr.Tests {
 				if csConfig.Cscli.Output == "human" {
 					log.Infof("Running test '%s'", test.Name)
 				}
@@ -214,11 +263,11 @@ func NewHubTestRunCmd() *cobra.Command {
 
 			return nil
 		},
-		PersistentPostRunE: func(cmd *cobra.Command, args []string) error {
+		PersistentPostRunE: func(_ *cobra.Command, _ []string) error {
 			success := true
 			testResult := make(map[string]bool)
-			for _, test := range HubTest.Tests {
-				if test.AutoGen {
+			for _, test := range hubPtr.Tests {
+				if test.AutoGen && !isAppsecTest {
 					if test.ParserAssert.AutoGenAssert {
 						log.Warningf("Assert file '%s' is empty, generating assertion:", test.ParserAssert.File)
 						fmt.Println()
@@ -293,9 +342,11 @@ func NewHubTestRunCmd() *cobra.Command {
 					}
 				}
 			}
-			if csConfig.Cscli.Output == "human" {
+
+			switch csConfig.Cscli.Output {
+			case "human":
 				hubTestResultTable(color.Output, testResult)
-			} else if csConfig.Cscli.Output == "json" {
+			case "json":
 				jsonResult := make(map[string][]string, 0)
 				jsonResult["success"] = make([]string, 0)
 				jsonResult["fail"] = make([]string, 0)
@@ -311,6 +362,8 @@ func NewHubTestRunCmd() *cobra.Command {
 					return fmt.Errorf("unable to json test result: %s", err)
 				}
 				fmt.Println(string(jsonStr))
+			default:
+				return fmt.Errorf("only human/json output modes are supported")
 			}
 
 			if !success {
@@ -320,23 +373,25 @@ func NewHubTestRunCmd() *cobra.Command {
 			return nil
 		},
 	}
-	cmdHubTestRun.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
-	cmdHubTestRun.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
-	cmdHubTestRun.Flags().BoolVar(&runAll, "all", false, "Run all tests")
 
-	return cmdHubTestRun
+	cmd.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
+	cmd.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
+	cmd.Flags().StringVar(&NucleiTargetHost, "target", hubtest.DefaultNucleiTarget, "Target for AppSec Test")
+	cmd.Flags().StringVar(&AppSecHost, "host", hubtest.DefaultAppsecHost, "Address to expose AppSec for hubtest")
+	cmd.Flags().BoolVar(&runAll, "all", false, "Run all tests")
+
+	return cmd
 }
 
-
-func NewHubTestCleanCmd() *cobra.Command {
-	var cmdHubTestClean = &cobra.Command{
+func (cli cliHubTest) NewCleanCmd() *cobra.Command {
+	var cmd = &cobra.Command{
 		Use:               "clean",
 		Short:             "clean [test_name]",
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(_ *cobra.Command, args []string) error {
 			for _, testName := range args {
-				test, err := HubTest.LoadTestItem(testName)
+				test, err := hubPtr.LoadTestItem(testName)
 				if err != nil {
 					return fmt.Errorf("unable to load test '%s': %s", testName, err)
 				}
@@ -349,28 +404,32 @@ func NewHubTestCleanCmd() *cobra.Command {
 		},
 	}
 
-	return cmdHubTestClean
+	return cmd
 }
 
-
-func NewHubTestInfoCmd() *cobra.Command {
-	var cmdHubTestInfo = &cobra.Command{
+func (cli cliHubTest) NewInfoCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "info",
 		Short:             "info [test_name]",
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(_ *cobra.Command, args []string) error {
 			for _, testName := range args {
-				test, err := HubTest.LoadTestItem(testName)
+				test, err := hubPtr.LoadTestItem(testName)
 				if err != nil {
 					return fmt.Errorf("unable to load test '%s': %s", testName, err)
 				}
 				fmt.Println()
 				fmt.Printf("  Test name                   :  %s\n", test.Name)
 				fmt.Printf("  Test path                   :  %s\n", test.Path)
-				fmt.Printf("  Log file                    :  %s\n", filepath.Join(test.Path, test.Config.LogFile))
-				fmt.Printf("  Parser assertion file       :  %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName))
-				fmt.Printf("  Scenario assertion file     :  %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName))
+				if isAppsecTest {
+					fmt.Printf("  Nuclei Template             :  %s\n", test.Config.NucleiTemplate)
+					fmt.Printf("  Appsec Rules                  :  %s\n", strings.Join(test.Config.AppsecRules, ", "))
+				} else {
+					fmt.Printf("  Log file                    :  %s\n", filepath.Join(test.Path, test.Config.LogFile))
+					fmt.Printf("  Parser assertion file       :  %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName))
+					fmt.Printf("  Scenario assertion file     :  %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName))
+				}
 				fmt.Printf("  Configuration File          :  %s\n", filepath.Join(test.Path, "config.yaml"))
 			}
 
@@ -378,25 +437,24 @@ func NewHubTestInfoCmd() *cobra.Command {
 		},
 	}
 
-	return cmdHubTestInfo
+	return cmd
 }
 
-
-func NewHubTestListCmd() *cobra.Command {
-	var cmdHubTestList = &cobra.Command{
+func (cli cliHubTest) NewListCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "list",
 		Short:             "list",
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := HubTest.LoadAllTests(); err != nil {
+		RunE: func(_ *cobra.Command, _ []string) error {
+			if err := hubPtr.LoadAllTests(); err != nil {
 				return fmt.Errorf("unable to load all tests: %s", err)
 			}
 
 			switch csConfig.Cscli.Output {
 			case "human":
-				hubTestListTable(color.Output, HubTest.Tests)
+				hubTestListTable(color.Output, hubPtr.Tests)
 			case "json":
-				j, err := json.MarshalIndent(HubTest.Tests, " ", "  ")
+				j, err := json.MarshalIndent(hubPtr.Tests, " ", "  ")
 				if err != nil {
 					return err
 				}
@@ -409,31 +467,34 @@ func NewHubTestListCmd() *cobra.Command {
 		},
 	}
 
-	return cmdHubTestList
+	return cmd
 }
 
-
-func NewHubTestCoverageCmd() *cobra.Command {
+func (cli cliHubTest) NewCoverageCmd() *cobra.Command {
 	var showParserCov bool
 	var showScenarioCov bool
 	var showOnlyPercent bool
+	var showAppsecCov bool
 
-	var cmdHubTestCoverage = &cobra.Command{
+	cmd := &cobra.Command{
 		Use:               "coverage",
 		Short:             "coverage",
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(_ *cobra.Command, _ []string) error {
+			//for this one we explicitly don't do for appsec
 			if err := HubTest.LoadAllTests(); err != nil {
 				return fmt.Errorf("unable to load all tests: %+v", err)
 			}
 			var err error
-			scenarioCoverage := []hubtest.ScenarioCoverage{}
-			parserCoverage := []hubtest.ParserCoverage{}
+			scenarioCoverage := []hubtest.Coverage{}
+			parserCoverage := []hubtest.Coverage{}
+			appsecRuleCoverage := []hubtest.Coverage{}
 			scenarioCoveragePercent := 0
 			parserCoveragePercent := 0
+			appsecRuleCoveragePercent := 0
 
 			// if both are false (flag by default), show both
-			showAll := !showScenarioCov && !showParserCov
+			showAll := !showScenarioCov && !showParserCov && !showAppsecCov
 
 			if showParserCov || showAll {
 				parserCoverage, err = HubTest.GetParsersCoverage()
@@ -443,7 +504,7 @@ func NewHubTestCoverageCmd() *cobra.Command {
 				parserTested := 0
 				for _, test := range parserCoverage {
 					if test.TestsCount > 0 {
-						parserTested += 1
+						parserTested++
 					}
 				}
 				parserCoveragePercent = int(math.Round((float64(parserTested) / float64(len(parserCoverage)) * 100)))
@@ -454,27 +515,47 @@ func NewHubTestCoverageCmd() *cobra.Command {
 				if err != nil {
 					return fmt.Errorf("while getting scenario coverage: %s", err)
 				}
+
 				scenarioTested := 0
 				for _, test := range scenarioCoverage {
 					if test.TestsCount > 0 {
-						scenarioTested += 1
+						scenarioTested++
 					}
 				}
+
 				scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100)))
 			}
 
+			if showAppsecCov || showAll {
+				appsecRuleCoverage, err = HubTest.GetAppsecCoverage()
+				if err != nil {
+					return fmt.Errorf("while getting scenario coverage: %s", err)
+				}
+
+				appsecRuleTested := 0
+				for _, test := range appsecRuleCoverage {
+					if test.TestsCount > 0 {
+						appsecRuleTested++
+					}
+				}
+				appsecRuleCoveragePercent = int(math.Round((float64(appsecRuleTested) / float64(len(appsecRuleCoverage)) * 100)))
+			}
+
 			if showOnlyPercent {
 				if showAll {
-					fmt.Printf("parsers=%d%%\nscenarios=%d%%", parserCoveragePercent, scenarioCoveragePercent)
+					fmt.Printf("parsers=%d%%\nscenarios=%d%%\nappsec_rules=%d%%", parserCoveragePercent, scenarioCoveragePercent, appsecRuleCoveragePercent)
 				} else if showParserCov {
 					fmt.Printf("parsers=%d%%", parserCoveragePercent)
 				} else if showScenarioCov {
 					fmt.Printf("scenarios=%d%%", scenarioCoveragePercent)
+				} else if showAppsecCov {
+					fmt.Printf("appsec_rules=%d%%", appsecRuleCoveragePercent)
 				}
 				os.Exit(0)
 			}
 
-			if csConfig.Cscli.Output == "human" {
+			switch csConfig.Cscli.Output {
+			case "human":
 				if showParserCov || showAll {
 					hubTestParserCoverageTable(color.Output, parserCoverage)
 				}
@@ -482,6 +563,11 @@ func NewHubTestCoverageCmd() *cobra.Command {
 				if showScenarioCov || showAll {
 					hubTestScenarioCoverageTable(color.Output, scenarioCoverage)
 				}
+
+				if showAppsecCov || showAll {
+					hubTestAppsecRuleCoverageTable(color.Output, appsecRuleCoverage)
+				}
+
 				fmt.Println()
 				if showParserCov || showAll {
 					fmt.Printf("PARSERS    : %d%% of coverage\n", parserCoveragePercent)
@@ -489,7 +575,10 @@ func NewHubTestCoverageCmd() *cobra.Command {
 				if showScenarioCov || showAll {
 					fmt.Printf("SCENARIOS  : %d%% of coverage\n", scenarioCoveragePercent)
 				}
-			} else if csConfig.Cscli.Output == "json" {
+				if showAppsecCov || showAll {
+					fmt.Printf("APPSEC RULES  : %d%% of coverage\n", appsecRuleCoveragePercent)
+				}
+			case "json":
 				dump, err := json.MarshalIndent(parserCoverage, "", " ")
 				if err != nil {
 					return err
@@ -500,61 +589,71 @@ func NewHubTestCoverageCmd() *cobra.Command {
 					return err
 				}
 				fmt.Printf("%s", dump)
-			} else {
+				dump, err = json.MarshalIndent(appsecRuleCoverage, "", " ")
+				if err != nil {
+					return err
+				}
+				fmt.Printf("%s", dump)
+			default:
 				return fmt.Errorf("only human/json output modes are supported")
 			}
 
 			return nil
 		},
 	}
-	cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
-	cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
-	cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
 
-	return cmdHubTestCoverage
+	cmd.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
+	cmd.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
+	cmd.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
+	cmd.PersistentFlags().BoolVar(&showAppsecCov, "appsec", false, "Show only appsec coverage")
+
+	return cmd
 }
 
-
-func NewHubTestEvalCmd() *cobra.Command {
+func (cli cliHubTest) NewEvalCmd() *cobra.Command {
 	var evalExpression string
-	var cmdHubTestEval = &cobra.Command{
+
+	cmd := &cobra.Command{
 		Use:               "eval",
 		Short:             "eval [test_name]",
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(_ *cobra.Command, args []string) error {
 			for _, testName := range args {
-				test, err := HubTest.LoadTestItem(testName)
+				test, err := hubPtr.LoadTestItem(testName)
 				if err != nil {
 					return fmt.Errorf("can't load test: %+v", err)
 				}
+
 				err = test.ParserAssert.LoadTest(test.ParserResultFile)
 				if err != nil {
 					return fmt.Errorf("can't load test results from '%s': %+v", test.ParserResultFile, err)
 				}
+
 				output, err := test.ParserAssert.EvalExpression(evalExpression)
 				if err != nil {
 					return err
 				}
+
 				fmt.Print(output)
 			}
 
 			return nil
 		},
 	}
-	cmdHubTestEval.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval")
 
-	return cmdHubTestEval
+	cmd.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval")
+
+	return cmd
 }
 
-
-func NewHubTestExplainCmd() *cobra.Command {
-	var cmdHubTestExplain = &cobra.Command{
+func (cli cliHubTest) NewExplainCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "explain",
 		Short:             "explain [test_name]",
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(_ *cobra.Command, args []string) error {
 			for _, testName := range args {
 				test, err := HubTest.LoadTestItem(testName)
 				if err != nil {
@@ -562,34 +661,32 @@ func NewHubTestExplainCmd() *cobra.Command {
 				}
 				err = test.ParserAssert.LoadTest(test.ParserResultFile)
 				if err != nil {
-					err := test.Run()
-					if err != nil {
+					if err = test.Run(); err != nil {
 						return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
 					}
-					err = test.ParserAssert.LoadTest(test.ParserResultFile)
-					if err != nil {
+
+					if err = test.ParserAssert.LoadTest(test.ParserResultFile); err != nil {
 						return fmt.Errorf("unable to load parser result after run: %s", err)
 					}
 				}
 
 				err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
 				if err != nil {
-					err := test.Run()
-					if err != nil {
+					if err = test.Run(); err != nil {
 						return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
 					}
-					err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
-					if err != nil {
+
+					if err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile); err != nil {
 						return fmt.Errorf("unable to load scenario result after run: %s", err)
 					}
 				}
-				opts := hubtest.DumpOpts{}
-				hubtest.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData, opts)
+				opts := dumps.DumpOpts{}
+				dumps.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData, opts)
 			}
 
 			return nil
 		},
 	}
 
-	return cmdHubTestExplain
+	return cmd
 }

cmd/crowdsec-cli/hubtest_table.go

@@ -41,39 +41,61 @@ func hubTestListTable(out io.Writer, tests []*hubtest.HubTestItem) {
 	t.Render()
 }
 
-func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.ParserCoverage) {
+func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
 	t := newLightTable(out)
 	t.SetHeaders("Parser", "Status", "Number of tests")
 	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	parserTested := 0
+
 	for _, test := range coverage {
 		status := emoji.RedCircle.String()
 		if test.TestsCount > 0 {
 			status = emoji.GreenCircle.String()
 			parserTested++
 		}
-		t.AddRow(test.Parser, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
+		t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
 	}
 
 	t.Render()
 }
 
-func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.ScenarioCoverage) {
+func hubTestAppsecRuleCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
+	t := newLightTable(out)
+	t.SetHeaders("Appsec Rule", "Status", "Number of tests")
+	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
+	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
+
+	parserTested := 0
+
+	for _, test := range coverage {
+		status := emoji.RedCircle.String()
+		if test.TestsCount > 0 {
+			status = emoji.GreenCircle.String()
+			parserTested++
+		}
+		t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
+	}
+
+	t.Render()
+}
+
+func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
 	t := newLightTable(out)
 	t.SetHeaders("Scenario", "Status", "Number of tests")
 	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	parserTested := 0
+
 	for _, test := range coverage {
 		status := emoji.RedCircle.String()
 		if test.TestsCount > 0 {
 			status = emoji.GreenCircle.String()
 			parserTested++
 		}
-		t.AddRow(test.Scenario, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
+		t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
 	}
 
 	t.Render()

cmd/crowdsec-cli/item_metrics.go

@@ -0,0 +1,297 @@
+package main
+
+import (
+	"fmt"
+	"math"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/fatih/color"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/prom2json"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/crowdsecurity/go-cs-lib/trace"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func ShowMetrics(hubItem *cwhub.Item) error {
+	switch hubItem.Type {
+	case cwhub.PARSERS:
+		metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
+		parserMetricsTable(color.Output, hubItem.Name, metrics)
+	case cwhub.SCENARIOS:
+		metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
+		scenarioMetricsTable(color.Output, hubItem.Name, metrics)
+	case cwhub.COLLECTIONS:
+		for _, sub := range hubItem.SubItems() {
+			if err := ShowMetrics(sub); err != nil {
+				return err
+			}
+		}
+	case cwhub.APPSEC_RULES:
+		metrics := GetAppsecRuleMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
+		appsecMetricsTable(color.Output, hubItem.Name, metrics)
+	default: // no metrics for this item type
+	}
+	return nil
+}
+
+// GetParserMetric is a complete rip from prom2json
+func GetParserMetric(url string, itemName string) map[string]map[string]int {
+	stats := make(map[string]map[string]int)
+
+	result := GetPrometheusMetric(url)
+	for idx, fam := range result {
+		if !strings.HasPrefix(fam.Name, "cs_") {
+			continue
+		}
+		log.Tracef("round %d", idx)
+		for _, m := range fam.Metrics {
+			metric, ok := m.(prom2json.Metric)
+			if !ok {
+				log.Debugf("failed to convert metric to prom2json.Metric")
+				continue
+			}
+			name, ok := metric.Labels["name"]
+			if !ok {
+				log.Debugf("no name in Metric %v", metric.Labels)
+			}
+			if name != itemName {
+				continue
+			}
+			source, ok := metric.Labels["source"]
+			if !ok {
+				log.Debugf("no source in Metric %v", metric.Labels)
+			} else {
+				if srctype, ok := metric.Labels["type"]; ok {
+					source = srctype + ":" + source
+				}
+			}
+			value := m.(prom2json.Metric).Value
+			fval, err := strconv.ParseFloat(value, 32)
+			if err != nil {
+				log.Errorf("Unexpected int value %s : %s", value, err)
+				continue
+			}
+			ival := int(fval)
+
+			switch fam.Name {
+			case "cs_reader_hits_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+					stats[source]["parsed"] = 0
+					stats[source]["reads"] = 0
+					stats[source]["unparsed"] = 0
+					stats[source]["hits"] = 0
+				}
+				stats[source]["reads"] += ival
+			case "cs_parser_hits_ok_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["parsed"] += ival
+			case "cs_parser_hits_ko_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["unparsed"] += ival
+			case "cs_node_hits_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["hits"] += ival
+			case "cs_node_hits_ok_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["parsed"] += ival
+			case "cs_node_hits_ko_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["unparsed"] += ival
+			default:
+				continue
+			}
+		}
+	}
+	return stats
+}
+
+func GetScenarioMetric(url string, itemName string) map[string]int {
+	stats := make(map[string]int)
+
+	stats["instantiation"] = 0
+	stats["curr_count"] = 0
+	stats["overflow"] = 0
+	stats["pour"] = 0
+	stats["underflow"] = 0
+
+	result := GetPrometheusMetric(url)
+	for idx, fam := range result {
+		if !strings.HasPrefix(fam.Name, "cs_") {
+			continue
+		}
+		log.Tracef("round %d", idx)
+		for _, m := range fam.Metrics {
+			metric, ok := m.(prom2json.Metric)
+			if !ok {
+				log.Debugf("failed to convert metric to prom2json.Metric")
+				continue
+			}
+			name, ok := metric.Labels["name"]
+			if !ok {
+				log.Debugf("no name in Metric %v", metric.Labels)
+			}
+			if name != itemName {
+				continue
+			}
+			value := m.(prom2json.Metric).Value
+			fval, err := strconv.ParseFloat(value, 32)
+			if err != nil {
+				log.Errorf("Unexpected int value %s : %s", value, err)
+				continue
+			}
+			ival := int(fval)
+
+			switch fam.Name {
+			case "cs_bucket_created_total":
+				stats["instantiation"] += ival
+			case "cs_buckets":
+				stats["curr_count"] += ival
+			case "cs_bucket_overflowed_total":
+				stats["overflow"] += ival
+			case "cs_bucket_poured_total":
+				stats["pour"] += ival
+			case "cs_bucket_underflowed_total":
+				stats["underflow"] += ival
+			default:
+				continue
+			}
+		}
+	}
+	return stats
+}
+
+func GetAppsecRuleMetric(url string, itemName string) map[string]int {
+	stats := make(map[string]int)
+
+	stats["inband_hits"] = 0
+	stats["outband_hits"] = 0
+
+	results := GetPrometheusMetric(url)
+	for idx, fam := range results {
+		if !strings.HasPrefix(fam.Name, "cs_") {
+			continue
+		}
+		log.Tracef("round %d", idx)
+		for _, m := range fam.Metrics {
+			metric, ok := m.(prom2json.Metric)
+			if !ok {
+				log.Debugf("failed to convert metric to prom2json.Metric")
+				continue
+			}
+			name, ok := metric.Labels["rule_name"]
+			if !ok {
+				log.Debugf("no rule_name in Metric %v", metric.Labels)
+			}
+			if name != itemName {
+				continue
+			}
+
+			band, ok := metric.Labels["type"]
+			if !ok {
+				log.Debugf("no type in Metric %v", metric.Labels)
+			}
+
+			value := m.(prom2json.Metric).Value
+			fval, err := strconv.ParseFloat(value, 32)
+			if err != nil {
+				log.Errorf("Unexpected int value %s : %s", value, err)
+				continue
+			}
+			ival := int(fval)
+
+			switch fam.Name {
+			case "cs_appsec_rule_hits":
+				switch band {
+				case "inband":
+					stats["inband_hits"] += ival
+				case "outband":
+					stats["outband_hits"] += ival
+				default:
+					continue
+				}
+			default:
+				continue
+			}
+		}
+	}
+	return stats
+}
+
+func GetPrometheusMetric(url string) []*prom2json.Family {
+	mfChan := make(chan *dto.MetricFamily, 1024)
+
+	// Start with the DefaultTransport for sane defaults.
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	// Conservatively disable HTTP keep-alives as this program will only
+	// ever need a single HTTP request.
+	transport.DisableKeepAlives = true
+	// Timeout early if the server doesn't even return the headers.
+	transport.ResponseHeaderTimeout = time.Minute
+
+	go func() {
+		defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
+		err := prom2json.FetchMetricFamilies(url, mfChan, transport)
+		if err != nil {
+			log.Fatalf("failed to fetch prometheus metrics : %v", err)
+		}
+	}()
+
+	result := []*prom2json.Family{}
+	for mf := range mfChan {
+		result = append(result, prom2json.NewFamily(mf))
+	}
+	log.Debugf("Finished reading prometheus output, %d entries", len(result))
+
+	return result
+}
+
+type unit struct {
+	value  int64
+	symbol string
+}
+
+var ranges = []unit{
+	{value: 1e18, symbol: "E"},
+	{value: 1e15, symbol: "P"},
+	{value: 1e12, symbol: "T"},
+	{value: 1e9, symbol: "G"},
+	{value: 1e6, symbol: "M"},
+	{value: 1e3, symbol: "k"},
+	{value: 1, symbol: ""},
+}
+
+func formatNumber(num int) string {
+	goodUnit := unit{}
+
+	for _, u := range ranges {
+		if int64(num) >= u.value {
+			goodUnit = u
+			break
+		}
+	}
+
+	if goodUnit.value == 1 {
+		return fmt.Sprintf("%d%s", num, goodUnit.symbol)
+	}
+
+	res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
+
+	return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
+}

cmd/crowdsec-cli/item_suggest.go

@@ -0,0 +1,85 @@
+package main
+
+import (
+	"fmt"
+	"slices"
+	"strings"
+
+	"github.com/agext/levenshtein"
+	"github.com/spf13/cobra"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+// suggestNearestMessage returns a message with the most similar item name, if one is found
+func suggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string {
+	const maxDistance = 7
+
+	score := 100
+	nearest := ""
+
+	for _, item := range hub.GetItemMap(itemType) {
+		d := levenshtein.Distance(itemName, item.Name, nil)
+		if d < score {
+			score = d
+			nearest = item.Name
+		}
+	}
+
+	msg := fmt.Sprintf("can't find '%s' in %s", itemName, itemType)
+
+	if score < maxDistance {
+		msg += fmt.Sprintf(", did you mean '%s'?", nearest)
+	}
+
+	return msg
+}
+
+func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+	hub, err := require.Hub(csConfig, nil, nil)
+	if err != nil {
+		return nil, cobra.ShellCompDirectiveDefault
+	}
+
+	comp := make([]string, 0)
+
+	for _, item := range hub.GetItemMap(itemType) {
+		if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
+			comp = append(comp, item.Name)
+		}
+	}
+
+	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
+
+	return comp, cobra.ShellCompDirectiveNoFileComp
+}
+
+func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+	hub, err := require.Hub(csConfig, nil, nil)
+	if err != nil {
+		return nil, cobra.ShellCompDirectiveDefault
+	}
+
+	items, err := hub.GetInstalledItemNames(itemType)
+	if err != nil {
+		cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
+		return nil, cobra.ShellCompDirectiveDefault
+	}
+
+	comp := make([]string, 0)
+
+	if toComplete != "" {
+		for _, item := range items {
+			if strings.Contains(item, toComplete) {
+				comp = append(comp, item)
+			}
+		}
+	} else {
+		comp = items
+	}
+
+	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
+
+	return comp, cobra.ShellCompDirectiveNoFileComp
+}

cmd/crowdsec-cli/itemcli.go

@@ -0,0 +1,588 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"strings"
+
+	"github.com/fatih/color"
+	"github.com/hexops/gotextdiff"
+	"github.com/hexops/gotextdiff/myers"
+	"github.com/hexops/gotextdiff/span"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/crowdsecurity/go-cs-lib/coalesce"
+
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+type cliHelp struct {
+	// Example is required, the others have a default value
+	// generated from the item type
+	use     string
+	short   string
+	long    string
+	example string
+}
+
+type cliItem struct {
+	name          string // plural, as used in the hub index
+	singular      string
+	oneOrMore     string // parenthetical pluralizaion: "parser(s)"
+	help          cliHelp
+	installHelp   cliHelp
+	removeHelp    cliHelp
+	upgradeHelp   cliHelp
+	inspectHelp   cliHelp
+	inspectDetail func(item *cwhub.Item) error
+	listHelp      cliHelp
+}
+
+func (cli cliItem) NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               coalesce.String(cli.help.use, fmt.Sprintf("%s <action> [item]...", cli.name)),
+		Short:             coalesce.String(cli.help.short, fmt.Sprintf("Manage hub %s", cli.name)),
+		Long:              cli.help.long,
+		Example:           cli.help.example,
+		Args:              cobra.MinimumNArgs(1),
+		Aliases:           []string{cli.singular},
+		DisableAutoGenTag: true,
+	}
+
+	cmd.AddCommand(cli.NewInstallCmd())
+	cmd.AddCommand(cli.NewRemoveCmd())
+	cmd.AddCommand(cli.NewUpgradeCmd())
+	cmd.AddCommand(cli.NewInspectCmd())
+	cmd.AddCommand(cli.NewListCmd())
+
+	return cmd
+}
+
+func (cli cliItem) Install(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	downloadOnly, err := flags.GetBool("download-only")
+	if err != nil {
+		return err
+	}
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	ignoreError, err := flags.GetBool("ignore")
+	if err != nil {
+		return err
+	}
+
+	hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), log.StandardLogger())
+	if err != nil {
+		return err
+	}
+
+	for _, name := range args {
+		item := hub.GetItem(cli.name, name)
+		if item == nil {
+			msg := suggestNearestMessage(hub, cli.name, name)
+			if !ignoreError {
+				return fmt.Errorf(msg)
+			}
+
+			log.Errorf(msg)
+
+			continue
+		}
+
+		if err := item.Install(force, downloadOnly); err != nil {
+			if !ignoreError {
+				return fmt.Errorf("error while installing '%s': %w", item.Name, err)
+			}
+
+			log.Errorf("Error while installing '%s': %s", item.Name, err)
+		}
+	}
+
+	log.Infof(ReloadMessage())
+
+	return nil
+}
+
+func (cli cliItem) NewInstallCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               coalesce.String(cli.installHelp.use, "install [item]..."),
+		Short:             coalesce.String(cli.installHelp.short, fmt.Sprintf("Install given %s", cli.oneOrMore)),
+		Long:              coalesce.String(cli.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", cli.name)),
+		Example:           cli.installHelp.example,
+		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compAllItems(cli.name, args, toComplete)
+		},
+		RunE: cli.Install,
+	}
+
+	flags := cmd.Flags()
+	flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
+	flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
+	flags.Bool("ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", cli.name))
+
+	return cmd
+}
+
+// return the names of the installed parents of an item, used to check if we can remove it
+func istalledParentNames(item *cwhub.Item) []string {
+	ret := make([]string, 0)
+
+	for _, parent := range item.Ancestors() {
+		if parent.State.Installed {
+			ret = append(ret, parent.Name)
+		}
+	}
+
+	return ret
+}
+
+func (cli cliItem) Remove(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	purge, err := flags.GetBool("purge")
+	if err != nil {
+		return err
+	}
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	hub, err := require.Hub(csConfig, nil, log.StandardLogger())
+	if err != nil {
+		return err
+	}
+
+	if all {
+		getter := hub.GetInstalledItems
+		if purge {
+			getter = hub.GetAllItems
+		}
+
+		items, err := getter(cli.name)
+		if err != nil {
+			return err
+		}
+
+		removed := 0
+
+		for _, item := range items {
+			didRemove, err := item.Remove(purge, force)
+			if err != nil {
+				return err
+			}
+
+			if didRemove {
+				log.Infof("Removed %s", item.Name)
+				removed++
+			}
+		}
+
+		log.Infof("Removed %d %s", removed, cli.name)
+
+		if removed > 0 {
+			log.Infof(ReloadMessage())
+		}
+
+		return nil
+	}
+
+	if len(args) == 0 {
+		return fmt.Errorf("specify at least one %s to remove or '--all'", cli.singular)
+	}
+
+	removed := 0
+
+	for _, itemName := range args {
+		item := hub.GetItem(cli.name, itemName)
+		if item == nil {
+			return fmt.Errorf("can't find '%s' in %s", itemName, cli.name)
+		}
+
+		parents := istalledParentNames(item)
+
+		if !force && len(parents) > 0 {
+			log.Warningf("%s belongs to collections: %s", item.Name, parents)
+			log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, cli.singular)
+
+			continue
+		}
+
+		didRemove, err := item.Remove(purge, force)
+		if err != nil {
+			return err
+		}
+
+		if didRemove {
+			log.Infof("Removed %s", item.Name)
+			removed++
+		}
+	}
+
+	log.Infof("Removed %d %s", removed, cli.name)
+
+	if removed > 0 {
+		log.Infof(ReloadMessage())
+	}
+
+	return nil
+}
+
+func (cli cliItem) NewRemoveCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               coalesce.String(cli.removeHelp.use, "remove [item]..."),
+		Short:             coalesce.String(cli.removeHelp.short, fmt.Sprintf("Remove given %s", cli.oneOrMore)),
+		Long:              coalesce.String(cli.removeHelp.long, fmt.Sprintf("Remove one or more %s", cli.name)),
+		Example:           cli.removeHelp.example,
+		Aliases:           []string{"delete"},
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(cli.name, args, toComplete)
+		},
+		RunE: cli.Remove,
+	}
+
+	flags := cmd.Flags()
+	flags.Bool("purge", false, "Delete source file too")
+	flags.Bool("force", false, "Force remove: remove tainted and outdated files")
+	flags.Bool("all", false, fmt.Sprintf("Remove all the %s", cli.name))
+
+	return cmd
+}
+
+func (cli cliItem) Upgrade(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	force, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), log.StandardLogger())
+	if err != nil {
+		return err
+	}
+
+	if all {
+		items, err := hub.GetInstalledItems(cli.name)
+		if err != nil {
+			return err
+		}
+
+		updated := 0
+
+		for _, item := range items {
+			didUpdate, err := item.Upgrade(force)
+			if err != nil {
+				return err
+			}
+
+			if didUpdate {
+				updated++
+			}
+		}
+
+		log.Infof("Updated %d %s", updated, cli.name)
+
+		if updated > 0 {
+			log.Infof(ReloadMessage())
+		}
+
+		return nil
+	}
+
+	if len(args) == 0 {
+		return fmt.Errorf("specify at least one %s to upgrade or '--all'", cli.singular)
+	}
+
+	updated := 0
+
+	for _, itemName := range args {
+		item := hub.GetItem(cli.name, itemName)
+		if item == nil {
+			return fmt.Errorf("can't find '%s' in %s", itemName, cli.name)
+		}
+
+		didUpdate, err := item.Upgrade(force)
+		if err != nil {
+			return err
+		}
+
+		if didUpdate {
+			log.Infof("Updated %s", item.Name)
+			updated++
+		}
+	}
+
+	if updated > 0 {
+		log.Infof(ReloadMessage())
+	}
+
+	return nil
+}
+
+func (cli cliItem) NewUpgradeCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               coalesce.String(cli.upgradeHelp.use, "upgrade [item]..."),
+		Short:             coalesce.String(cli.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", cli.oneOrMore)),
+		Long:              coalesce.String(cli.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", cli.name)),
+		Example:           cli.upgradeHelp.example,
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(cli.name, args, toComplete)
+		},
+		RunE: cli.Upgrade,
+	}
+
+	flags := cmd.Flags()
+	flags.BoolP("all", "a", false, fmt.Sprintf("Upgrade all the %s", cli.name))
+	flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
+
+	return cmd
+}
+
+func (cli cliItem) Inspect(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	url, err := flags.GetString("url")
+	if err != nil {
+		return err
+	}
+
+	if url != "" {
+		csConfig.Cscli.PrometheusUrl = url
+	}
+
+	diff, err := flags.GetBool("diff")
+	if err != nil {
+		return err
+	}
+
+	rev, err := flags.GetBool("rev")
+	if err != nil {
+		return err
+	}
+
+	noMetrics, err := flags.GetBool("no-metrics")
+	if err != nil {
+		return err
+	}
+
+	remote := (*cwhub.RemoteHubCfg)(nil)
+
+	if diff {
+		remote = require.RemoteHub(csConfig)
+	}
+
+	hub, err := require.Hub(csConfig, remote, log.StandardLogger())
+	if err != nil {
+		return err
+	}
+
+	for _, name := range args {
+		item := hub.GetItem(cli.name, name)
+		if item == nil {
+			return fmt.Errorf("can't find '%s' in %s", name, cli.name)
+		}
+
+		if diff {
+			fmt.Println(cli.whyTainted(hub, item, rev))
+
+			continue
+		}
+
+		if err = InspectItem(item, !noMetrics); err != nil {
+			return err
+		}
+
+		if cli.inspectDetail != nil {
+			if err = cli.inspectDetail(item); err != nil {
+				return err
+			}
+		}
+	}
+
+	return nil
+}
+
+func (cli cliItem) NewInspectCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               coalesce.String(cli.inspectHelp.use, "inspect [item]..."),
+		Short:             coalesce.String(cli.inspectHelp.short, fmt.Sprintf("Inspect given %s", cli.oneOrMore)),
+		Long:              coalesce.String(cli.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", cli.name)),
+		Example:           cli.inspectHelp.example,
+		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(cli.name, args, toComplete)
+		},
+		PreRunE: func(cmd *cobra.Command, _ []string) error {
+			flags := cmd.Flags()
+
+			diff, err := flags.GetBool("diff")
+			if err != nil {
+				return err
+			}
+
+			rev, err := flags.GetBool("rev")
+			if err != nil {
+				return err
+			}
+
+			if rev && !diff {
+				return fmt.Errorf("--rev can only be used with --diff")
+			}
+
+			return nil
+		},
+		RunE: cli.Inspect,
+	}
+
+	flags := cmd.Flags()
+	flags.StringP("url", "u", "", "Prometheus url")
+	flags.Bool("diff", false, "Show diff with latest version (for tainted items)")
+	flags.Bool("rev", false, "Reverse diff output")
+	flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
+
+	return cmd
+}
+
+func (cli cliItem) List(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	all, err := flags.GetBool("all")
+	if err != nil {
+		return err
+	}
+
+	hub, err := require.Hub(csConfig, nil, log.StandardLogger())
+	if err != nil {
+		return err
+	}
+
+	items := make(map[string][]*cwhub.Item)
+
+	items[cli.name], err = selectItems(hub, cli.name, args, !all)
+	if err != nil {
+		return err
+	}
+
+	if err = listItems(color.Output, []string{cli.name}, items, false); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (cli cliItem) NewListCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               coalesce.String(cli.listHelp.use, "list [item... | -a]"),
+		Short:             coalesce.String(cli.listHelp.short, fmt.Sprintf("List %s", cli.oneOrMore)),
+		Long:              coalesce.String(cli.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", cli.name)),
+		Example:           cli.listHelp.example,
+		DisableAutoGenTag: true,
+		RunE:              cli.List,
+	}
+
+	flags := cmd.Flags()
+	flags.BoolP("all", "a", false, "List disabled items as well")
+
+	return cmd
+}
+
+// return the diff between the installed version and the latest version
+func (cli cliItem) itemDiff(item *cwhub.Item, reverse bool) (string, error) {
+	if !item.State.Installed {
+		return "", fmt.Errorf("'%s' is not installed", item.FQName())
+	}
+
+	latestContent, remoteURL, err := item.FetchLatest()
+	if err != nil {
+		return "", err
+	}
+
+	localContent, err := os.ReadFile(item.State.LocalPath)
+	if err != nil {
+		return "", fmt.Errorf("while reading %s: %w", item.State.LocalPath, err)
+	}
+
+	file1 := item.State.LocalPath
+	file2 := remoteURL
+	content1 := string(localContent)
+	content2 := string(latestContent)
+
+	if reverse {
+		file1, file2 = file2, file1
+		content1, content2 = content2, content1
+	}
+
+	edits := myers.ComputeEdits(span.URIFromPath(file1), content1, content2)
+	diff := gotextdiff.ToUnified(file1, file2, content1, edits)
+
+	return fmt.Sprintf("%s", diff), nil
+}
+
+func (cli cliItem) whyTainted(hub *cwhub.Hub, item *cwhub.Item, reverse bool) string {
+	if !item.State.Installed {
+		return fmt.Sprintf("# %s is not installed", item.FQName())
+	}
+
+	if !item.State.Tainted {
+		return fmt.Sprintf("# %s is not tainted", item.FQName())
+	}
+
+	if len(item.State.TaintedBy) == 0 {
+		return fmt.Sprintf("# %s is tainted but we don't know why. please report this as a bug", item.FQName())
+	}
+
+	ret := []string{
+		fmt.Sprintf("# Let's see why %s is tainted.", item.FQName()),
+	}
+
+	for _, fqsub := range item.State.TaintedBy {
+		ret = append(ret, fmt.Sprintf("\n-> %s\n", fqsub))
+
+		sub, err := hub.GetItemFQ(fqsub)
+		if err != nil {
+			ret = append(ret, err.Error())
+		}
+
+		diff, err := cli.itemDiff(sub, reverse)
+		if err != nil {
+			ret = append(ret, err.Error())
+		}
+
+		if diff != "" {
+			ret = append(ret, diff)
+		} else if len(sub.State.TaintedBy) > 0 {
+			taintList := strings.Join(sub.State.TaintedBy, ", ")
+			if sub.FQName() == taintList {
+				// hack: avoid message "item is tainted by itself"
+				continue
+			}
+			ret = append(ret, fmt.Sprintf("# %s is tainted by %s", sub.FQName(), taintList))
+		}
+	}
+
+	return strings.Join(ret, "\n")
+}

cmd/crowdsec-cli/items.go

@@ -0,0 +1,185 @@
+package main
+
+import (
+	"encoding/csv"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"slices"
+	"strings"
+
+	"gopkg.in/yaml.v3"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+// selectItems returns a slice of items of a given type, selected by name and sorted by case-insensitive name
+func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]*cwhub.Item, error) {
+	itemNames := hub.GetItemNames(itemType)
+
+	notExist := []string{}
+
+	if len(args) > 0 {
+		for _, arg := range args {
+			if !slices.Contains(itemNames, arg) {
+				notExist = append(notExist, arg)
+			}
+		}
+	}
+
+	if len(notExist) > 0 {
+		return nil, fmt.Errorf("item(s) '%s' not found in %s", strings.Join(notExist, ", "), itemType)
+	}
+
+	if len(args) > 0 {
+		itemNames = args
+		installedOnly = false
+	}
+
+	items := make([]*cwhub.Item, 0, len(itemNames))
+
+	for _, itemName := range itemNames {
+		item := hub.GetItem(itemType, itemName)
+		if installedOnly && !item.State.Installed {
+			continue
+		}
+
+		items = append(items, item)
+	}
+
+	cwhub.SortItemSlice(items)
+
+	return items, nil
+}
+
+func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item, omitIfEmpty bool) error {
+	switch csConfig.Cscli.Output {
+	case "human":
+		nothingToDisplay := true
+
+		for _, itemType := range itemTypes {
+			if omitIfEmpty && len(items[itemType]) == 0 {
+				continue
+			}
+
+			listHubItemTable(out, "\n"+strings.ToUpper(itemType), items[itemType])
+
+			nothingToDisplay = false
+		}
+
+		if nothingToDisplay {
+			fmt.Println("No items to display")
+		}
+	case "json":
+		type itemHubStatus struct {
+			Name         string `json:"name"`
+			LocalVersion string `json:"local_version"`
+			LocalPath    string `json:"local_path"`
+			Description  string `json:"description"`
+			UTF8Status   string `json:"utf8_status"`
+			Status       string `json:"status"`
+		}
+
+		hubStatus := make(map[string][]itemHubStatus)
+		for _, itemType := range itemTypes {
+			// empty slice in case there are no items of this type
+			hubStatus[itemType] = make([]itemHubStatus, len(items[itemType]))
+
+			for i, item := range items[itemType] {
+				status := item.State.Text()
+				statusEmo := item.State.Emoji()
+				hubStatus[itemType][i] = itemHubStatus{
+					Name:         item.Name,
+					LocalVersion: item.State.LocalVersion,
+					LocalPath:    item.State.LocalPath,
+					Description:  item.Description,
+					Status:       status,
+					UTF8Status:   fmt.Sprintf("%v  %s", statusEmo, status),
+				}
+			}
+		}
+
+		x, err := json.MarshalIndent(hubStatus, "", " ")
+		if err != nil {
+			return fmt.Errorf("failed to unmarshal: %w", err)
+		}
+
+		out.Write(x)
+	case "raw":
+		csvwriter := csv.NewWriter(out)
+
+		header := []string{"name", "status", "version", "description"}
+		if len(itemTypes) > 1 {
+			header = append(header, "type")
+		}
+
+		if err := csvwriter.Write(header); err != nil {
+			return fmt.Errorf("failed to write header: %s", err)
+		}
+
+		for _, itemType := range itemTypes {
+			for _, item := range items[itemType] {
+				row := []string{
+					item.Name,
+					item.State.Text(),
+					item.State.LocalVersion,
+					item.Description,
+				}
+				if len(itemTypes) > 1 {
+					row = append(row, itemType)
+				}
+
+				if err := csvwriter.Write(row); err != nil {
+					return fmt.Errorf("failed to write raw output: %s", err)
+				}
+			}
+		}
+
+		csvwriter.Flush()
+	default:
+		return fmt.Errorf("unknown output format '%s'", csConfig.Cscli.Output)
+	}
+
+	return nil
+}
+
+func InspectItem(item *cwhub.Item, showMetrics bool) error {
+	switch csConfig.Cscli.Output {
+	case "human", "raw":
+		enc := yaml.NewEncoder(os.Stdout)
+		enc.SetIndent(2)
+
+		if err := enc.Encode(item); err != nil {
+			return fmt.Errorf("unable to encode item: %s", err)
+		}
+	case "json":
+		b, err := json.MarshalIndent(*item, "", "  ")
+		if err != nil {
+			return fmt.Errorf("unable to marshal item: %s", err)
+		}
+
+		fmt.Print(string(b))
+	}
+
+	if csConfig.Cscli.Output != "human" {
+		return nil
+	}
+
+	if item.State.Tainted {
+		fmt.Println()
+		fmt.Printf(`This item is tainted. Use "%s %s inspect --diff %s" to see why.`, filepath.Base(os.Args[0]), item.Type, item.Name)
+		fmt.Println()
+	}
+
+	if showMetrics {
+		fmt.Printf("\nCurrent metrics: \n")
+
+		if err := ShowMetrics(item); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

cmd/crowdsec-cli/lapi.go

@@ -2,10 +2,10 @@ package main
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net/url"
 	"os"
-	"slices"
 	"sort"
 	"strings"
 
@@ -13,6 +13,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
+	"slices"
 
 	"github.com/crowdsecurity/go-cs-lib/version"
 
@@ -26,25 +27,24 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/parser"
 )
 
-var LAPIURLPrefix string = "v1"
+const LAPIURLPrefix = "v1"
 
 func runLapiStatus(cmd *cobra.Command, args []string) error {
-	var err error
-
 	password := strfmt.Password(csConfig.API.Client.Credentials.Password)
 	apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL)
 	login := csConfig.API.Client.Credentials.Login
 	if err != nil {
-		log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
+		return fmt.Errorf("parsing api url: %w", err)
 	}
 
-	if err := require.Hub(csConfig); err != nil {
-		log.Fatal(err)
-	}
-
-	scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
+	hub, err := require.Hub(csConfig, nil, nil)
 	if err != nil {
-		log.Fatalf("failed to get scenarios : %s", err)
+		return err
+	}
+
+	scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
+	if err != nil {
+		return fmt.Errorf("failed to get scenarios: %w", err)
 	}
 
 	Client, err = apiclient.NewDefaultClient(apiurl,
@@ -52,28 +52,27 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
 		fmt.Sprintf("crowdsec/%s", version.String()),
 		nil)
 	if err != nil {
-		log.Fatalf("init default client: %s", err)
+		return fmt.Errorf("init default client: %w", err)
 	}
 	t := models.WatcherAuthRequest{
 		MachineID: &login,
 		Password:  &password,
 		Scenarios: scenarios,
 	}
+
 	log.Infof("Loaded credentials from %s", csConfig.API.Client.CredentialsFilePath)
 	log.Infof("Trying to authenticate with username %s on %s", login, apiurl)
+
 	_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
 	if err != nil {
-		log.Fatalf("Failed to authenticate to Local API (LAPI) : %s", err)
-	} else {
-		log.Infof("You can successfully interact with Local API (LAPI)")
+		return fmt.Errorf("failed to authenticate to Local API (LAPI): %w", err)
 	}
 
+	log.Infof("You can successfully interact with Local API (LAPI)")
 	return nil
 }
 
 func runLapiRegister(cmd *cobra.Command, args []string) error {
-	var err error
-
 	flags := cmd.Flags()
 
 	apiURL, err := flags.GetString("url")
@@ -94,16 +93,15 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
 	if lapiUser == "" {
 		lapiUser, err = generateID("")
 		if err != nil {
-			log.Fatalf("unable to generate machine id: %s", err)
+			return fmt.Errorf("unable to generate machine id: %w", err)
 		}
 	}
 	password := strfmt.Password(generatePassword(passwordLength))
 	if apiURL == "" {
-		if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" {
-			apiURL = csConfig.API.Client.Credentials.URL
-		} else {
-			log.Fatalf("No Local API URL. Please provide it in your configuration or with the -u parameter")
+		if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil || csConfig.API.Client.Credentials.URL == "" {
+			return fmt.Errorf("no Local API URL. Please provide it in your configuration or with the -u parameter")
 		}
+		apiURL = csConfig.API.Client.Credentials.URL
 	}
 	/*URL needs to end with /, but user doesn't care*/
 	if !strings.HasSuffix(apiURL, "/") {
@@ -115,7 +113,7 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
 	}
 	apiurl, err := url.Parse(apiURL)
 	if err != nil {
-		log.Fatalf("parsing api url: %s", err)
+		return fmt.Errorf("parsing api url: %w", err)
 	}
 	_, err = apiclient.RegisterClient(&apiclient.Config{
 		MachineID:     lapiUser,
@@ -126,7 +124,7 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
 	}, nil)
 
 	if err != nil {
-		log.Fatalf("api client register: %s", err)
+		return fmt.Errorf("api client register: %w", err)
 	}
 
 	log.Printf("Successfully registered to Local API (LAPI)")
@@ -146,14 +144,14 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
 	}
 	apiConfigDump, err := yaml.Marshal(apiCfg)
 	if err != nil {
-		log.Fatalf("unable to marshal api credentials: %s", err)
+		return fmt.Errorf("unable to marshal api credentials: %w", err)
 	}
 	if dumpFile != "" {
-		err = os.WriteFile(dumpFile, apiConfigDump, 0644)
+		err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
 		if err != nil {
-			log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
+			return fmt.Errorf("write api credentials to '%s' failed: %w", dumpFile, err)
 		}
-		log.Printf("Local API credentials dumped to '%s'", dumpFile)
+		log.Printf("Local API credentials written to '%s'", dumpFile)
 	} else {
 		fmt.Printf("%s\n", string(apiConfigDump))
 	}
@@ -194,7 +192,7 @@ Keep in mind the machine needs to be validated by an administrator on LAPI side
 }
 
 func NewLapiCmd() *cobra.Command {
-	var cmdLapi = &cobra.Command{
+	cmdLapi := &cobra.Command{
 		Use:               "lapi [action]",
 		Short:             "Manage interaction with Local API (LAPI)",
 		Args:              cobra.MinimumNArgs(1),
@@ -220,6 +218,7 @@ func AddContext(key string, values []string) error {
 	}
 	if _, ok := csConfig.Crowdsec.ContextToSend[key]; !ok {
 		csConfig.Crowdsec.ContextToSend[key] = make([]string, 0)
+
 		log.Infof("key '%s' added", key)
 	}
 	data := csConfig.Crowdsec.ContextToSend[key]
@@ -246,11 +245,11 @@ func NewLapiContextCmd() *cobra.Command {
 			if err := csConfig.LoadCrowdsec(); err != nil {
 				fileNotFoundMessage := fmt.Sprintf("failed to open context file: open %s: no such file or directory", csConfig.Crowdsec.ConsoleContextPath)
 				if err.Error() != fileNotFoundMessage {
-					log.Fatalf("Unable to load CrowdSec Agent: %s", err)
+					return fmt.Errorf("unable to load CrowdSec agent configuration: %w", err)
 				}
 			}
 			if csConfig.DisableAgent {
-				log.Fatalf("Agent is disabled and lapi context can only be used on the agent")
+				return errors.New("agent is disabled and lapi context can only be used on the agent")
 			}
 
 			return nil
@@ -270,12 +269,21 @@ cscli lapi context add --key file_source --value evt.Line.Src
 cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user 
 		`,
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
+			hub, err := require.Hub(csConfig, nil, nil)
+			if err != nil {
+				return err
+			}
+
+			if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil {
+				return fmt.Errorf("while loading context: %w", err)
+			}
+
 			if keyToAdd != "" {
 				if err := AddContext(keyToAdd, valuesToAdd); err != nil {
-					log.Fatalf(err.Error())
+					return err
 				}
-				return
+				return nil
 			}
 
 			for _, v := range valuesToAdd {
@@ -283,9 +291,11 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
 				key := keySlice[len(keySlice)-1]
 				value := []string{v}
 				if err := AddContext(key, value); err != nil {
-					log.Fatalf(err.Error())
+					return err
 				}
 			}
+
+			return nil
 		},
 	}
 	cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
@@ -297,19 +307,29 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
 		Use:               "status",
 		Short:             "List context to send with alerts",
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
+		RunE: func(cmd *cobra.Command, args []string) error {
+			hub, err := require.Hub(csConfig, nil, nil)
+			if err != nil {
+				return err
+			}
+
+			if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil {
+				return fmt.Errorf("while loading context: %w", err)
+			}
+
 			if len(csConfig.Crowdsec.ContextToSend) == 0 {
 				fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.")
-				return
+				return nil
 			}
 
 			dump, err := yaml.Marshal(csConfig.Crowdsec.ContextToSend)
 			if err != nil {
-				log.Fatalf("unable to show context status: %s", err)
+				return fmt.Errorf("unable to show context status: %w", err)
 			}
 
-			fmt.Println(string(dump))
+			fmt.Print(string(dump))
 
+			return nil
 		},
 	}
 	cmdContext.AddCommand(cmdContextStatus)
@@ -322,30 +342,27 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
 cscli lapi context detect crowdsecurity/sshd-logs
 		`,
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			var err error
-
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if !detectAll && len(args) == 0 {
 				log.Infof("Please provide parsers to detect or --all flag.")
 				printHelp(cmd)
 			}
 
 			// to avoid all the log.Info from the loaders functions
-			log.SetLevel(log.ErrorLevel)
+			log.SetLevel(log.WarnLevel)
 
-			err = exprhelpers.Init(nil)
+			if err := exprhelpers.Init(nil); err != nil {
+				return fmt.Errorf("failed to init expr helpers: %w", err)
+			}
+
+			hub, err := require.Hub(csConfig, nil, nil)
 			if err != nil {
-				log.Fatalf("Failed to init expr helpers : %s", err)
+				return err
 			}
 
-			// Populate cwhub package tools
-			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
-				log.Fatalf("Failed to load hub index : %s", err)
-			}
-
-			csParsers := parser.NewParsers()
+			csParsers := parser.NewParsers(hub)
 			if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil {
-				log.Fatalf("unable to load parsers: %s", err)
+				return fmt.Errorf("unable to load parsers: %w", err)
 			}
 
 			fieldByParsers := make(map[string][]string)
@@ -365,7 +382,6 @@ cscli lapi context detect crowdsecurity/sshd-logs
 						fieldByParsers[node.Name] = append(fieldByParsers[node.Name], field)
 					}
 				}
-
 			}
 
 			fmt.Printf("Acquisition :\n\n")
@@ -398,59 +414,25 @@ cscli lapi context detect crowdsecurity/sshd-logs
 					log.Errorf("parser '%s' not found, can't detect fields", parserNotFound)
 				}
 			}
+
+			return nil
 		},
 	}
 	cmdContextDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser")
 	cmdContext.AddCommand(cmdContextDetect)
 
-	var keysToDelete []string
-	var valuesToDelete []string
 	cmdContextDelete := &cobra.Command{
-		Use:   "delete",
-		Short: "Delete context to send with alerts",
-		Example: `cscli lapi context delete --key source_ip
-cscli lapi context delete --value evt.Line.Src
-		`,
+		Use:               "delete",
 		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			if len(keysToDelete) == 0 && len(valuesToDelete) == 0 {
-				log.Fatalf("please provide at least a key or a value to delete")
+		RunE: func(_ *cobra.Command, _ []string) error {
+			filePath := csConfig.Crowdsec.ConsoleContextPath
+			if filePath == "" {
+				filePath = "the context file"
 			}
-
-			for _, key := range keysToDelete {
-				if _, ok := csConfig.Crowdsec.ContextToSend[key]; ok {
-					delete(csConfig.Crowdsec.ContextToSend, key)
-					log.Infof("key '%s' has been removed", key)
-				} else {
-					log.Warningf("key '%s' doesn't exist", key)
-				}
-			}
-
-			for _, value := range valuesToDelete {
-				valueFound := false
-				for key, context := range csConfig.Crowdsec.ContextToSend {
-					if slices.Contains(context, value) {
-						valueFound = true
-						csConfig.Crowdsec.ContextToSend[key] = removeFromSlice(value, context)
-						log.Infof("value '%s' has been removed from key '%s'", value, key)
-					}
-					if len(csConfig.Crowdsec.ContextToSend[key]) == 0 {
-						delete(csConfig.Crowdsec.ContextToSend, key)
-					}
-				}
-				if !valueFound {
-					log.Warningf("value '%s' not found", value)
-				}
-			}
-
-			if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
-				log.Fatalf(err.Error())
-			}
-
+			fmt.Printf("Command \"delete\" is deprecated, please manually edit %s.", filePath)
+			return nil
 		},
 	}
-	cmdContextDelete.Flags().StringSliceVarP(&keysToDelete, "key", "k", []string{}, "The keys to delete")
-	cmdContextDelete.Flags().StringSliceVar(&valuesToDelete, "value", []string{}, "The expr fields to delete")
 	cmdContext.AddCommand(cmdContextDelete)
 
 	return cmdContext
@@ -458,6 +440,7 @@ cscli lapi context delete --value evt.Line.Src
 
 func detectStaticField(GrokStatics []parser.ExtraField) []string {
 	ret := make([]string, 0)
+
 	for _, static := range GrokStatics {
 		if static.Parsed != "" {
 			fieldName := fmt.Sprintf("evt.Parsed.%s", static.Parsed)
@@ -486,7 +469,8 @@ func detectStaticField(GrokStatics []parser.ExtraField) []string {
 }
 
 func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
-	var ret = make([]string, 0)
+	ret := make([]string, 0)
+
 	if node.Grok.RunTimeRegexp != nil {
 		for _, capturedField := range node.Grok.RunTimeRegexp.Names() {
 			fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
@@ -498,13 +482,13 @@ func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
 
 	if node.Grok.RegexpName != "" {
 		grokCompiled, err := parserCTX.Grok.Get(node.Grok.RegexpName)
-		if err != nil {
-			log.Warningf("Can't get subgrok: %s", err)
-		}
-		for _, capturedField := range grokCompiled.Names() {
-			fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
-			if !slices.Contains(ret, fieldName) {
-				ret = append(ret, fieldName)
+		// ignore error (parser does not exist?)
+		if err == nil {
+			for _, capturedField := range grokCompiled.Names() {
+				fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
+				if !slices.Contains(ret, fieldName) {
+					ret = append(ret, fieldName)
+				}
 			}
 		}
 	}
@@ -544,13 +528,13 @@ func detectSubNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
 		}
 		if subnode.Grok.RegexpName != "" {
 			grokCompiled, err := parserCTX.Grok.Get(subnode.Grok.RegexpName)
-			if err != nil {
-				log.Warningf("Can't get subgrok: %s", err)
-			}
-			for _, capturedField := range grokCompiled.Names() {
-				fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
-				if !slices.Contains(ret, fieldName) {
-					ret = append(ret, fieldName)
+			if err == nil {
+				// ignore error (parser does not exist?)
+				for _, capturedField := range grokCompiled.Names() {
+					fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
+					if !slices.Contains(ret, fieldName) {
+						ret = append(ret, fieldName)
+					}
 				}
 			}
 		}

cmd/crowdsec-cli/machines.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"math/big"
 	"os"
-	"slices"
 	"strings"
 	"time"
 
@@ -18,21 +17,19 @@ import (
 	"github.com/google/uuid"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v3"
+	"slices"
 
 	"github.com/crowdsecurity/machineid"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/database/ent"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
-var (
-	passwordLength = 64
-)
+const passwordLength = 64
 
 func generatePassword(length int) string {
 	upper := "ABCDEFGHIJKLMNOPQRSTUVWXY"
@@ -43,6 +40,7 @@ func generatePassword(length int) string {
 	charsetLength := len(charset)
 
 	buf := make([]byte, length)
+
 	for i := 0; i < length; i++ {
 		rInt, err := saferand.Int(saferand.Reader, big.NewInt(int64(charsetLength)))
 		if err != nil {
@@ -64,9 +62,9 @@ func generateIDPrefix() (string, error) {
 	}
 	log.Debugf("failed to get machine-id with usual files: %s", err)
 
-	bId, err := uuid.NewRandom()
+	bID, err := uuid.NewRandom()
 	if err == nil {
-		return bId.String(), nil
+		return bID.String(), nil
 	}
 	return "", fmt.Errorf("generating machine id: %w", err)
 }
@@ -108,27 +106,27 @@ func getAgents(out io.Writer, dbClient *database.Client) error {
 	if err != nil {
 		return fmt.Errorf("unable to list machines: %s", err)
 	}
-	if csConfig.Cscli.Output == "human" {
+
+	switch csConfig.Cscli.Output {
+	case "human":
 		getAgentsTable(out, machines)
-	} else if csConfig.Cscli.Output == "json" {
+	case "json":
 		enc := json.NewEncoder(out)
 		enc.SetIndent("", "  ")
 		if err := enc.Encode(machines); err != nil {
 			return fmt.Errorf("failed to marshal")
 		}
 		return nil
-	} else if csConfig.Cscli.Output == "raw" {
+	case "raw":
 		csvwriter := csv.NewWriter(out)
 		err := csvwriter.Write([]string{"machine_id", "ip_address", "updated_at", "validated", "version", "auth_type", "last_heartbeat"})
 		if err != nil {
 			return fmt.Errorf("failed to write header: %s", err)
 		}
 		for _, m := range machines {
-			var validated string
+			validated := "false"
 			if m.IsValidated {
 				validated = "true"
-			} else {
-				validated = "false"
 			}
 			hb, _ := getLastHeartbeat(m)
 			err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb})
@@ -137,21 +135,59 @@ func getAgents(out io.Writer, dbClient *database.Client) error {
 			}
 		}
 		csvwriter.Flush()
-	} else {
-		log.Errorf("unknown output '%s'", csConfig.Cscli.Output)
+	default:
+		return fmt.Errorf("unknown output '%s'", csConfig.Cscli.Output)
 	}
 	return nil
 }
 
-func NewMachinesListCmd() *cobra.Command {
-	cmdMachinesList := &cobra.Command{
+type cliMachines struct{}
+
+func NewCLIMachines() *cliMachines {
+	return &cliMachines{}
+}
+
+func (cli cliMachines) NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "machines [action]",
+		Short: "Manage local API machines [requires local API]",
+		Long: `To list/add/delete/validate/prune machines.
+Note: This command requires database direct access, so is intended to be run on the local API machine.
+`,
+		Example:           `cscli machines [action]`,
+		DisableAutoGenTag: true,
+		Aliases:           []string{"machine"},
+		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
+			var err error
+			if err = require.LAPI(csConfig); err != nil {
+				return err
+			}
+			dbClient, err = database.NewClient(csConfig.DbConfig)
+			if err != nil {
+				return fmt.Errorf("unable to create new database client: %s", err)
+			}
+			return nil
+		},
+	}
+
+	cmd.AddCommand(cli.NewListCmd())
+	cmd.AddCommand(cli.NewAddCmd())
+	cmd.AddCommand(cli.NewDeleteCmd())
+	cmd.AddCommand(cli.NewValidateCmd())
+	cmd.AddCommand(cli.NewPruneCmd())
+
+	return cmd
+}
+
+func (cli cliMachines) NewListCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "list",
 		Short:             "list all machines in the database",
 		Long:              `list all machines in the database with their status and last heartbeat`,
 		Example:           `cscli machines list`,
 		Args:              cobra.NoArgs,
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(_ *cobra.Command, _ []string) error {
 			err := getAgents(color.Output, dbClient)
 			if err != nil {
 				return fmt.Errorf("unable to list machines: %s", err)
@@ -161,11 +197,11 @@ func NewMachinesListCmd() *cobra.Command {
 		},
 	}
 
-	return cmdMachinesList
+	return cmd
 }
 
-func NewMachinesAddCmd() *cobra.Command {
-	cmdMachinesAdd := &cobra.Command{
+func (cli cliMachines) NewAddCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "add",
 		Short:             "add a single machine to the database",
 		DisableAutoGenTag: true,
@@ -175,10 +211,10 @@ cscli machines add --auto
 cscli machines add MyTestMachine --auto
 cscli machines add MyTestMachine --password MyPassword
 `,
-		RunE: runMachinesAdd,
+		RunE: cli.add,
 	}
 
-	flags := cmdMachinesAdd.Flags()
+	flags := cmd.Flags()
 	flags.StringP("password", "p", "", "machine password to login to the API")
 	flags.StringP("file", "f", "", "output file destination (defaults to "+csconfig.DefaultConfigPath("local_api_credentials.yaml")+")")
 	flags.StringP("url", "u", "", "URL of the local API")
@@ -186,13 +222,10 @@ cscli machines add MyTestMachine --password MyPassword
 	flags.BoolP("auto", "a", false, "automatically generate password (and username if not provided)")
 	flags.Bool("force", false, "will force add the machine if it already exist")
 
-	return cmdMachinesAdd
+	return cmd
 }
 
-func runMachinesAdd(cmd *cobra.Command, args []string) error {
-	var dumpFile string
-	var err error
-
+func (cli cliMachines) add(cmd *cobra.Command, args []string) error {
 	flags := cmd.Flags()
 
 	machinePassword, err := flags.GetString("password")
@@ -200,7 +233,7 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	outputFile, err := flags.GetString("file")
+	dumpFile, err := flags.GetString("file")
 	if err != nil {
 		return err
 	}
@@ -220,7 +253,7 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	forceAdd, err := flags.GetBool("force")
+	force, err := flags.GetBool("force")
 	if err != nil {
 		return err
 	}
@@ -242,17 +275,29 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
 	}
 
 	/*check if file already exists*/
-	if outputFile != "" {
-		dumpFile = outputFile
-	} else if csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" {
-		dumpFile = csConfig.API.Client.CredentialsFilePath
+	if dumpFile == "" && csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" {
+		credFile := csConfig.API.Client.CredentialsFilePath
+		// use the default only if the file does not exist
+		_, err = os.Stat(credFile)
+
+		switch {
+		case os.IsNotExist(err) || force:
+			dumpFile = csConfig.API.Client.CredentialsFilePath
+		case err != nil:
+			return fmt.Errorf("unable to stat '%s': %s", credFile, err)
+		default:
+			return fmt.Errorf(`credentials file '%s' already exists: please remove it, use "--force" or specify a different file with "-f" ("-f -" for standard output)`, credFile)
+		}
+	}
+
+	if dumpFile == "" {
+		return fmt.Errorf(`please specify a file to dump credentials to, with -f ("-f -" for standard output)`)
 	}
 
 	// create a password if it's not specified by user
 	if machinePassword == "" && !interactive {
 		if !autoAdd {
-			printHelp(cmd)
-			return nil
+			return fmt.Errorf("please specify a password with --password or use --auto")
 		}
 		machinePassword = generatePassword(passwordLength)
 	} else if machinePassword == "" && interactive {
@@ -262,11 +307,11 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
 		survey.AskOne(qs, &machinePassword)
 	}
 	password := strfmt.Password(machinePassword)
-	_, err = dbClient.CreateMachine(&machineID, &password, "", true, forceAdd, types.PasswordAuthType)
+	_, err = dbClient.CreateMachine(&machineID, &password, "", true, force, types.PasswordAuthType)
 	if err != nil {
 		return fmt.Errorf("unable to create machine: %s", err)
 	}
-	log.Infof("Machine '%s' successfully added to the local API", machineID)
+	fmt.Printf("Machine '%s' successfully added to the local API.\n", machineID)
 
 	if apiURL == "" {
 		if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" {
@@ -287,11 +332,11 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("unable to marshal api credentials: %s", err)
 	}
 	if dumpFile != "" && dumpFile != "-" {
-		err = os.WriteFile(dumpFile, apiConfigDump, 0644)
+		err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
 		if err != nil {
 			return fmt.Errorf("write api credentials in '%s' failed: %s", dumpFile, err)
 		}
-		log.Printf("API credentials dumped to '%s'", dumpFile)
+		fmt.Printf("API credentials written to '%s'.\n", dumpFile)
 	} else {
 		fmt.Printf("%s\n", string(apiConfigDump))
 	}
@@ -299,15 +344,15 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-func NewMachinesDeleteCmd() *cobra.Command {
-	cmdMachinesDelete := &cobra.Command{
+func (cli cliMachines) NewDeleteCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "delete [machine_name]...",
 		Short:             "delete machine(s) by name",
 		Example:           `cscli machines delete "machine1" "machine2"`,
 		Args:              cobra.MinimumNArgs(1),
 		Aliases:           []string{"remove"},
 		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+		ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
 			machines, err := dbClient.ListMachines()
 			if err != nil {
 				cobra.CompError("unable to list machines " + err.Error())
@@ -320,13 +365,13 @@ func NewMachinesDeleteCmd() *cobra.Command {
 			}
 			return ret, cobra.ShellCompDirectiveNoFileComp
 		},
-		RunE: runMachinesDelete,
+		RunE: cli.delete,
 	}
 
-	return cmdMachinesDelete
+	return cmd
 }
 
-func runMachinesDelete(cmd *cobra.Command, args []string) error {
+func (cli cliMachines) delete(_ *cobra.Command, args []string) error {
 	for _, machineID := range args {
 		err := dbClient.DeleteWatcher(machineID)
 		if err != nil {
@@ -339,9 +384,9 @@ func runMachinesDelete(cmd *cobra.Command, args []string) error {
 	return nil
 }
 
-func NewMachinesPruneCmd() *cobra.Command {
+func (cli cliMachines) NewPruneCmd() *cobra.Command {
 	var parsedDuration time.Duration
-	cmdMachinesPrune := &cobra.Command{
+	cmd := &cobra.Command{
 		Use:   "prune",
 		Short: "prune multiple machines from the database",
 		Long:  `prune multiple machines that are not validated or have not connected to the local API in a given duration.`,
@@ -350,7 +395,7 @@ cscli machines prune --duration 1h
 cscli machines prune --not-validated-only --force`,
 		Args:              cobra.NoArgs,
 		DisableAutoGenTag: true,
-		PreRunE: func(cmd *cobra.Command, args []string) error {
+		PreRunE: func(cmd *cobra.Command, _ []string) error {
 			dur, _ := cmd.Flags().GetString("duration")
 			var err error
 			parsedDuration, err = time.ParseDuration(fmt.Sprintf("-%s", dur))
@@ -359,7 +404,7 @@ cscli machines prune --not-validated-only --force`,
 			}
 			return nil
 		},
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, _ []string) error {
 			notValidOnly, _ := cmd.Flags().GetBool("not-validated-only")
 			force, _ := cmd.Flags().GetBool("force")
 			if parsedDuration >= 0-60*time.Second && !notValidOnly {
@@ -412,22 +457,22 @@ cscli machines prune --not-validated-only --force`,
 			return nil
 		},
 	}
-	cmdMachinesPrune.Flags().StringP("duration", "d", "10m", "duration of time since validated machine last heartbeat")
-	cmdMachinesPrune.Flags().Bool("not-validated-only", false, "only prune machines that are not validated")
-	cmdMachinesPrune.Flags().Bool("force", false, "force prune without asking for confirmation")
+	cmd.Flags().StringP("duration", "d", "10m", "duration of time since validated machine last heartbeat")
+	cmd.Flags().Bool("not-validated-only", false, "only prune machines that are not validated")
+	cmd.Flags().Bool("force", false, "force prune without asking for confirmation")
 
-	return cmdMachinesPrune
+	return cmd
 }
 
-func NewMachinesValidateCmd() *cobra.Command {
-	cmdMachinesValidate := &cobra.Command{
+func (cli cliMachines) NewValidateCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "validate",
 		Short:             "validate a machine to access the local API",
 		Long:              `validate a machine to access the local API.`,
 		Example:           `cscli machines validate "machine_name"`,
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
+		RunE: func(_ *cobra.Command, args []string) error {
 			machineID := args[0]
 			if err := dbClient.ValidateMachine(machineID); err != nil {
 				return fmt.Errorf("unable to validate machine '%s': %s", machineID, err)
@@ -438,37 +483,5 @@ func NewMachinesValidateCmd() *cobra.Command {
 		},
 	}
 
-	return cmdMachinesValidate
-}
-
-func NewMachinesCmd() *cobra.Command {
-	var cmdMachines = &cobra.Command{
-		Use:   "machines [action]",
-		Short: "Manage local API machines [requires local API]",
-		Long: `To list/add/delete/validate/prune machines.
-Note: This command requires database direct access, so is intended to be run on the local API machine.
-`,
-		Example:           `cscli machines [action]`,
-		DisableAutoGenTag: true,
-		Aliases:           []string{"machine"},
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			var err error
-			if err := require.LAPI(csConfig); err != nil {
-				return err
-			}
-			dbClient, err = database.NewClient(csConfig.DbConfig)
-			if err != nil {
-				return fmt.Errorf("unable to create new database client: %s", err)
-			}
-			return nil
-		},
-	}
-
-	cmdMachines.AddCommand(NewMachinesListCmd())
-	cmdMachines.AddCommand(NewMachinesAddCmd())
-	cmdMachines.AddCommand(NewMachinesDeleteCmd())
-	cmdMachines.AddCommand(NewMachinesValidateCmd())
-	cmdMachines.AddCommand(NewMachinesPruneCmd())
-
-	return cmdMachines
+	return cmd
 }

cmd/crowdsec-cli/main.go

@@ -1,21 +1,16 @@
 package main
 
 import (
-	"fmt"
 	"os"
-	"path/filepath"
 	"slices"
-	"strings"
+	"time"
 
 	"github.com/fatih/color"
 	cc "github.com/ivanpirog/coloredcobra"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"github.com/spf13/cobra/doc"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/fflag"
 )
@@ -29,17 +24,14 @@ var dbClient *database.Client
 var OutputFormat string
 var OutputColor string
 
-var downloadOnly bool
-var forceAction bool
-var purge bool
-var all bool
-
-var prometheusURL string
-
 var mergedConfig string
 
+// flagBranch overrides the value in csConfig.Cscli.HubBranch
+var flagBranch = ""
+
 func initConfig() {
 	var err error
+
 	if trace_lvl {
 		log.SetLevel(log.TraceLevel)
 	} else if dbg_lvl {
@@ -58,9 +50,6 @@ func initConfig() {
 		if err != nil {
 			log.Fatal(err)
 		}
-		if err := csConfig.LoadCSCLI(); err != nil {
-			log.Fatal(err)
-		}
 	} else {
 		csConfig = csconfig.NewDefaultConfig()
 	}
@@ -71,15 +60,13 @@ func initConfig() {
 		log.Debugf("Enabled feature flags: %s", fflist)
 	}
 
-	if csConfig.Cscli == nil {
-		log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath)
+	if flagBranch != "" {
+		csConfig.Cscli.HubBranch = flagBranch
 	}
 
-	if cwhub.HubBranch == "" && csConfig.Cscli.HubBranch != "" {
-		cwhub.HubBranch = csConfig.Cscli.HubBranch
-	}
 	if OutputFormat != "" {
 		csConfig.Cscli.Output = OutputFormat
+
 		if OutputFormat != "json" && OutputFormat != "raw" && OutputFormat != "human" {
 			log.Fatalf("output format %s unknown", OutputFormat)
 		}
@@ -96,45 +83,32 @@ func initConfig() {
 
 	if OutputColor != "" {
 		csConfig.Cscli.Color = OutputColor
+
 		if OutputColor != "yes" && OutputColor != "no" && OutputColor != "auto" {
 			log.Fatalf("output color %s unknown", OutputColor)
 		}
 	}
 }
 
+// list of valid subcommands for the shell completion
 var validArgs = []string{
-	"scenarios", "parsers", "collections", "capi", "lapi", "postoverflows", "machines",
-	"metrics", "bouncers", "alerts", "decisions", "simulation", "hub", "dashboard",
-	"config", "completion", "version", "console", "notifications", "support",
+	"alerts", "appsec-configs", "appsec-rules", "bouncers", "capi", "collections",
+	"completion", "config", "console", "contexts", "dashboard", "decisions", "explain",
+	"hub", "hubtest", "lapi", "machines", "metrics", "notifications", "parsers",
+	"postoverflows", "scenarios", "simulation", "support", "version",
 }
 
-func prepender(filename string) string {
-	const header = `---
-id: %s
-title: %s
----
-`
-	name := filepath.Base(filename)
-	base := strings.TrimSuffix(name, filepath.Ext(name))
-	return fmt.Sprintf(header, base, strings.ReplaceAll(base, "_", " "))
+var NoNeedConfig = []string{
+	"doc",
+	"help",
+	"completion",
+	"version",
+	"hubtest",
 }
 
-func linkHandler(name string) string {
-	return fmt.Sprintf("/cscli/%s", name)
-}
-
-var (
-	NoNeedConfig = []string{
-		"help",
-		"completion",
-		"version",
-		"hubtest",
-	}
-)
-
 func main() {
 	// set the formatter asap and worry about level later
-	logFormatter := &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}
+	logFormatter := &log.TextFormatter{TimestampFormat: time.RFC3339, FullTimestamp: true}
 	log.SetFormatter(logFormatter)
 
 	if err := fflag.RegisterAllFeatures(); err != nil {
@@ -145,7 +119,7 @@ func main() {
 		log.Fatalf("failed to set feature flags from env: %s", err)
 	}
 
-	var rootCmd = &cobra.Command{
+	cmd := &cobra.Command{
 		Use:   "cscli",
 		Short: "cscli allows you to manage crowdsec",
 		Long: `cscli is the main command to interact with your crowdsec service, scenarios & db.
@@ -158,7 +132,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	}
 
 	cc.Init(&cc.Config{
-		RootCmd:       rootCmd,
+		RootCmd:       cmd,
 		Headings:      cc.Yellow,
 		Commands:      cc.Green + cc.Bold,
 		CmdShortDescr: cc.Cyan,
@@ -169,45 +143,19 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 		Flags:         cc.Green,
 		FlagsDescr:    cc.Cyan,
 	})
-	rootCmd.SetOut(color.Output)
+	cmd.SetOut(color.Output)
 
-	var cmdDocGen = &cobra.Command{
-		Use:               "doc",
-		Short:             "Generate the documentation in `./doc/`. Directory must exist.",
-		Args:              cobra.ExactArgs(0),
-		Hidden:            true,
-		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if err := doc.GenMarkdownTreeCustom(rootCmd, "./doc/", prepender, linkHandler); err != nil {
-				return fmt.Errorf("Failed to generate cobra doc: %s", err)
-			}
-			return nil
-		},
-	}
-	rootCmd.AddCommand(cmdDocGen)
-	/*usage*/
-	var cmdVersion = &cobra.Command{
-		Use:               "version",
-		Short:             "Display version",
-		Args:              cobra.ExactArgs(0),
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			cwversion.Show()
-		},
-	}
-	rootCmd.AddCommand(cmdVersion)
+	cmd.PersistentFlags().StringVarP(&ConfigFilePath, "config", "c", csconfig.DefaultConfigPath("config.yaml"), "path to crowdsec config file")
+	cmd.PersistentFlags().StringVarP(&OutputFormat, "output", "o", "", "Output format: human, json, raw")
+	cmd.PersistentFlags().StringVarP(&OutputColor, "color", "", "auto", "Output color: yes, no, auto")
+	cmd.PersistentFlags().BoolVar(&dbg_lvl, "debug", false, "Set logging to debug")
+	cmd.PersistentFlags().BoolVar(&nfo_lvl, "info", false, "Set logging to info")
+	cmd.PersistentFlags().BoolVar(&wrn_lvl, "warning", false, "Set logging to warning")
+	cmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error")
+	cmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace")
 
-	rootCmd.PersistentFlags().StringVarP(&ConfigFilePath, "config", "c", csconfig.DefaultConfigPath("config.yaml"), "path to crowdsec config file")
-	rootCmd.PersistentFlags().StringVarP(&OutputFormat, "output", "o", "", "Output format: human, json, raw")
-	rootCmd.PersistentFlags().StringVarP(&OutputColor, "color", "", "auto", "Output color: yes, no, auto")
-	rootCmd.PersistentFlags().BoolVar(&dbg_lvl, "debug", false, "Set logging to debug")
-	rootCmd.PersistentFlags().BoolVar(&nfo_lvl, "info", false, "Set logging to info")
-	rootCmd.PersistentFlags().BoolVar(&wrn_lvl, "warning", false, "Set logging to warning")
-	rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error")
-	rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace")
-
-	rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "", "Override hub branch on github")
-	if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil {
+	cmd.PersistentFlags().StringVar(&flagBranch, "branch", "", "Override hub branch on github")
+	if err := cmd.PersistentFlags().MarkHidden("branch"); err != nil {
 		log.Fatalf("failed to hide flag: %s", err)
 	}
 
@@ -231,40 +179,46 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	}
 
 	/*don't sort flags so we can enforce order*/
-	rootCmd.Flags().SortFlags = false
-	rootCmd.PersistentFlags().SortFlags = false
+	cmd.Flags().SortFlags = false
+	cmd.PersistentFlags().SortFlags = false
 
-	rootCmd.AddCommand(NewConfigCmd())
-	rootCmd.AddCommand(NewHubCmd())
-	rootCmd.AddCommand(NewMetricsCmd())
-	rootCmd.AddCommand(NewDashboardCmd())
-	rootCmd.AddCommand(NewDecisionsCmd())
-	rootCmd.AddCommand(NewAlertsCmd())
-	rootCmd.AddCommand(NewSimulationCmds())
-	rootCmd.AddCommand(NewBouncersCmd())
-	rootCmd.AddCommand(NewMachinesCmd())
-	rootCmd.AddCommand(NewParsersCmd())
-	rootCmd.AddCommand(NewScenariosCmd())
-	rootCmd.AddCommand(NewCollectionsCmd())
-	rootCmd.AddCommand(NewPostOverflowsCmd())
-	rootCmd.AddCommand(NewCapiCmd())
-	rootCmd.AddCommand(NewLapiCmd())
-	rootCmd.AddCommand(NewCompletionCmd())
-	rootCmd.AddCommand(NewConsoleCmd())
-	rootCmd.AddCommand(NewExplainCmd())
-	rootCmd.AddCommand(NewHubTestCmd())
-	rootCmd.AddCommand(NewNotificationsCmd())
-	rootCmd.AddCommand(NewSupportCmd())
+	cmd.AddCommand(NewCLIDoc().NewCommand(cmd))
+	cmd.AddCommand(NewCLIVersion().NewCommand())
+	cmd.AddCommand(NewConfigCmd())
+	cmd.AddCommand(NewCLIHub().NewCommand())
+	cmd.AddCommand(NewMetricsCmd())
+	cmd.AddCommand(NewCLIDashboard().NewCommand())
+	cmd.AddCommand(NewCLIDecisions().NewCommand())
+	cmd.AddCommand(NewCLIAlerts().NewCommand())
+	cmd.AddCommand(NewCLISimulation().NewCommand())
+	cmd.AddCommand(NewCLIBouncers().NewCommand())
+	cmd.AddCommand(NewCLIMachines().NewCommand())
+	cmd.AddCommand(NewCLICapi().NewCommand())
+	cmd.AddCommand(NewLapiCmd())
+	cmd.AddCommand(NewCompletionCmd())
+	cmd.AddCommand(NewConsoleCmd())
+	cmd.AddCommand(NewCLIExplain().NewCommand())
+	cmd.AddCommand(NewCLIHubTest().NewCommand())
+	cmd.AddCommand(NewCLINotifications().NewCommand())
+	cmd.AddCommand(NewCLISupport().NewCommand())
+	cmd.AddCommand(NewCLIPapi().NewCommand())
+	cmd.AddCommand(NewCLICollection().NewCommand())
+	cmd.AddCommand(NewCLIParser().NewCommand())
+	cmd.AddCommand(NewCLIScenario().NewCommand())
+	cmd.AddCommand(NewCLIPostOverflow().NewCommand())
+	cmd.AddCommand(NewCLIContext().NewCommand())
+	cmd.AddCommand(NewCLIAppsecConfig().NewCommand())
+	cmd.AddCommand(NewCLIAppsecRule().NewCommand())
 
 	if fflag.CscliSetup.IsEnabled() {
-		rootCmd.AddCommand(NewSetupCmd())
+		cmd.AddCommand(NewSetupCmd())
 	}
 
 	if fflag.PapiClient.IsEnabled() {
-		rootCmd.AddCommand(NewPapiCmd())
+		cmd.AddCommand(NewCLIPapi().NewCommand())
 	}
 
-	if err := rootCmd.Execute(); err != nil {
+	if err := cmd.Execute(); err != nil {
 		log.Fatal(err)
 	}
 }

cmd/crowdsec-cli/metrics.go

@@ -63,6 +63,8 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
 	lapi_machine_stats := map[string]map[string]map[string]int{}
 	lapi_bouncer_stats := map[string]map[string]map[string]int{}
 	decisions_stats := map[string]map[string]map[string]int{}
+	appsec_engine_stats := map[string]map[string]int{}
+	appsec_rule_stats := map[string]map[string]map[string]int{}
 	alerts_stats := map[string]int{}
 	stash_stats := map[string]struct {
 		Type  string
@@ -226,10 +228,30 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
 					Type  string
 					Count int
 				}{Type: mtype, Count: ival}
+			case "cs_appsec_reqs_total":
+				if _, ok := appsec_engine_stats[metric.Labels["appsec_engine"]]; !ok {
+					appsec_engine_stats[metric.Labels["appsec_engine"]] = make(map[string]int, 0)
+				}
+				appsec_engine_stats[metric.Labels["appsec_engine"]]["processed"] = ival
+			case "cs_appsec_block_total":
+				if _, ok := appsec_engine_stats[metric.Labels["appsec_engine"]]; !ok {
+					appsec_engine_stats[metric.Labels["appsec_engine"]] = make(map[string]int, 0)
+				}
+				appsec_engine_stats[metric.Labels["appsec_engine"]]["blocked"] = ival
+			case "cs_appsec_rule_hits":
+				appsecEngine := metric.Labels["appsec_engine"]
+				ruleID := metric.Labels["rule_name"]
+				if _, ok := appsec_rule_stats[appsecEngine]; !ok {
+					appsec_rule_stats[appsecEngine] = make(map[string]map[string]int, 0)
+				}
+				if _, ok := appsec_rule_stats[appsecEngine][ruleID]; !ok {
+					appsec_rule_stats[appsecEngine][ruleID] = make(map[string]int, 0)
+				}
+				appsec_rule_stats[appsecEngine][ruleID]["triggered"] = ival
 			default:
+				log.Debugf("unknown: %+v", fam.Name)
 				continue
 			}
-
 		}
 	}
 
@@ -244,6 +266,8 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
 		decisionStatsTable(out, decisions_stats)
 		alertStatsTable(out, alerts_stats)
 		stashStatsTable(out, stash_stats)
+		appsecMetricsToTable(out, appsec_engine_stats)
+		appsecRulesToTable(out, appsec_rule_stats)
 		return nil
 	}
 
@@ -282,10 +306,21 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
 
 var noUnit bool
 
-
 func runMetrics(cmd *cobra.Command, args []string) error {
-	if err := csConfig.LoadPrometheus(); err != nil {
-		return fmt.Errorf("failed to load prometheus config: %w", err)
+	flags := cmd.Flags()
+
+	url, err := flags.GetString("url")
+	if err != nil {
+		return err
+	}
+
+	if url != "" {
+		csConfig.Cscli.PrometheusUrl = url
+	}
+
+	noUnit, err = flags.GetBool("no-unit")
+	if err != nil {
+		return err
 	}
 
 	if csConfig.Prometheus == nil {
@@ -296,22 +331,12 @@ func runMetrics(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("prometheus is not enabled, can't show metrics")
 	}
 
-	if prometheusURL == "" {
-		prometheusURL = csConfig.Cscli.PrometheusUrl
-	}
-
-	if prometheusURL == "" {
-		return fmt.Errorf("no prometheus url, please specify in %s or via -u", *csConfig.FilePath)
-	}
-
-	err := FormatPrometheusMetrics(color.Output, prometheusURL+"/metrics", csConfig.Cscli.Output)
-	if err != nil {
-		return fmt.Errorf("could not fetch prometheus metrics: %w", err)
+	if err = FormatPrometheusMetrics(color.Output, csConfig.Cscli.PrometheusUrl, csConfig.Cscli.Output); err != nil {
+		return err
 	}
 	return nil
 }
 
-
 func NewMetricsCmd() *cobra.Command {
 	cmdMetrics := &cobra.Command{
 		Use:               "metrics",
@@ -319,10 +344,12 @@ func NewMetricsCmd() *cobra.Command {
 		Long:              `Fetch metrics from the prometheus server and display them in a human-friendly way`,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		RunE: runMetrics,
+		RunE:              runMetrics,
 	}
-	cmdMetrics.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
-	cmdMetrics.PersistentFlags().BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
+
+	flags := cmdMetrics.PersistentFlags()
+	flags.StringP("url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
+	flags.Bool("no-unit", false, "Show the real number instead of formatted with units")
 
 	return cmdMetrics
 }

cmd/crowdsec-cli/metrics_table.go

@@ -90,7 +90,7 @@ func bucketStatsTable(out io.Writer, stats map[string]map[string]int) {
 	keys := []string{"curr_count", "overflow", "instantiation", "pour", "underflow"}
 
 	if numRows, err := metricsToTable(t, stats, keys); err != nil {
-		log.Warningf("while collecting acquis stats: %s", err)
+		log.Warningf("while collecting bucket stats: %s", err)
 	} else if numRows > 0 {
 		renderTableTitle(out, "\nBucket Metrics:")
 		t.Render()
@@ -113,6 +113,37 @@ func acquisStatsTable(out io.Writer, stats map[string]map[string]int) {
 	}
 }
 
+func appsecMetricsToTable(out io.Writer, metrics map[string]map[string]int) {
+	t := newTable(out)
+	t.SetRowLines(false)
+	t.SetHeaders("Appsec Engine", "Processed", "Blocked")
+	t.SetAlignment(table.AlignLeft, table.AlignLeft)
+	keys := []string{"processed", "blocked"}
+	if numRows, err := metricsToTable(t, metrics, keys); err != nil {
+		log.Warningf("while collecting appsec stats: %s", err)
+	} else if numRows > 0 {
+		renderTableTitle(out, "\nAppsec Metrics:")
+		t.Render()
+	}
+}
+
+func appsecRulesToTable(out io.Writer, metrics map[string]map[string]map[string]int) {
+	for appsecEngine, appsecEngineRulesStats := range metrics {
+		t := newTable(out)
+		t.SetRowLines(false)
+		t.SetHeaders("Rule ID", "Triggered")
+		t.SetAlignment(table.AlignLeft, table.AlignLeft)
+		keys := []string{"triggered"}
+		if numRows, err := metricsToTable(t, appsecEngineRulesStats, keys); err != nil {
+			log.Warningf("while collecting appsec rules stats: %s", err)
+		} else if numRows > 0 {
+			renderTableTitle(out, fmt.Sprintf("\nAppsec '%s' Rules Metrics:", appsecEngine))
+			t.Render()
+		}
+	}
+
+}
+
 func parserStatsTable(out io.Writer, stats map[string]map[string]int) {
 	t := newTable(out)
 	t.SetRowLines(false)
@@ -122,7 +153,7 @@ func parserStatsTable(out io.Writer, stats map[string]map[string]int) {
 	keys := []string{"hits", "parsed", "unparsed"}
 
 	if numRows, err := metricsToTable(t, stats, keys); err != nil {
-		log.Warningf("while collecting acquis stats: %s", err)
+		log.Warningf("while collecting parsers stats: %s", err)
 	} else if numRows > 0 {
 		renderTableTitle(out, "\nParser Metrics:")
 		t.Render()

cmd/crowdsec-cli/notifications.go

@@ -18,15 +18,19 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"gopkg.in/tomb.v2"
+	"gopkg.in/yaml.v3"
 
+	"github.com/crowdsecurity/go-cs-lib/ptr"
 	"github.com/crowdsecurity/go-cs-lib/version"
 
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/csplugin"
 	"github.com/crowdsecurity/crowdsec/pkg/csprofiles"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
 
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
+	"github.com/crowdsecurity/crowdsec/pkg/models"
 )
 
 type NotificationsCfg struct {
@@ -35,8 +39,14 @@ type NotificationsCfg struct {
 	ids      []uint
 }
 
-func NewNotificationsCmd() *cobra.Command {
-	var cmdNotifications = &cobra.Command{
+type cliNotifications struct{}
+
+func NewCLINotifications() *cliNotifications {
+	return &cliNotifications{}
+}
+
+func (cli cliNotifications) NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "notifications [action]",
 		Short:             "Helper for notification plugin configuration",
 		Long:              "To list/inspect/test notification template",
@@ -47,8 +57,8 @@ func NewNotificationsCmd() *cobra.Command {
 			if err := require.LAPI(csConfig); err != nil {
 				return err
 			}
-			if err := require.Profiles(csConfig); err != nil {
-				return err
+			if err := csConfig.LoadAPIClient(); err != nil {
+				return fmt.Errorf("loading api client: %w", err)
 			}
 			if err := require.Notifications(csConfig); err != nil {
 				return err
@@ -58,14 +68,15 @@ func NewNotificationsCmd() *cobra.Command {
 		},
 	}
 
-	cmdNotifications.AddCommand(NewNotificationsListCmd())
-	cmdNotifications.AddCommand(NewNotificationsInspectCmd())
-	cmdNotifications.AddCommand(NewNotificationsReinjectCmd())
+	cmd.AddCommand(cli.NewListCmd())
+	cmd.AddCommand(cli.NewInspectCmd())
+	cmd.AddCommand(cli.NewReinjectCmd())
+	cmd.AddCommand(cli.NewTestCmd())
 
-	return cmdNotifications
+	return cmd
 }
 
-func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
+func getPluginConfigs() (map[string]csplugin.PluginConfig, error) {
 	pcfgs := map[string]csplugin.PluginConfig{}
 	wf := func(path string, info fs.FileInfo, err error) error {
 		if info == nil {
@@ -78,6 +89,7 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
 				return fmt.Errorf("loading notifification plugin configuration with %s: %w", name, err)
 			}
 			for _, t := range ts {
+				csplugin.SetRequiredFields(&t)
 				pcfgs[t.Name] = t
 			}
 		}
@@ -87,57 +99,53 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
 	if err := filepath.Walk(csConfig.ConfigPaths.NotificationDir, wf); err != nil {
 		return nil, fmt.Errorf("while loading notifification plugin configuration: %w", err)
 	}
+	return pcfgs, nil
+}
 
+func getProfilesConfigs() (map[string]NotificationsCfg, error) {
 	// A bit of a tricky stuf now: reconcile profiles and notification plugins
+	pcfgs, err := getPluginConfigs()
+	if err != nil {
+		return nil, err
+	}
 	ncfgs := map[string]NotificationsCfg{}
+	for _, pc := range pcfgs {
+		ncfgs[pc.Name] = NotificationsCfg{
+			Config: pc,
+		}
+	}
 	profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
 	if err != nil {
 		return nil, fmt.Errorf("while extracting profiles from configuration: %w", err)
 	}
 	for profileID, profile := range profiles {
-	loop:
 		for _, notif := range profile.Cfg.Notifications {
-			for name, pc := range pcfgs {
-				if notif == name {
-					if _, ok := ncfgs[pc.Name]; !ok {
-						ncfgs[pc.Name] = NotificationsCfg{
-							Config:   pc,
-							Profiles: []*csconfig.ProfileCfg{profile.Cfg},
-							ids:      []uint{uint(profileID)},
-						}
-						continue loop
-					}
-					tmp := ncfgs[pc.Name]
-					for _, pr := range tmp.Profiles {
-						var profiles []*csconfig.ProfileCfg
-						if pr.Name == profile.Cfg.Name {
-							continue
-						}
-						profiles = append(tmp.Profiles, profile.Cfg)
-						ids := append(tmp.ids, uint(profileID))
-						ncfgs[pc.Name] = NotificationsCfg{
-							Config:   tmp.Config,
-							Profiles: profiles,
-							ids:      ids,
-						}
-					}
-				}
+			pc, ok := pcfgs[notif]
+			if !ok {
+				return nil, fmt.Errorf("notification plugin '%s' does not exist", notif)
 			}
+			tmp, ok := ncfgs[pc.Name]
+			if !ok {
+				return nil, fmt.Errorf("notification plugin '%s' does not exist", pc.Name)
+			}
+			tmp.Profiles = append(tmp.Profiles, profile.Cfg)
+			tmp.ids = append(tmp.ids, uint(profileID))
+			ncfgs[pc.Name] = tmp
 		}
 	}
 	return ncfgs, nil
 }
 
-func NewNotificationsListCmd() *cobra.Command {
-	var cmdNotificationsList = &cobra.Command{
+func (cli cliNotifications) NewListCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "list",
-		Short:             "List active notifications plugins",
-		Long:              `List active notifications plugins`,
+		Short:             "list active notifications plugins",
+		Long:              `list active notifications plugins`,
 		Example:           `cscli notifications list`,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, arg []string) error {
-			ncfgs, err := getNotificationsConfiguration()
+			ncfgs, err := getProfilesConfigs()
 			if err != nil {
 				return fmt.Errorf("can't build profiles configuration: %w", err)
 			}
@@ -172,36 +180,32 @@ func NewNotificationsListCmd() *cobra.Command {
 		},
 	}
 
-	return cmdNotificationsList
+	return cmd
 }
 
-func NewNotificationsInspectCmd() *cobra.Command {
-	var cmdNotificationsInspect = &cobra.Command{
+func (cli cliNotifications) NewInspectCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "inspect",
 		Short:             "Inspect active notifications plugin configuration",
 		Long:              `Inspect active notifications plugin and show configuration`,
 		Example:           `cscli notifications inspect <plugin_name>`,
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, arg []string) error {
-			var (
-				cfg NotificationsCfg
-				ok  bool
-			)
-
-			pluginName := arg[0]
-
-			if pluginName == "" {
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			if args[0] == "" {
 				return fmt.Errorf("please provide a plugin name to inspect")
 			}
-			ncfgs, err := getNotificationsConfiguration()
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ncfgs, err := getProfilesConfigs()
 			if err != nil {
 				return fmt.Errorf("can't build profiles configuration: %w", err)
 			}
-			if cfg, ok = ncfgs[pluginName]; !ok {
-				return fmt.Errorf("plugin '%s' does not exist or is not active", pluginName)
+			cfg, ok := ncfgs[args[0]]
+			if !ok {
+				return fmt.Errorf("plugin '%s' does not exist or is not active", args[0])
 			}
-
 			if csConfig.Cscli.Output == "human" || csConfig.Cscli.Output == "raw" {
 				fmt.Printf(" - %15s: %15s\n", "Type", cfg.Config.Type)
 				fmt.Printf(" - %15s: %15s\n", "Name", cfg.Config.Name)
@@ -221,78 +225,128 @@ func NewNotificationsInspectCmd() *cobra.Command {
 		},
 	}
 
-	return cmdNotificationsInspect
+	return cmd
 }
 
-func NewNotificationsReinjectCmd() *cobra.Command {
-	var remediation bool
-	var alertOverride string
+func (cli cliNotifications) NewTestCmd() *cobra.Command {
+	var (
+		pluginBroker  csplugin.PluginBroker
+		pluginTomb    tomb.Tomb
+		alertOverride string
+	)
+	cmd := &cobra.Command{
+		Use:               "test [plugin name]",
+		Short:             "send a generic test alert to notification plugin",
+		Long:              `send a generic test alert to a notification plugin to test configuration even if is not active`,
+		Example:           `cscli notifications test [plugin_name]`,
+		Args:              cobra.ExactArgs(1),
+		DisableAutoGenTag: true,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			pconfigs, err := getPluginConfigs()
+			if err != nil {
+				return fmt.Errorf("can't build profiles configuration: %w", err)
+			}
+			cfg, ok := pconfigs[args[0]]
+			if !ok {
+				return fmt.Errorf("plugin name: '%s' does not exist", args[0])
+			}
+			//Create a single profile with plugin name as notification name
+			return pluginBroker.Init(csConfig.PluginConfig, []*csconfig.ProfileCfg{
+				{
+					Notifications: []string{
+						cfg.Name,
+					},
+				},
+			}, csConfig.ConfigPaths)
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			pluginTomb.Go(func() error {
+				pluginBroker.Run(&pluginTomb)
+				return nil
+			})
+			alert := &models.Alert{
+				Capacity: ptr.Of(int32(0)),
+				Decisions: []*models.Decision{{
+					Duration: ptr.Of("4h"),
+					Scope:    ptr.Of("Ip"),
+					Value:    ptr.Of("10.10.10.10"),
+					Type:     ptr.Of("ban"),
+					Scenario: ptr.Of("test alert"),
+					Origin:   ptr.Of(types.CscliOrigin),
+				}},
+				Events:          []*models.Event{},
+				EventsCount:     ptr.Of(int32(1)),
+				Leakspeed:       ptr.Of("0"),
+				Message:         ptr.Of("test alert"),
+				ScenarioHash:    ptr.Of(""),
+				Scenario:        ptr.Of("test alert"),
+				ScenarioVersion: ptr.Of(""),
+				Simulated:       ptr.Of(false),
+				Source: &models.Source{
+					AsName:   "",
+					AsNumber: "",
+					Cn:       "",
+					IP:       "10.10.10.10",
+					Range:    "",
+					Scope:    ptr.Of("Ip"),
+					Value:    ptr.Of("10.10.10.10"),
+				},
+				StartAt:   ptr.Of(time.Now().UTC().Format(time.RFC3339)),
+				StopAt:    ptr.Of(time.Now().UTC().Format(time.RFC3339)),
+				CreatedAt: time.Now().UTC().Format(time.RFC3339),
+			}
+			if err := yaml.Unmarshal([]byte(alertOverride), alert); err != nil {
+				return fmt.Errorf("failed to unmarshal alert override: %w", err)
+			}
+			pluginBroker.PluginChannel <- csplugin.ProfileAlert{
+				ProfileID: uint(0),
+				Alert:     alert,
+			}
+			//time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
+			pluginTomb.Kill(fmt.Errorf("terminating"))
+			pluginTomb.Wait()
+			return nil
+		},
+	}
+	cmd.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the generic alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
 
-	var cmdNotificationsReinject = &cobra.Command{
+	return cmd
+}
+
+func (cli cliNotifications) NewReinjectCmd() *cobra.Command {
+	var alertOverride string
+	var alert *models.Alert
+
+	cmd := &cobra.Command{
 		Use:   "reinject",
-		Short: "reinject alert into notifications system",
-		Long:  `Reinject alert into notifications system`,
+		Short: "reinject an alert into profiles to trigger notifications",
+		Long:  `reinject an alert into profiles to be evaluated by the filter and sent to matched notifications plugins`,
 		Example: `
 cscli notifications reinject <alert_id>
-cscli notifications reinject <alert_id> --remediation
+cscli notifications reinject <alert_id> -a '{"remediation": false,"scenario":"notification/test"}'
 cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"notification/test"}'
 `,
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			var err error
+			alert, err = FetchAlertFromArgString(args[0])
+			if err != nil {
+				return err
+			}
+			return nil
+		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			var (
 				pluginBroker csplugin.PluginBroker
 				pluginTomb   tomb.Tomb
 			)
-			if len(args) != 1 {
-				printHelp(cmd)
-				return fmt.Errorf("wrong number of argument: there should be one argument")
-			}
-
-			//first: get the alert
-			id, err := strconv.Atoi(args[0])
-			if err != nil {
-				return fmt.Errorf("bad alert id %s", args[0])
-			}
-			if err := csConfig.LoadAPIClient(); err != nil {
-				return fmt.Errorf("loading api client: %w", err)
-			}
-			if csConfig.API.Client == nil {
-				return fmt.Errorf("missing configuration on 'api_client:'")
-			}
-			if csConfig.API.Client.Credentials == nil {
-				return fmt.Errorf("missing API credentials in '%s'", csConfig.API.Client.CredentialsFilePath)
-			}
-			apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
-			if err != nil {
-				return fmt.Errorf("error parsing the URL of the API: %w", err)
-			}
-			client, err := apiclient.NewClient(&apiclient.Config{
-				MachineID:     csConfig.API.Client.Credentials.Login,
-				Password:      strfmt.Password(csConfig.API.Client.Credentials.Password),
-				UserAgent:     fmt.Sprintf("crowdsec/%s", version.String()),
-				URL:           apiURL,
-				VersionPrefix: "v1",
-			})
-			if err != nil {
-				return fmt.Errorf("error creating the client for the API: %w", err)
-			}
-			alert, _, err := client.Alerts.GetByID(context.Background(), id)
-			if err != nil {
-				return fmt.Errorf("can't find alert with id %s: %w", args[0], err)
-			}
-
 			if alertOverride != "" {
-				if err = json.Unmarshal([]byte(alertOverride), alert); err != nil {
+				if err := json.Unmarshal([]byte(alertOverride), alert); err != nil {
 					return fmt.Errorf("can't unmarshal data in the alert flag: %w", err)
 				}
 			}
-			if !remediation {
-				alert.Remediation = true
-			}
-
-			// second we start plugins
-			err = pluginBroker.Init(csConfig.PluginConfig, csConfig.API.Server.Profiles, csConfig.ConfigPaths)
+			err := pluginBroker.Init(csConfig.PluginConfig, csConfig.API.Server.Profiles, csConfig.ConfigPaths)
 			if err != nil {
 				return fmt.Errorf("can't initialize plugins: %w", err)
 			}
@@ -302,8 +356,6 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
 				return nil
 			})
 
-			//third: get the profile(s), and process the whole stuff
-
 			profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
 			if err != nil {
 				return fmt.Errorf("cannot extract profiles from configuration: %w", err)
@@ -338,15 +390,39 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
 					break
 				}
 			}
-
-			//			time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
+			//time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
 			pluginTomb.Kill(fmt.Errorf("terminating"))
 			pluginTomb.Wait()
 			return nil
 		},
 	}
-	cmdNotificationsReinject.Flags().BoolVarP(&remediation, "remediation", "r", false, "Set Alert.Remediation to false in the reinjected alert (see your profile filter configuration)")
-	cmdNotificationsReinject.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the reinjected alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
+	cmd.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the reinjected alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
 
-	return cmdNotificationsReinject
+	return cmd
+}
+
+func FetchAlertFromArgString(toParse string) (*models.Alert, error) {
+	id, err := strconv.Atoi(toParse)
+	if err != nil {
+		return nil, fmt.Errorf("bad alert id %s", toParse)
+	}
+	apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
+	if err != nil {
+		return nil, fmt.Errorf("error parsing the URL of the API: %w", err)
+	}
+	client, err := apiclient.NewClient(&apiclient.Config{
+		MachineID:     csConfig.API.Client.Credentials.Login,
+		Password:      strfmt.Password(csConfig.API.Client.Credentials.Password),
+		UserAgent:     fmt.Sprintf("crowdsec/%s", version.String()),
+		URL:           apiURL,
+		VersionPrefix: "v1",
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error creating the client for the API: %w", err)
+	}
+	alert, _, err := client.Alerts.GetByID(context.Background(), id)
+	if err != nil {
+		return nil, fmt.Errorf("can't find alert with id %d: %w", id, err)
+	}
+	return alert, nil
 }

cmd/crowdsec-cli/notifications_table.go

@@ -2,24 +2,36 @@ package main
 
 import (
 	"io"
+	"sort"
 	"strings"
 
 	"github.com/aquasecurity/table"
+	"github.com/enescakir/emoji"
 )
 
 func notificationListTable(out io.Writer, ncfgs map[string]NotificationsCfg) {
 	t := newLightTable(out)
-	t.SetHeaders("Name", "Type", "Profile name")
-	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
-	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
-
-	for _, b := range ncfgs {
+	t.SetHeaders("Active", "Name", "Type", "Profile name")
+	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
+	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
+	keys := make([]string, 0, len(ncfgs))
+	for k := range ncfgs {
+		keys = append(keys, k)
+	}
+	sort.Slice(keys, func(i, j int) bool {
+		return len(ncfgs[keys[i]].Profiles) > len(ncfgs[keys[j]].Profiles)
+	})
+	for _, k := range keys {
+		b := ncfgs[k]
 		profilesList := []string{}
 		for _, p := range b.Profiles {
 			profilesList = append(profilesList, p.Name)
 		}
-		t.AddRow(b.Config.Name, b.Config.Type, strings.Join(profilesList, ", "))
+		active := emoji.CheckMark.String()
+		if len(profilesList) == 0 {
+			active = emoji.Prohibited.String()
+		}
+		t.AddRow(active, b.Config.Name, b.Config.Type, strings.Join(profilesList, ", "))
 	}
-
 	t.Render()
 }

cmd/crowdsec-cli/papi.go

@@ -15,8 +15,14 @@ import (
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 )
 
-func NewPapiCmd() *cobra.Command {
-	var cmdLapi = &cobra.Command{
+type cliPapi struct {}
+
+func NewCLIPapi() *cliPapi {
+	return &cliPapi{}
+}
+
+func (cli cliPapi) NewCommand() *cobra.Command {
+	var cmd = &cobra.Command{
 		Use:               "papi [action]",
 		Short:             "Manage interaction with Polling API (PAPI)",
 		Args:              cobra.MinimumNArgs(1),
@@ -35,14 +41,14 @@ func NewPapiCmd() *cobra.Command {
 		},
 	}
 
-	cmdLapi.AddCommand(NewPapiStatusCmd())
-	cmdLapi.AddCommand(NewPapiSyncCmd())
+	cmd.AddCommand(cli.NewStatusCmd())
+	cmd.AddCommand(cli.NewSyncCmd())
 
-	return cmdLapi
+	return cmd
 }
 
-func NewPapiStatusCmd() *cobra.Command {
-	cmdCapiStatus := &cobra.Command{
+func (cli cliPapi) NewStatusCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "status",
 		Short:             "Get status of the Polling API",
 		Args:              cobra.MinimumNArgs(0),
@@ -87,11 +93,11 @@ func NewPapiStatusCmd() *cobra.Command {
 		},
 	}
 
-	return cmdCapiStatus
+	return cmd
 }
 
-func NewPapiSyncCmd() *cobra.Command {
-	cmdCapiSync := &cobra.Command{
+func (cli cliPapi) NewSyncCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "sync",
 		Short:             "Sync with the Polling API, pulling all non-expired orders for the instance",
 		Args:              cobra.MinimumNArgs(0),
@@ -135,5 +141,5 @@ func NewPapiSyncCmd() *cobra.Command {
 		},
 	}
 
-	return cmdCapiSync
+	return cmd
 }

cmd/crowdsec-cli/parsers.go

@@ -1,194 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/fatih/color"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewParsersCmd() *cobra.Command {
-	var cmdParsers = &cobra.Command{
-		Use:   "parsers [action] [config]",
-		Short: "Install/Remove/Upgrade/Inspect parser(s) from hub",
-		Example: `cscli parsers install crowdsecurity/sshd-logs
-cscli parsers inspect crowdsecurity/sshd-logs
-cscli parsers upgrade crowdsecurity/sshd-logs
-cscli parsers list
-cscli parsers remove crowdsecurity/sshd-logs
-`,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"parser"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := require.Hub(csConfig); err != nil {
-				return err
-			}
-
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			if cmd.Name() == "inspect" || cmd.Name() == "list" {
-				return
-			}
-			log.Infof(ReloadMessage())
-		},
-	}
-
-	cmdParsers.AddCommand(NewParsersInstallCmd())
-	cmdParsers.AddCommand(NewParsersRemoveCmd())
-	cmdParsers.AddCommand(NewParsersUpgradeCmd())
-	cmdParsers.AddCommand(NewParsersInspectCmd())
-	cmdParsers.AddCommand(NewParsersListCmd())
-
-	return cmdParsers
-}
-
-func NewParsersInstallCmd() *cobra.Command {
-	var ignoreError bool
-
-	var cmdParsersInstall = &cobra.Command{
-		Use:               "install [config]",
-		Short:             "Install given parser(s)",
-		Long:              `Fetch and install given parser(s) from hub`,
-		Example:           `cscli parsers install crowdsec/xxx crowdsec/xyz`,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(cwhub.PARSERS, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			for _, name := range args {
-				t := cwhub.GetItem(cwhub.PARSERS, name)
-				if t == nil {
-					nearestItem, score := GetDistance(cwhub.PARSERS, name)
-					Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError)
-					continue
-				}
-				if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
-					if !ignoreError {
-						return fmt.Errorf("error while installing '%s': %w", name, err)
-					}
-					log.Errorf("Error while installing '%s': %s", name, err)
-				}
-			}
-			return nil
-		},
-	}
-
-	cmdParsersInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
-	cmdParsersInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
-	cmdParsersInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple parsers")
-
-	return cmdParsersInstall
-}
-
-func NewParsersRemoveCmd() *cobra.Command {
-	cmdParsersRemove := &cobra.Command{
-		Use:               "remove [config]",
-		Short:             "Remove given parser(s)",
-		Long:              `Remove given parse(s) from hub`,
-		Example:           `cscli parsers remove crowdsec/xxx crowdsec/xyz`,
-		Aliases:           []string{"delete"},
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, forceAction)
-				return nil
-			}
-
-			if len(args) == 0 {
-				return fmt.Errorf("specify at least one parser to remove or '--all'")
-			}
-
-			for _, name := range args {
-				cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, forceAction)
-			}
-
-			return nil
-		},
-	}
-
-	cmdParsersRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
-	cmdParsersRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
-	cmdParsersRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the parsers")
-
-	return cmdParsersRemove
-}
-
-func NewParsersUpgradeCmd() *cobra.Command {
-	cmdParsersUpgrade := &cobra.Command{
-		Use:               "upgrade [config]",
-		Short:             "Upgrade given parser(s)",
-		Long:              `Fetch and upgrade given parser(s) from hub`,
-		Example:           `cscli parsers upgrade crowdsec/xxx crowdsec/xyz`,
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
-			} else {
-				if len(args) == 0 {
-					return fmt.Errorf("specify at least one parser to upgrade or '--all'")
-				}
-				for _, name := range args {
-					cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, forceAction)
-				}
-			}
-			return nil
-		},
-	}
-
-	cmdParsersUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the parsers")
-	cmdParsersUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
-
-	return cmdParsersUpgrade
-}
-
-func NewParsersInspectCmd() *cobra.Command {
-	var cmdParsersInspect = &cobra.Command{
-		Use:               "inspect [name]",
-		Short:             "Inspect given parser",
-		Long:              `Inspect given parser`,
-		Example:           `cscli parsers inspect crowdsec/xxx`,
-		DisableAutoGenTag: true,
-		Args:              cobra.MinimumNArgs(1),
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS, args, toComplete)
-		},
-		Run: func(cmd *cobra.Command, args []string) {
-			InspectItem(args[0], cwhub.PARSERS)
-		},
-	}
-
-	cmdParsersInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
-
-	return cmdParsersInspect
-}
-
-func NewParsersListCmd() *cobra.Command {
-	var cmdParsersList = &cobra.Command{
-		Use:   "list [name]",
-		Short: "List all parsers or given one",
-		Long:  `List all parsers or given one`,
-		Example: `cscli parsers list
-cscli parser list crowdsecurity/xxx`,
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all)
-		},
-	}
-
-	cmdParsersList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
-
-	return cmdParsersList
-}

cmd/crowdsec-cli/postoverflows.go

@@ -1,191 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/fatih/color"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewPostOverflowsCmd() *cobra.Command {
-	cmdPostOverflows := &cobra.Command{
-		Use:   "postoverflows [action] [config]",
-		Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub",
-		Example: `cscli postoverflows install crowdsecurity/cdn-whitelist
-		cscli postoverflows inspect crowdsecurity/cdn-whitelist
-		cscli postoverflows upgrade crowdsecurity/cdn-whitelist
-		cscli postoverflows list
-		cscli postoverflows remove crowdsecurity/cdn-whitelist`,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"postoverflow"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := require.Hub(csConfig); err != nil {
-				return err
-			}
-
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			if cmd.Name() == "inspect" || cmd.Name() == "list" {
-				return
-			}
-			log.Infof(ReloadMessage())
-		},
-	}
-
-	cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd())
-	cmdPostOverflows.AddCommand(NewPostOverflowsListCmd())
-
-	return cmdPostOverflows
-}
-
-func NewPostOverflowsInstallCmd() *cobra.Command {
-	var ignoreError bool
-
-	cmdPostOverflowsInstall := &cobra.Command{
-		Use:               "install [config]",
-		Short:             "Install given postoverflow(s)",
-		Long:              `Fetch and install given postoverflow(s) from hub`,
-		Example:           `cscli postoverflows install crowdsec/xxx crowdsec/xyz`,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(cwhub.PARSERS_OVFLW, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			for _, name := range args {
-				t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name)
-				if t == nil {
-					nearestItem, score := GetDistance(cwhub.PARSERS_OVFLW, name)
-					Suggest(cwhub.PARSERS_OVFLW, name, nearestItem.Name, score, ignoreError)
-					continue
-				}
-				if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil {
-					if !ignoreError {
-						return fmt.Errorf("error while installing '%s': %w", name, err)
-					}
-					log.Errorf("Error while installing '%s': %s", name, err)
-				}
-			}
-			return nil
-		},
-	}
-
-	cmdPostOverflowsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
-	cmdPostOverflowsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
-	cmdPostOverflowsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple postoverflows")
-
-	return cmdPostOverflowsInstall
-}
-
-func NewPostOverflowsRemoveCmd() *cobra.Command {
-	cmdPostOverflowsRemove := &cobra.Command{
-		Use:               "remove [config]",
-		Short:             "Remove given postoverflow(s)",
-		Long:              `remove given postoverflow(s)`,
-		Example:           `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`,
-		Aliases:           []string{"delete"},
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, forceAction)
-				return nil
-			}
-
-			if len(args) == 0 {
-				return fmt.Errorf("specify at least one postoverflow to remove or '--all'")
-			}
-
-			for _, name := range args {
-				cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, forceAction)
-			}
-
-			return nil
-		},
-	}
-
-	cmdPostOverflowsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
-	cmdPostOverflowsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
-	cmdPostOverflowsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the postoverflows")
-
-	return cmdPostOverflowsRemove
-}
-
-func NewPostOverflowsUpgradeCmd() *cobra.Command {
-	cmdPostOverflowsUpgrade := &cobra.Command{
-		Use:               "upgrade [config]",
-		Short:             "Upgrade given postoverflow(s)",
-		Long:              `Fetch and Upgrade given postoverflow(s) from hub`,
-		Example:           `cscli postoverflows upgrade crowdsec/xxx crowdsec/xyz`,
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
-		},
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
-			} else {
-				if len(args) == 0 {
-					return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'")
-				}
-				for _, name := range args {
-					cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, forceAction)
-				}
-			}
-			return nil
-		},
-	}
-
-	cmdPostOverflowsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the postoverflows")
-	cmdPostOverflowsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
-
-	return cmdPostOverflowsUpgrade
-}
-
-func NewPostOverflowsInspectCmd() *cobra.Command {
-	cmdPostOverflowsInspect := &cobra.Command{
-		Use:               "inspect [config]",
-		Short:             "Inspect given postoverflow",
-		Long:              `Inspect given postoverflow`,
-		Example:           `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`,
-		DisableAutoGenTag: true,
-		Args:              cobra.MinimumNArgs(1),
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
-		},
-		Run: func(cmd *cobra.Command, args []string) {
-			InspectItem(args[0], cwhub.PARSERS_OVFLW)
-		},
-	}
-
-	return cmdPostOverflowsInspect
-}
-
-func NewPostOverflowsListCmd() *cobra.Command {
-	cmdPostOverflowsList := &cobra.Command{
-		Use:   "list [config]",
-		Short: "List all postoverflows or given one",
-		Long:  `List all postoverflows or given one`,
-		Example: `cscli postoverflows list
-cscli postoverflows list crowdsecurity/xxx`,
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all)
-		},
-	}
-
-	cmdPostOverflowsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
-
-	return cmdPostOverflowsList
-}

cmd/crowdsec-cli/require/branch.go

@@ -0,0 +1,58 @@
+package require
+
+// Set the appropriate hub branch according to config settings and crowdsec version
+
+import (
+	log "github.com/sirupsen/logrus"
+	"golang.org/x/mod/semver"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+)
+
+func chooseBranch(cfg *csconfig.Config) string {
+	// this was set from config.yaml or flag
+	if cfg.Cscli.HubBranch != "" {
+		log.Debugf("Hub override from config: branch '%s'", cfg.Cscli.HubBranch)
+		return cfg.Cscli.HubBranch
+	}
+
+	latest, err := cwversion.Latest()
+	if err != nil {
+		log.Warningf("Unable to retrieve latest crowdsec version: %s, using hub branch 'master'", err)
+		return "master"
+	}
+
+	csVersion := cwversion.VersionStrip()
+	if csVersion == latest {
+		log.Debugf("Latest crowdsec version (%s), using hub branch 'master'", csVersion)
+		return "master"
+	}
+
+	// if current version is greater than the latest we are in pre-release
+	if semver.Compare(csVersion, latest) == 1 {
+		log.Debugf("Your current crowdsec version seems to be a pre-release (%s), using hub branch 'master'", csVersion)
+		return "master"
+	}
+
+	if csVersion == "" {
+		log.Warning("Crowdsec version is not set, using hub branch 'master'")
+		return "master"
+	}
+
+	log.Warnf("A new CrowdSec release is available (%s). "+
+		"Your version is '%s'. Please update it to use new parsers/scenarios/collections.",
+		latest, csVersion)
+	return csVersion
+}
+
+
+// HubBranch sets the branch (in cscli config) and returns its value
+// It can be "master", or the branch corresponding to the current crowdsec version, or the value overridden in config/flag
+func HubBranch(cfg *csconfig.Config) string {
+	branch := chooseBranch(cfg)
+
+	cfg.Cscli.HubBranch = branch
+
+	return branch
+}

cmd/crowdsec-cli/require/require.go

@@ -2,6 +2,9 @@ package require
 
 import (
 	"fmt"
+	"io"
+
+	"github.com/sirupsen/logrus"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
@@ -23,6 +26,7 @@ func CAPI(c *csconfig.Config) error {
 	if c.API.Server.OnlineClient == nil {
 		return fmt.Errorf("no configuration for Central API (CAPI) in '%s'", *c.FilePath)
 	}
+
 	return nil
 }
 
@@ -30,6 +34,7 @@ func PAPI(c *csconfig.Config) error {
 	if c.API.Server.OnlineClient.Credentials.PapiURL == "" {
 		return fmt.Errorf("no PAPI URL in configuration")
 	}
+
 	return nil
 }
 
@@ -45,13 +50,6 @@ func DB(c *csconfig.Config) error {
 	if err := c.LoadDBConfig(); err != nil {
 		return fmt.Errorf("this command requires direct database access (must be run on the local API machine): %w", err)
 	}
-	return nil
-}
-
-func Profiles(c *csconfig.Config) error {
-	if err := c.API.Server.LoadProfiles(); err != nil {
-		return fmt.Errorf("while loading profiles: %w", err)
-	}
 
 	return nil
 }
@@ -64,20 +62,38 @@ func Notifications(c *csconfig.Config) error {
 	return nil
 }
 
-func Hub (c *csconfig.Config) error {
-	if err := c.LoadHub(); err != nil {
-		return err
+// RemoteHub returns the configuration required to download hub index and items: url, branch, etc.
+func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg {
+	// set branch in config, and log if necessary
+	branch := HubBranch(c)
+	remote := &cwhub.RemoteHubCfg{
+		Branch:      branch,
+		URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s",
+		// URLTemplate: "http://localhost:8000/crowdsecurity/%s/hub/%s",
+		IndexPath: ".index.json",
 	}
 
-	if c.Hub == nil {
-		return fmt.Errorf("you must configure cli before interacting with hub")
-	}
-
-	cwhub.SetHubBranch()
-
-	if err := cwhub.GetHubIdx(c.Hub); err != nil {
-		return fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err)
-	}
-
-	return nil
+	return remote
+}
+
+// Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items.
+// If no remote parameter is provided, the hub can only be used for local operations.
+func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg, logger *logrus.Logger) (*cwhub.Hub, error) {
+	local := c.Hub
+
+	if local == nil {
+		return nil, fmt.Errorf("you must configure cli before interacting with hub")
+	}
+
+	if logger == nil {
+		logger = logrus.New()
+		logger.SetOutput(io.Discard)
+	}
+
+	hub, err := cwhub.NewHub(local, remote, false, logger)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read Hub index: %w. Run 'sudo cscli hub update' to download the index again", err)
+	}
+
+	return hub, nil
 }

cmd/crowdsec-cli/scenarios.go

@@ -1,188 +0,0 @@
-package main
-
-import (
-	"fmt"
-
-	"github.com/fatih/color"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewScenariosCmd() *cobra.Command {
-	var cmdScenarios = &cobra.Command{
-		Use:   "scenarios [action] [config]",
-		Short: "Install/Remove/Upgrade/Inspect scenario(s) from hub",
-		Example: `cscli scenarios list [-a]
-cscli scenarios install crowdsecurity/ssh-bf
-cscli scenarios inspect crowdsecurity/ssh-bf
-cscli scenarios upgrade crowdsecurity/ssh-bf
-cscli scenarios remove crowdsecurity/ssh-bf
-`,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"scenario"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
-			if err := require.Hub(csConfig); err != nil {
-				return err
-			}
-
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-			if cmd.Name() == "inspect" || cmd.Name() == "list" {
-				return
-			}
-			log.Infof(ReloadMessage())
-		},
-	}
-
-	cmdScenarios.AddCommand(NewCmdScenariosInstall())
-	cmdScenarios.AddCommand(NewCmdScenariosRemove())
-	cmdScenarios.AddCommand(NewCmdScenariosUpgrade())
-	cmdScenarios.AddCommand(NewCmdScenariosInspect())
-	cmdScenarios.AddCommand(NewCmdScenariosList())
-
-	return cmdScenarios
-}
-
-func NewCmdScenariosInstall() *cobra.Command {
-	var ignoreError bool
-
-	var cmdScenariosInstall = &cobra.Command{
-		Use:     "install [config]",
-		Short:   "Install given scenario(s)",
-		Long:    `Fetch and install given scenario(s) from hub`,
-		Example: `cscli scenarios install crowdsec/xxx crowdsec/xyz`,
-		Args:    cobra.MinimumNArgs(1),
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(cwhub.SCENARIOS, args, toComplete)
-		},
-		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			for _, name := range args {
-				t := cwhub.GetItem(cwhub.SCENARIOS, name)
-				if t == nil {
-					nearestItem, score := GetDistance(cwhub.SCENARIOS, name)
-					Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError)
-					continue
-				}
-				if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
-					if !ignoreError {
-						return fmt.Errorf("error while installing '%s': %w", name, err)
-					}
-					log.Errorf("Error while installing '%s': %s", name, err)
-				}
-			}
-			return nil
-		},
-	}
-	cmdScenariosInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
-	cmdScenariosInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
-	cmdScenariosInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple scenarios")
-
-	return cmdScenariosInstall
-}
-
-func NewCmdScenariosRemove() *cobra.Command {
-	var cmdScenariosRemove = &cobra.Command{
-		Use:     "remove [config]",
-		Short:   "Remove given scenario(s)",
-		Long:    `remove given scenario(s)`,
-		Example: `cscli scenarios remove crowdsec/xxx crowdsec/xyz`,
-		Aliases: []string{"delete"},
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
-		},
-		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, forceAction)
-				return nil
-			}
-
-			if len(args) == 0 {
-				return fmt.Errorf("specify at least one scenario to remove or '--all'")
-			}
-
-			for _, name := range args {
-				cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, forceAction)
-			}
-			return nil
-		},
-	}
-	cmdScenariosRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
-	cmdScenariosRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
-	cmdScenariosRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the scenarios")
-
-	return cmdScenariosRemove
-}
-
-func NewCmdScenariosUpgrade() *cobra.Command {
-	var cmdScenariosUpgrade = &cobra.Command{
-		Use:     "upgrade [config]",
-		Short:   "Upgrade given scenario(s)",
-		Long:    `Fetch and Upgrade given scenario(s) from hub`,
-		Example: `cscli scenarios upgrade crowdsec/xxx crowdsec/xyz`,
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
-		},
-		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if all {
-				cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
-			} else {
-				if len(args) == 0 {
-					return fmt.Errorf("specify at least one scenario to upgrade or '--all'")
-				}
-				for _, name := range args {
-					cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, forceAction)
-				}
-			}
-			return nil
-		},
-	}
-	cmdScenariosUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the scenarios")
-	cmdScenariosUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
-
-	return cmdScenariosUpgrade
-}
-
-func NewCmdScenariosInspect() *cobra.Command {
-	var cmdScenariosInspect = &cobra.Command{
-		Use:     "inspect [config]",
-		Short:   "Inspect given scenario",
-		Long:    `Inspect given scenario`,
-		Example: `cscli scenarios inspect crowdsec/xxx`,
-		Args:    cobra.MinimumNArgs(1),
-		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
-		},
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			InspectItem(args[0], cwhub.SCENARIOS)
-		},
-	}
-	cmdScenariosInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
-
-	return cmdScenariosInspect
-}
-
-func NewCmdScenariosList() *cobra.Command {
-	var cmdScenariosList = &cobra.Command{
-		Use:   "list [config]",
-		Short: "List all scenario(s) or given one",
-		Long:  `List all scenario(s) or given one`,
-		Example: `cscli scenarios list
-cscli scenarios list crowdsecurity/xxx`,
-		DisableAutoGenTag: true,
-		Run: func(cmd *cobra.Command, args []string) {
-			ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all)
-		},
-	}
-	cmdScenariosList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
-
-	return cmdScenariosList
-}

cmd/crowdsec-cli/setup.go

@@ -6,11 +6,12 @@ import (
 	"os"
 	"os/exec"
 
+	goccyyaml "github.com/goccy/go-yaml"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v3"
-	goccyyaml "github.com/goccy/go-yaml"
 
+	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/setup"
 )
@@ -303,7 +304,12 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("while reading file %s: %w", fromFile, err)
 	}
 
-	if err = setup.InstallHubItems(csConfig, input, dryRun); err != nil {
+	hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), log.StandardLogger())
+	if err != nil {
+		return err
+	}
+
+	if err = setup.InstallHubItems(hub, input, dryRun); err != nil {
 		return err
 	}
 

cmd/crowdsec-cli/simulation.go

@@ -3,105 +3,24 @@ package main
 import (
 	"fmt"
 	"os"
-	"slices"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v2"
+	"slices"
 
 	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-func addToExclusion(name string) error {
-	csConfig.Cscli.SimulationConfig.Exclusions = append(csConfig.Cscli.SimulationConfig.Exclusions, name)
-	return nil
+type cliSimulation struct{}
+
+func NewCLISimulation() *cliSimulation {
+	return &cliSimulation{}
 }
 
-func removeFromExclusion(name string) error {
-	index := slices.Index(csConfig.Cscli.SimulationConfig.Exclusions, name)
-
-	// Remove element from the slice
-	csConfig.Cscli.SimulationConfig.Exclusions[index] = csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
-	csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1] = ""
-	csConfig.Cscli.SimulationConfig.Exclusions = csConfig.Cscli.SimulationConfig.Exclusions[:len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
-
-	return nil
-}
-
-func enableGlobalSimulation() error {
-	csConfig.Cscli.SimulationConfig.Simulation = new(bool)
-	*csConfig.Cscli.SimulationConfig.Simulation = true
-	csConfig.Cscli.SimulationConfig.Exclusions = []string{}
-
-	if err := dumpSimulationFile(); err != nil {
-		log.Fatalf("unable to dump simulation file: %s", err)
-	}
-
-	log.Printf("global simulation: enabled")
-
-	return nil
-}
-
-func dumpSimulationFile() error {
-	newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
-	if err != nil {
-		return fmt.Errorf("unable to marshal simulation configuration: %s", err)
-	}
-	err = os.WriteFile(csConfig.ConfigPaths.SimulationFilePath, newConfigSim, 0644)
-	if err != nil {
-		return fmt.Errorf("write simulation config in '%s' failed: %s", csConfig.ConfigPaths.SimulationFilePath, err)
-	}
-	log.Debugf("updated simulation file %s", csConfig.ConfigPaths.SimulationFilePath)
-
-	return nil
-}
-
-func disableGlobalSimulation() error {
-	csConfig.Cscli.SimulationConfig.Simulation = new(bool)
-	*csConfig.Cscli.SimulationConfig.Simulation = false
-
-	csConfig.Cscli.SimulationConfig.Exclusions = []string{}
-	newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
-	if err != nil {
-		return fmt.Errorf("unable to marshal new simulation configuration: %s", err)
-	}
-	err = os.WriteFile(csConfig.ConfigPaths.SimulationFilePath, newConfigSim, 0644)
-	if err != nil {
-		return fmt.Errorf("unable to write new simulation config in '%s' : %s", csConfig.ConfigPaths.SimulationFilePath, err)
-	}
-
-	log.Printf("global simulation: disabled")
-	return nil
-}
-
-func simulationStatus() error {
-	if csConfig.Cscli.SimulationConfig == nil {
-		log.Printf("global simulation: disabled (configuration file is missing)")
-		return nil
-	}
-	if *csConfig.Cscli.SimulationConfig.Simulation {
-		log.Println("global simulation: enabled")
-		if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
-			log.Println("Scenarios not in simulation mode :")
-			for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
-				log.Printf("  - %s", scenario)
-			}
-		}
-	} else {
-		log.Println("global simulation: disabled")
-		if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
-			log.Println("Scenarios in simulation mode :")
-			for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
-				log.Printf("  - %s", scenario)
-			}
-		}
-	}
-	return nil
-}
-
-func NewSimulationCmds() *cobra.Command {
-	var cmdSimulation = &cobra.Command{
+func (cli cliSimulation) NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:   "simulation [command]",
 		Short: "Manage simulation status of scenarios",
 		Example: `cscli simulation status
@@ -112,9 +31,6 @@ cscli simulation disable crowdsecurity/ssh-bf`,
 			if err := csConfig.LoadSimulation(); err != nil {
 				log.Fatal(err)
 			}
-			if csConfig.Cscli == nil {
-				return fmt.Errorf("you must configure cli before using simulation")
-			}
 			if csConfig.Cscli.SimulationConfig == nil {
 				return fmt.Errorf("no simulation configured")
 			}
@@ -126,37 +42,38 @@ cscli simulation disable crowdsecurity/ssh-bf`,
 			}
 		},
 	}
-	cmdSimulation.Flags().SortFlags = false
-	cmdSimulation.PersistentFlags().SortFlags = false
+	cmd.Flags().SortFlags = false
+	cmd.PersistentFlags().SortFlags = false
 
-	cmdSimulation.AddCommand(NewSimulationEnableCmd())
-	cmdSimulation.AddCommand(NewSimulationDisableCmd())
-	cmdSimulation.AddCommand(NewSimulationStatusCmd())
+	cmd.AddCommand(cli.NewEnableCmd())
+	cmd.AddCommand(cli.NewDisableCmd())
+	cmd.AddCommand(cli.NewStatusCmd())
 
-	return cmdSimulation
+	return cmd
 }
 
-func NewSimulationEnableCmd() *cobra.Command {
+func (cli cliSimulation) NewEnableCmd() *cobra.Command {
 	var forceGlobalSimulation bool
 
-	var cmdSimulationEnable = &cobra.Command{
+	cmd := &cobra.Command{
 		Use:               "enable [scenario] [-global]",
 		Short:             "Enable the simulation, globally or on specified scenarios",
 		Example:           `cscli simulation enable`,
 		DisableAutoGenTag: true,
 		Run: func(cmd *cobra.Command, args []string) {
-			if err := require.Hub(csConfig); err != nil {
+			hub, err := require.Hub(csConfig, nil, nil)
+			if err != nil {
 				log.Fatal(err)
 			}
 
 			if len(args) > 0 {
 				for _, scenario := range args {
-					var item = cwhub.GetItem(cwhub.SCENARIOS, scenario)
+					var item = hub.GetItem(cwhub.SCENARIOS, scenario)
 					if item == nil {
 						log.Errorf("'%s' doesn't exist or is not a scenario", scenario)
 						continue
 					}
-					if !item.Installed {
+					if !item.State.Installed {
 						log.Warningf("'%s' isn't enabled", scenario)
 					}
 					isExcluded := slices.Contains(csConfig.Cscli.SimulationConfig.Exclusions, scenario)
@@ -192,15 +109,15 @@ func NewSimulationEnableCmd() *cobra.Command {
 			}
 		},
 	}
-	cmdSimulationEnable.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Enable global simulation (reverse mode)")
+	cmd.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Enable global simulation (reverse mode)")
 
-	return cmdSimulationEnable
+	return cmd
 }
 
-func NewSimulationDisableCmd() *cobra.Command {
+func (cli cliSimulation) NewDisableCmd() *cobra.Command {
 	var forceGlobalSimulation bool
 
-	var cmdSimulationDisable = &cobra.Command{
+	cmd := &cobra.Command{
 		Use:               "disable [scenario]",
 		Short:             "Disable the simulation mode. Disable only specified scenarios",
 		Example:           `cscli simulation disable`,
@@ -241,13 +158,13 @@ func NewSimulationDisableCmd() *cobra.Command {
 			}
 		},
 	}
-	cmdSimulationDisable.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Disable global simulation (reverse mode)")
+	cmd.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Disable global simulation (reverse mode)")
 
-	return cmdSimulationDisable
+	return cmd
 }
 
-func NewSimulationStatusCmd() *cobra.Command {
-	var cmdSimulationStatus = &cobra.Command{
+func (cli cliSimulation) NewStatusCmd() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "status",
 		Short:             "Show simulation mode status",
 		Example:           `cscli simulation status`,
@@ -261,5 +178,92 @@ func NewSimulationStatusCmd() *cobra.Command {
 		},
 	}
 
-	return cmdSimulationStatus
+	return cmd
+}
+
+func addToExclusion(name string) error {
+	csConfig.Cscli.SimulationConfig.Exclusions = append(csConfig.Cscli.SimulationConfig.Exclusions, name)
+	return nil
+}
+
+func removeFromExclusion(name string) error {
+	index := slices.Index(csConfig.Cscli.SimulationConfig.Exclusions, name)
+
+	// Remove element from the slice
+	csConfig.Cscli.SimulationConfig.Exclusions[index] = csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
+	csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1] = ""
+	csConfig.Cscli.SimulationConfig.Exclusions = csConfig.Cscli.SimulationConfig.Exclusions[:len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
+
+	return nil
+}
+
+func enableGlobalSimulation() error {
+	csConfig.Cscli.SimulationConfig.Simulation = new(bool)
+	*csConfig.Cscli.SimulationConfig.Simulation = true
+	csConfig.Cscli.SimulationConfig.Exclusions = []string{}
+
+	if err := dumpSimulationFile(); err != nil {
+		log.Fatalf("unable to dump simulation file: %s", err)
+	}
+
+	log.Printf("global simulation: enabled")
+
+	return nil
+}
+
+func dumpSimulationFile() error {
+	newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
+	if err != nil {
+		return fmt.Errorf("unable to marshal simulation configuration: %s", err)
+	}
+	err = os.WriteFile(csConfig.ConfigPaths.SimulationFilePath, newConfigSim, 0o644)
+	if err != nil {
+		return fmt.Errorf("write simulation config in '%s' failed: %s", csConfig.ConfigPaths.SimulationFilePath, err)
+	}
+	log.Debugf("updated simulation file %s", csConfig.ConfigPaths.SimulationFilePath)
+
+	return nil
+}
+
+func disableGlobalSimulation() error {
+	csConfig.Cscli.SimulationConfig.Simulation = new(bool)
+	*csConfig.Cscli.SimulationConfig.Simulation = false
+
+	csConfig.Cscli.SimulationConfig.Exclusions = []string{}
+	newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
+	if err != nil {
+		return fmt.Errorf("unable to marshal new simulation configuration: %s", err)
+	}
+	err = os.WriteFile(csConfig.ConfigPaths.SimulationFilePath, newConfigSim, 0o644)
+	if err != nil {
+		return fmt.Errorf("unable to write new simulation config in '%s' : %s", csConfig.ConfigPaths.SimulationFilePath, err)
+	}
+
+	log.Printf("global simulation: disabled")
+	return nil
+}
+
+func simulationStatus() error {
+	if csConfig.Cscli.SimulationConfig == nil {
+		log.Printf("global simulation: disabled (configuration file is missing)")
+		return nil
+	}
+	if *csConfig.Cscli.SimulationConfig.Simulation {
+		log.Println("global simulation: enabled")
+		if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
+			log.Println("Scenarios not in simulation mode :")
+			for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
+				log.Printf("  - %s", scenario)
+			}
+		}
+	} else {
+		log.Println("global simulation: disabled")
+		if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
+			log.Println("Scenarios in simulation mode :")
+			for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
+				log.Printf("  - %s", scenario)
+			}
+		}
+	}
+	return nil
 }

cmd/crowdsec-cli/support.go

@@ -37,6 +37,7 @@ const (
 	SUPPORT_OS_INFO_PATH                 = "osinfo.txt"
 	SUPPORT_PARSERS_PATH                 = "hub/parsers.txt"
 	SUPPORT_SCENARIOS_PATH               = "hub/scenarios.txt"
+	SUPPORT_CONTEXTS_PATH                = "hub/scenarios.txt"
 	SUPPORT_COLLECTIONS_PATH             = "hub/collections.txt"
 	SUPPORT_POSTOVERFLOWS_PATH           = "hub/postoverflows.txt"
 	SUPPORT_BOUNCERS_PATH                = "lapi/bouncers.txt"
@@ -58,10 +59,6 @@ func stripAnsiString(str string) string {
 
 func collectMetrics() ([]byte, []byte, error) {
 	log.Info("Collecting prometheus metrics")
-	err := csConfig.LoadPrometheus()
-	if err != nil {
-		return nil, nil, err
-	}
 
 	if csConfig.Cscli.PrometheusUrl == "" {
 		log.Warn("No Prometheus URL configured, metrics will not be collected")
@@ -69,13 +66,13 @@ func collectMetrics() ([]byte, []byte, error) {
 	}
 
 	humanMetrics := bytes.NewBuffer(nil)
-	err = FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl+"/metrics", "human")
+	err := FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl, "human")
 
 	if err != nil {
 		return nil, nil, fmt.Errorf("could not fetch promtheus metrics: %s", err)
 	}
 
-	req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl+"/metrics", nil)
+	req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl, nil)
 	if err != nil {
 		return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err)
 	}
@@ -132,10 +129,21 @@ func collectOSInfo() ([]byte, error) {
 	return w.Bytes(), nil
 }
 
-func collectHubItems(itemType string) []byte {
+func collectHubItems(hub *cwhub.Hub, itemType string) []byte {
+	var err error
+
 	out := bytes.NewBuffer(nil)
 	log.Infof("Collecting %s list", itemType)
-	ListItems(out, []string{itemType}, []string{}, false, true, all)
+
+	items := make(map[string][]*cwhub.Item)
+
+	if items[itemType], err = selectItems(hub, itemType, nil, true); err != nil {
+		log.Warnf("could not collect %s list: %s", itemType, err)
+	}
+
+	if err := listItems(out, []string{itemType}, items, false); err != nil {
+		log.Warnf("could not collect %s list: %s", itemType, err)
+	}
 	return out.Bytes()
 }
 
@@ -157,7 +165,7 @@ func collectAgents(dbClient *database.Client) ([]byte, error) {
 	return out.Bytes(), nil
 }
 
-func collectAPIStatus(login string, password string, endpoint string, prefix string) []byte {
+func collectAPIStatus(login string, password string, endpoint string, prefix string, hub *cwhub.Hub) []byte {
 	if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil {
 		return []byte("No agent credentials found, are we LAPI ?")
 	}
@@ -167,7 +175,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
 	if err != nil {
 		return []byte(fmt.Sprintf("cannot parse API URL: %s", err))
 	}
-	scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
+	scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
 	if err != nil {
 		return []byte(fmt.Sprintf("could not collect scenarios: %s", err))
 	}
@@ -230,8 +238,14 @@ func collectAcquisitionConfig() map[string][]byte {
 	return ret
 }
 
-func NewSupportCmd() *cobra.Command {
-	var cmdSupport = &cobra.Command{
+type cliSupport struct{}
+
+func NewCLISupport() *cliSupport {
+	return &cliSupport{}
+}
+
+func (cli cliSupport) NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
 		Use:               "support [action]",
 		Short:             "Provide commands to help during support",
 		Args:              cobra.MinimumNArgs(1),
@@ -241,9 +255,15 @@ func NewSupportCmd() *cobra.Command {
 		},
 	}
 
+	cmd.AddCommand(cli.NewDumpCmd())
+
+	return cmd
+}
+
+func (cli cliSupport) NewDumpCmd() *cobra.Command {
 	var outFile string
 
-	cmdDump := &cobra.Command{
+	cmd := &cobra.Command{
 		Use:   "dump",
 		Short: "Dump all your configuration to a zip file for easier support",
 		Long: `Dump the following informations:
@@ -253,6 +273,7 @@ func NewSupportCmd() *cobra.Command {
 - Installed parsers list
 - Installed scenarios list
 - Installed postoverflows list
+- Installed context list
 - Bouncers list
 - Machines list
 - CAPI status
@@ -295,12 +316,14 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				skipAgent = true
 			}
 
-			if err := require.Hub(csConfig); err != nil {
+			hub, err := require.Hub(csConfig, nil, nil)
+			if err != nil {
 				log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
 				skipHub = true
 				infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())
 				infos[SUPPORT_SCENARIOS_PATH] = []byte(err.Error())
 				infos[SUPPORT_POSTOVERFLOWS_PATH] = []byte(err.Error())
+				infos[SUPPORT_CONTEXTS_PATH] = []byte(err.Error())
 				infos[SUPPORT_COLLECTIONS_PATH] = []byte(err.Error())
 			}
 
@@ -333,10 +356,11 @@ cscli support dump -f /tmp/crowdsec-support.zip
 			infos[SUPPORT_CROWDSEC_CONFIG_PATH] = collectCrowdsecConfig()
 
 			if !skipHub {
-				infos[SUPPORT_PARSERS_PATH] = collectHubItems(cwhub.PARSERS)
-				infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(cwhub.SCENARIOS)
-				infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.PARSERS_OVFLW)
-				infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(cwhub.COLLECTIONS)
+				infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS)
+				infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS)
+				infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
+				infos[SUPPORT_CONTEXTS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
+				infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS)
 			}
 
 			if !skipDB {
@@ -358,7 +382,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				infos[SUPPORT_CAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Server.OnlineClient.Credentials.Login,
 					csConfig.API.Server.OnlineClient.Credentials.Password,
 					csConfig.API.Server.OnlineClient.Credentials.URL,
-					CAPIURLPrefix)
+					CAPIURLPrefix,
+					hub)
 			}
 
 			if !skipLAPI {
@@ -366,7 +391,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				infos[SUPPORT_LAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Client.Credentials.Login,
 					csConfig.API.Client.Credentials.Password,
 					csConfig.API.Client.Credentials.URL,
-					LAPIURLPrefix)
+					LAPIURLPrefix,
+					hub)
 				infos[SUPPORT_CROWDSEC_PROFILE_PATH] = collectCrowdsecProfile()
 			}
 
@@ -397,7 +423,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				log.Fatalf("could not finalize zip file: %s", err)
 			}
 
-			err = os.WriteFile(outFile, w.Bytes(), 0600)
+			err = os.WriteFile(outFile, w.Bytes(), 0o600)
 			if err != nil {
 				log.Fatalf("could not write zip file to %s: %s", outFile, err)
 			}
@@ -405,8 +431,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
 			log.Infof("Written zip file to %s", outFile)
 		},
 	}
-	cmdDump.Flags().StringVarP(&outFile, "outFile", "f", "", "File to dump the information to")
-	cmdSupport.AddCommand(cmdDump)
 
-	return cmdSupport
+	cmd.Flags().StringVarP(&outFile, "outFile", "f", "", "File to dump the information to")
+
+	return cmd
 }

cmd/crowdsec-cli/utils.go

@@ -1,236 +1,24 @@
 package main
 
 import (
-	"encoding/csv"
-	"encoding/json"
 	"fmt"
-	"io"
-	"math"
 	"net"
-	"net/http"
-	"slices"
-	"strconv"
 	"strings"
-	"time"
 
-	"github.com/fatih/color"
-	dto "github.com/prometheus/client_model/go"
-	"github.com/prometheus/prom2json"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"github.com/agext/levenshtein"
-	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/trace"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
-const MaxDistance = 7
-
 func printHelp(cmd *cobra.Command) {
-	err := cmd.Help()
-	if err != nil {
+	if err := cmd.Help(); err != nil {
 		log.Fatalf("unable to print help(): %s", err)
 	}
 }
 
-func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
-	errMsg := ""
-	if score < MaxDistance {
-		errMsg = fmt.Sprintf("unable to find %s '%s', did you mean %s ?", itemType, baseItem, suggestItem)
-	} else {
-		errMsg = fmt.Sprintf("unable to find %s '%s'", itemType, baseItem)
-	}
-	if ignoreErr {
-		log.Error(errMsg)
-	} else {
-		log.Fatalf(errMsg)
-	}
-}
-
-func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
-	allItems := make([]string, 0)
-	nearestScore := 100
-	nearestItem := &cwhub.Item{}
-	hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
-	for _, item := range hubItems {
-		allItems = append(allItems, item.Name)
-	}
-
-	for _, s := range allItems {
-		d := levenshtein.Distance(itemName, s, nil)
-		if d < nearestScore {
-			nearestScore = d
-			nearestItem = cwhub.GetItem(itemType, s)
-		}
-	}
-	return nearestItem, nearestScore
-}
-
-func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	if err := require.Hub(csConfig); err != nil {
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
-	comp := make([]string, 0)
-	hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
-	for _, item := range hubItems {
-		if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
-			comp = append(comp, item.Name)
-		}
-	}
-	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
-	return comp, cobra.ShellCompDirectiveNoFileComp
-}
-
-func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	if err := require.Hub(csConfig); err != nil {
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
-	items, err := cwhub.GetInstalledItemsAsString(itemType)
-	if err != nil {
-		cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
-	comp := make([]string, 0)
-
-	if toComplete != "" {
-		for _, item := range items {
-			if strings.Contains(item, toComplete) {
-				comp = append(comp, item)
-			}
-		}
-	} else {
-		comp = items
-	}
-
-	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
-
-	return comp, cobra.ShellCompDirectiveNoFileComp
-}
-
-func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) {
-	var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus)
-
-	for _, itemType := range itemTypes {
-		itemName := ""
-		if len(args) == 1 {
-			itemName = args[0]
-		}
-		hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all)
-	}
-
-	if csConfig.Cscli.Output == "human" {
-		for _, itemType := range itemTypes {
-			var statuses []cwhub.ItemHubStatus
-			var ok bool
-			if statuses, ok = hubStatusByItemType[itemType]; !ok {
-				log.Errorf("unknown item type: %s", itemType)
-				continue
-			}
-			listHubItemTable(out, "\n"+strings.ToUpper(itemType), statuses)
-		}
-	} else if csConfig.Cscli.Output == "json" {
-		x, err := json.MarshalIndent(hubStatusByItemType, "", " ")
-		if err != nil {
-			log.Fatalf("failed to unmarshal")
-		}
-		out.Write(x)
-	} else if csConfig.Cscli.Output == "raw" {
-		csvwriter := csv.NewWriter(out)
-		if showHeader {
-			header := []string{"name", "status", "version", "description"}
-			if showType {
-				header = append(header, "type")
-			}
-			err := csvwriter.Write(header)
-			if err != nil {
-				log.Fatalf("failed to write header: %s", err)
-			}
-
-		}
-		for _, itemType := range itemTypes {
-			var statuses []cwhub.ItemHubStatus
-			var ok bool
-			if statuses, ok = hubStatusByItemType[itemType]; !ok {
-				log.Errorf("unknown item type: %s", itemType)
-				continue
-			}
-			for _, status := range statuses {
-				if status.LocalVersion == "" {
-					status.LocalVersion = "n/a"
-				}
-				row := []string{
-					status.Name,
-					status.Status,
-					status.LocalVersion,
-					status.Description,
-				}
-				if showType {
-					row = append(row, itemType)
-				}
-				err := csvwriter.Write(row)
-				if err != nil {
-					log.Fatalf("failed to write raw output : %s", err)
-				}
-			}
-		}
-		csvwriter.Flush()
-	}
-}
-
-func InspectItem(name string, objecitemType string) {
-
-	hubItem := cwhub.GetItem(objecitemType, name)
-	if hubItem == nil {
-		log.Fatalf("unable to retrieve item.")
-	}
-	var b []byte
-	var err error
-	switch csConfig.Cscli.Output {
-	case "human", "raw":
-		b, err = yaml.Marshal(*hubItem)
-		if err != nil {
-			log.Fatalf("unable to marshal item : %s", err)
-		}
-	case "json":
-		b, err = json.MarshalIndent(*hubItem, "", " ")
-		if err != nil {
-			log.Fatalf("unable to marshal item : %s", err)
-		}
-	}
-	fmt.Printf("%s", string(b))
-	if csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
-		return
-	}
-
-	if prometheusURL == "" {
-		//This is technically wrong to do this, as the prometheus section contains a listen address, not an URL to query prometheus
-		//But for ease of use, we will use the listen address as the prometheus URL because it will be 127.0.0.1 in the default case
-		listenAddr := csConfig.Prometheus.ListenAddr
-		if listenAddr == "" {
-			listenAddr = "127.0.0.1"
-		}
-		listenPort := csConfig.Prometheus.ListenPort
-		if listenPort == 0 {
-			listenPort = 6060
-		}
-		prometheusURL = fmt.Sprintf("http://%s:%d/metrics", listenAddr, listenPort)
-		log.Debugf("No prometheus URL provided using: %s", prometheusURL)
-	}
-
-	fmt.Printf("\nCurrent metrics : \n")
-	ShowMetrics(hubItem)
-}
-
 func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
-
 	/*if a range is provided, change the scope*/
 	if *ipRange != "" {
 		_, _, err := net.ParseCIDR(*ipRange)
@@ -259,234 +47,7 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
 	return nil
 }
 
-func ShowMetrics(hubItem *cwhub.Item) {
-	switch hubItem.Type {
-	case cwhub.PARSERS:
-		metrics := GetParserMetric(prometheusURL, hubItem.Name)
-		parserMetricsTable(color.Output, hubItem.Name, metrics)
-	case cwhub.SCENARIOS:
-		metrics := GetScenarioMetric(prometheusURL, hubItem.Name)
-		scenarioMetricsTable(color.Output, hubItem.Name, metrics)
-	case cwhub.COLLECTIONS:
-		for _, item := range hubItem.Parsers {
-			metrics := GetParserMetric(prometheusURL, item)
-			parserMetricsTable(color.Output, item, metrics)
-		}
-		for _, item := range hubItem.Scenarios {
-			metrics := GetScenarioMetric(prometheusURL, item)
-			scenarioMetricsTable(color.Output, item, metrics)
-		}
-		for _, item := range hubItem.Collections {
-			hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
-			if hubItem == nil {
-				log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
-			}
-			ShowMetrics(hubItem)
-		}
-	default:
-		log.Errorf("item of type '%s' is unknown", hubItem.Type)
-	}
-}
-
-// GetParserMetric is a complete rip from prom2json
-func GetParserMetric(url string, itemName string) map[string]map[string]int {
-	stats := make(map[string]map[string]int)
-
-	result := GetPrometheusMetric(url)
-	for idx, fam := range result {
-		if !strings.HasPrefix(fam.Name, "cs_") {
-			continue
-		}
-		log.Tracef("round %d", idx)
-		for _, m := range fam.Metrics {
-			metric, ok := m.(prom2json.Metric)
-			if !ok {
-				log.Debugf("failed to convert metric to prom2json.Metric")
-				continue
-			}
-			name, ok := metric.Labels["name"]
-			if !ok {
-				log.Debugf("no name in Metric %v", metric.Labels)
-			}
-			if name != itemName {
-				continue
-			}
-			source, ok := metric.Labels["source"]
-			if !ok {
-				log.Debugf("no source in Metric %v", metric.Labels)
-			} else {
-				if srctype, ok := metric.Labels["type"]; ok {
-					source = srctype + ":" + source
-				}
-			}
-			value := m.(prom2json.Metric).Value
-			fval, err := strconv.ParseFloat(value, 32)
-			if err != nil {
-				log.Errorf("Unexpected int value %s : %s", value, err)
-				continue
-			}
-			ival := int(fval)
-
-			switch fam.Name {
-			case "cs_reader_hits_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-					stats[source]["parsed"] = 0
-					stats[source]["reads"] = 0
-					stats[source]["unparsed"] = 0
-					stats[source]["hits"] = 0
-				}
-				stats[source]["reads"] += ival
-			case "cs_parser_hits_ok_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["parsed"] += ival
-			case "cs_parser_hits_ko_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["unparsed"] += ival
-			case "cs_node_hits_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["hits"] += ival
-			case "cs_node_hits_ok_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["parsed"] += ival
-			case "cs_node_hits_ko_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["unparsed"] += ival
-			default:
-				continue
-			}
-		}
-	}
-	return stats
-}
-
-func GetScenarioMetric(url string, itemName string) map[string]int {
-	stats := make(map[string]int)
-
-	stats["instantiation"] = 0
-	stats["curr_count"] = 0
-	stats["overflow"] = 0
-	stats["pour"] = 0
-	stats["underflow"] = 0
-
-	result := GetPrometheusMetric(url)
-	for idx, fam := range result {
-		if !strings.HasPrefix(fam.Name, "cs_") {
-			continue
-		}
-		log.Tracef("round %d", idx)
-		for _, m := range fam.Metrics {
-			metric, ok := m.(prom2json.Metric)
-			if !ok {
-				log.Debugf("failed to convert metric to prom2json.Metric")
-				continue
-			}
-			name, ok := metric.Labels["name"]
-			if !ok {
-				log.Debugf("no name in Metric %v", metric.Labels)
-			}
-			if name != itemName {
-				continue
-			}
-			value := m.(prom2json.Metric).Value
-			fval, err := strconv.ParseFloat(value, 32)
-			if err != nil {
-				log.Errorf("Unexpected int value %s : %s", value, err)
-				continue
-			}
-			ival := int(fval)
-
-			switch fam.Name {
-			case "cs_bucket_created_total":
-				stats["instantiation"] += ival
-			case "cs_buckets":
-				stats["curr_count"] += ival
-			case "cs_bucket_overflowed_total":
-				stats["overflow"] += ival
-			case "cs_bucket_poured_total":
-				stats["pour"] += ival
-			case "cs_bucket_underflowed_total":
-				stats["underflow"] += ival
-			default:
-				continue
-			}
-		}
-	}
-	return stats
-}
-
-func GetPrometheusMetric(url string) []*prom2json.Family {
-	mfChan := make(chan *dto.MetricFamily, 1024)
-
-	// Start with the DefaultTransport for sane defaults.
-	transport := http.DefaultTransport.(*http.Transport).Clone()
-	// Conservatively disable HTTP keep-alives as this program will only
-	// ever need a single HTTP request.
-	transport.DisableKeepAlives = true
-	// Timeout early if the server doesn't even return the headers.
-	transport.ResponseHeaderTimeout = time.Minute
-
-	go func() {
-		defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
-		err := prom2json.FetchMetricFamilies(url, mfChan, transport)
-		if err != nil {
-			log.Fatalf("failed to fetch prometheus metrics : %v", err)
-		}
-	}()
-
-	result := []*prom2json.Family{}
-	for mf := range mfChan {
-		result = append(result, prom2json.NewFamily(mf))
-	}
-	log.Debugf("Finished reading prometheus output, %d entries", len(result))
-
-	return result
-}
-
-type unit struct {
-	value  int64
-	symbol string
-}
-
-var ranges = []unit{
-	{value: 1e18, symbol: "E"},
-	{value: 1e15, symbol: "P"},
-	{value: 1e12, symbol: "T"},
-	{value: 1e9,  symbol: "G"},
-	{value: 1e6,  symbol: "M"},
-	{value: 1e3,  symbol: "k"},
-	{value: 1,    symbol: ""},
-}
-
-func formatNumber(num int) string {
-	goodUnit := unit{}
-	for _, u := range ranges {
-		if int64(num) >= u.value {
-			goodUnit = u
-			break
-		}
-	}
-
-	if goodUnit.value == 1 {
-		return fmt.Sprintf("%d%s", num, goodUnit.symbol)
-	}
-
-	res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
-	return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
-}
-
 func getDBClient() (*database.Client, error) {
-	var err error
 	if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
 		return nil, err
 	}
@@ -518,5 +79,4 @@ func removeFromSlice(val string, slice []string) []string {
 	}
 
 	return slice
-
 }

cmd/crowdsec-cli/utils_table.go

@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"io"
+	"strconv"
 
 	"github.com/aquasecurity/table"
 	"github.com/enescakir/emoji"
@@ -10,19 +11,33 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatus) {
+func listHubItemTable(out io.Writer, title string, items []*cwhub.Item) {
 	t := newLightTable(out)
 	t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path")
 	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
-	for _, status := range statuses {
-		t.AddRow(status.Name, status.UTF8Status, status.LocalVersion, status.LocalPath)
+	for _, item := range items {
+		status := fmt.Sprintf("%v  %s", item.State.Emoji(), item.State.Text())
+		t.AddRow(item.Name, status, item.State.LocalVersion, item.State.LocalPath)
 	}
 	renderTableTitle(out, title)
 	t.Render()
 }
 
+func appsecMetricsTable(out io.Writer, itemName string, metrics map[string]int) {
+	t := newTable(out)
+	t.SetHeaders("Inband Hits", "Outband Hits")
+
+	t.AddRow(
+		strconv.Itoa(metrics["inband_hits"]),
+		strconv.Itoa(metrics["outband_hits"]),
+	)
+
+	renderTableTitle(out, fmt.Sprintf("\n - (AppSec Rule) %s:", itemName))
+	t.Render()
+}
+
 func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int) {
 	if metrics["instantiation"] == 0 {
 		return
@@ -31,11 +46,11 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
 	t.SetHeaders("Current Count", "Overflows", "Instantiated", "Poured", "Expired")
 
 	t.AddRow(
-		fmt.Sprintf("%d", metrics["curr_count"]),
-		fmt.Sprintf("%d", metrics["overflow"]),
-		fmt.Sprintf("%d", metrics["instantiation"]),
-		fmt.Sprintf("%d", metrics["pour"]),
-		fmt.Sprintf("%d", metrics["underflow"]),
+		strconv.Itoa(metrics["curr_count"]),
+		strconv.Itoa(metrics["overflow"]),
+		strconv.Itoa(metrics["instantiation"]),
+		strconv.Itoa(metrics["pour"]),
+		strconv.Itoa(metrics["underflow"]),
 	)
 
 	renderTableTitle(out, fmt.Sprintf("\n - (Scenario) %s:", itemName))
@@ -43,23 +58,25 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
 }
 
 func parserMetricsTable(out io.Writer, itemName string, metrics map[string]map[string]int) {
-	skip := true
 	t := newTable(out)
 	t.SetHeaders("Parsers", "Hits", "Parsed", "Unparsed")
 
+	// don't show table if no hits
+	showTable := false
+
 	for source, stats := range metrics {
 		if stats["hits"] > 0 {
 			t.AddRow(
 				source,
-				fmt.Sprintf("%d", stats["hits"]),
-				fmt.Sprintf("%d", stats["parsed"]),
-				fmt.Sprintf("%d", stats["unparsed"]),
+				strconv.Itoa(stats["hits"]),
+				strconv.Itoa(stats["parsed"]),
+				strconv.Itoa(stats["unparsed"]),
 			)
-			skip = false
+			showTable = true
 		}
 	}
 
-	if !skip {
+	if showTable {
 		renderTableTitle(out, fmt.Sprintf("\n - (Parser) %s:", itemName))
 		t.Render()
 	}

cmd/crowdsec-cli/version.go

@@ -0,0 +1,27 @@
+package main
+
+import (
+	"github.com/spf13/cobra"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+)
+
+type cliVersion struct{}
+
+func NewCLIVersion() *cliVersion {
+	return &cliVersion{}
+}
+
+func (cli cliVersion) NewCommand() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:               "version",
+		Short:             "Display version",
+		Args:              cobra.ExactArgs(0),
+		DisableAutoGenTag: true,
+		Run: func(_ *cobra.Command, _ []string) {
+			cwversion.Show()
+		},
+	}
+
+	return cmd
+}

cmd/crowdsec/crowdsec.go

@@ -13,6 +13,8 @@ import (
 	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
+	"github.com/crowdsecurity/crowdsec/pkg/appsec"
+	"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
@@ -20,31 +22,35 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
-func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
+func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) {
 	var err error
 
-	// Populate cwhub package tools
-	if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
-		return nil, fmt.Errorf("while loading hub index: %w", err)
+	if err = alertcontext.LoadConsoleContext(cConfig, hub); err != nil {
+		return nil, fmt.Errorf("while loading context: %w", err)
 	}
 
 	// Start loading configs
-	csParsers := parser.NewParsers()
+	csParsers := parser.NewParsers(hub)
 	if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
 		return nil, fmt.Errorf("while loading parsers: %w", err)
 	}
 
-	if err := LoadBuckets(cConfig); err != nil {
+	if err := LoadBuckets(cConfig, hub); err != nil {
 		return nil, fmt.Errorf("while loading scenarios: %w", err)
 	}
 
+	if err := appsec.LoadAppsecRules(hub); err != nil {
+		return nil, fmt.Errorf("while loading appsec rules: %w", err)
+	}
+
 	if err := LoadAcquisition(cConfig); err != nil {
 		return nil, fmt.Errorf("while loading acquisition config: %w", err)
 	}
+
 	return csParsers, nil
 }
 
-func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
+func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.Hub) error {
 	inputEventChan = make(chan types.Event)
 	inputLineChan = make(chan types.Event)
 
@@ -99,7 +105,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
 		for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ {
 			outputsTomb.Go(func() error {
 				defer trace.CatchPanic("crowdsec/runOutput")
-				if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials); err != nil {
+				if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials, hub); err != nil {
 					log.Fatalf("starting outputs error : %s", err)
 					return err
 				}
@@ -131,7 +137,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
 	return nil
 }
 
-func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady chan bool) {
+func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub.Hub, agentReady chan bool) {
 	crowdsecTomb.Go(func() error {
 		defer trace.CatchPanic("crowdsec/serveCrowdsec")
 		go func() {
@@ -139,7 +145,7 @@ func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady
 			// this logs every time, even at config reload
 			log.Debugf("running agent after %s ms", time.Since(crowdsecT0))
 			agentReady <- true
-			if err := runCrowdsec(cConfig, parsers); err != nil {
+			if err := runCrowdsec(cConfig, parsers, hub); err != nil {
 				log.Fatalf("unable to start crowdsec routines: %s", err)
 			}
 		}()

cmd/crowdsec/hook.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"io"
+	"os"
+
+	log "github.com/sirupsen/logrus"
+)
+
+type ConditionalHook struct {
+	Writer    io.Writer
+	LogLevels []log.Level
+	Enabled   bool
+}
+
+func (hook *ConditionalHook) Fire(entry *log.Entry) error {
+	if hook.Enabled {
+		line, err := entry.String()
+		if err != nil {
+			return err
+		}
+
+		_, err = hook.Writer.Write([]byte(line))
+
+		return err
+	}
+
+	return nil
+}
+
+func (hook *ConditionalHook) Levels() []log.Level {
+	return hook.LogLevels
+}
+
+// The primal logging hook is set up before parsing config.yaml.
+// Once config.yaml is parsed, the primal hook is disabled if the
+// configured logger is writing to stderr. Otherwise it's used to
+// report fatal errors and panics to stderr in addition to the log file.
+var primalHook = &ConditionalHook{
+	Writer:    os.Stderr,
+	LogLevels: []log.Level{log.FatalLevel, log.PanicLevel},
+	Enabled:   true,
+}

cmd/crowdsec/main.go

@@ -6,6 +6,7 @@ import (
 	_ "net/http/pprof"
 	"os"
 	"runtime"
+	"runtime/pprof"
 	"strings"
 	"time"
 
@@ -71,24 +72,27 @@ type Flags struct {
 	DisableCAPI    bool
 	Transform      string
 	OrderEvent     bool
+	CpuProfile     string
 }
 
 type labelsMap map[string]string
 
-func LoadBuckets(cConfig *csconfig.Config) error {
+func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error {
 	var (
 		err   error
 		files []string
 	)
-	for _, hubScenarioItem := range cwhub.GetItemMap(cwhub.SCENARIOS) {
-		if hubScenarioItem.Installed {
-			files = append(files, hubScenarioItem.LocalPath)
+
+	for _, hubScenarioItem := range hub.GetItemMap(cwhub.SCENARIOS) {
+		if hubScenarioItem.State.Installed {
+			files = append(files, hubScenarioItem.State.LocalPath)
 		}
 	}
+
 	buckets = leakybucket.NewBuckets()
 
 	log.Infof("Loading %d scenario files", len(files))
-	holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, files, &bucketsTomb, buckets, flags.OrderEvent)
+	holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, hub, files, &bucketsTomb, buckets, flags.OrderEvent)
 
 	if err != nil {
 		return fmt.Errorf("scenario loading failed: %v", err)
@@ -99,6 +103,7 @@ func LoadBuckets(cConfig *csconfig.Config) error {
 			holders[holderIndex].Profiling = true
 		}
 	}
+
 	return nil
 }
 
@@ -143,8 +148,10 @@ func (l labelsMap) Set(label string) error {
 		if len(split) != 2 {
 			return fmt.Errorf("invalid format for label '%s', must be key:value", pair)
 		}
+
 		l[split[0]] = split[1]
 	}
+
 	return nil
 }
 
@@ -168,10 +175,13 @@ func (f *Flags) Parse() {
 	flag.BoolVar(&f.DisableAPI, "no-api", false, "disable local API")
 	flag.BoolVar(&f.DisableCAPI, "no-capi", false, "disable communication with Central API")
 	flag.BoolVar(&f.OrderEvent, "order-event", false, "enforce event ordering with significant performance cost")
+
 	if runtime.GOOS == "windows" {
 		flag.StringVar(&f.WinSvc, "winsvc", "", "Windows service Action: Install, Remove etc..")
 	}
+
 	flag.StringVar(&dumpFolder, "dump-data", "", "dump parsers/buckets raw outputs")
+	flag.StringVar(&f.CpuProfile, "cpu-profile", "", "write cpu profile to file")
 	flag.Parse()
 }
 
@@ -205,6 +215,7 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
 		// avoid returning a new ptr to the same value
 		return curLevelPtr
 	}
+
 	return &ret
 }
 
@@ -212,11 +223,7 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
 func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*csconfig.Config, error) {
 	cConfig, _, err := csconfig.NewConfig(configFile, disableAgent, disableAPI, quiet)
 	if err != nil {
-		return nil, err
-	}
-
-	if (cConfig.Common == nil || *cConfig.Common == csconfig.CommonCfg{}) {
-		return nil, fmt.Errorf("unable to load configuration: common section is empty")
+		return nil, fmt.Errorf("while loading configuration file: %w", err)
 	}
 
 	cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags)
@@ -228,11 +235,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 		dumpStates = true
 	}
 
-	// Configuration paths are dependency to load crowdsec configuration
-	if err := cConfig.LoadConfigurationPaths(); err != nil {
-		return nil, err
-	}
-
 	if flags.SingleFileType != "" && flags.OneShotDSN != "" {
 		// if we're in time-machine mode, we don't want to log to file
 		cConfig.Common.LogMedia = "stdout"
@@ -247,6 +249,8 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 		return nil, err
 	}
 
+	primalHook.Enabled = (cConfig.Common.LogMedia != "stdout")
+
 	if err := csconfig.LoadFeatureFlagsFile(configFile, log.StandardLogger()); err != nil {
 		return nil, err
 	}
@@ -271,10 +275,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 		return nil, errors.New("You must run at least the API Server or crowdsec")
 	}
 
-	if flags.TestMode && !cConfig.DisableAgent {
-		cConfig.Crowdsec.LintOnly = true
-	}
-
 	if flags.OneShotDSN != "" && flags.SingleFileType == "" {
 		return nil, errors.New("-dsn requires a -type argument")
 	}
@@ -295,6 +295,7 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 		if cConfig.DisableAPI {
 			cConfig.Common.Daemonize = false
 		}
+
 		log.Infof("single file mode : log_media=%s daemonize=%t", cConfig.Common.LogMedia, cConfig.Common.Daemonize)
 	}
 
@@ -304,6 +305,7 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 
 	if cConfig.Common.Daemonize && runtime.GOOS == "windows" {
 		log.Debug("Daemonization is not supported on Windows, disabling")
+
 		cConfig.Common.Daemonize = false
 	}
 
@@ -321,6 +323,8 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 var crowdsecT0 time.Time
 
 func main() {
+	log.AddHook(primalHook)
+
 	if err := fflag.RegisterAllFeatures(); err != nil {
 		log.Fatalf("failed to register features: %s", err)
 	}
@@ -351,9 +355,25 @@ func main() {
 		os.Exit(0)
 	}
 
+	if flags.CpuProfile != "" {
+		f, err := os.Create(flags.CpuProfile)
+		if err != nil {
+			log.Fatalf("could not create CPU profile: %s", err)
+		}
+		log.Infof("CPU profile will be written to %s", flags.CpuProfile)
+		if err := pprof.StartCPUProfile(f); err != nil {
+			f.Close()
+			log.Fatalf("could not start CPU profile: %s", err)
+		}
+		defer f.Close()
+		defer pprof.StopCPUProfile()
+	}
+
 	err := StartRunSvc()
 	if err != nil {
-		log.Fatal(err)
+		pprof.StopCPUProfile()
+		log.Fatal(err) //nolint:gocritic // Disable warning for the defer pprof.StopCPUProfile() call
 	}
+
 	os.Exit(0)
 }

cmd/crowdsec/metrics.go

@@ -151,14 +151,6 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
 	if !config.Enabled {
 		return
 	}
-	if config.ListenAddr == "" {
-		log.Warning("prometheus is enabled, but the listen address is empty, using '127.0.0.1'")
-		config.ListenAddr = "127.0.0.1"
-	}
-	if config.ListenPort == 0 {
-		log.Warning("prometheus is enabled, but the listen port is empty, using '6060'")
-		config.ListenPort = 6060
-	}
 
 	// Registering prometheus
 	// If in aggregated mode, do not register events associated with a source, to keep the cardinality low
@@ -169,7 +161,8 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
 			leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow,
 			v1.LapiRouteHits,
 			leaky.BucketsCurrentCount,
-			cache.CacheMetrics, exprhelpers.RegexpCacheMetrics)
+			cache.CacheMetrics, exprhelpers.RegexpCacheMetrics,
+		)
 	} else {
 		log.Infof("Loading prometheus collectors")
 		prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo,
@@ -178,7 +171,8 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
 			v1.LapiRouteHits, v1.LapiMachineHits, v1.LapiBouncerHits, v1.LapiNilDecisions, v1.LapiNonNilDecisions, v1.LapiResponseTime,
 			leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, leaky.BucketsCurrentCount,
 			globalActiveDecisions, globalAlerts,
-			cache.CacheMetrics, exprhelpers.RegexpCacheMetrics)
+			cache.CacheMetrics, exprhelpers.RegexpCacheMetrics,
+		)
 
 	}
 }

cmd/crowdsec/output.go

@@ -62,7 +62,8 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error
 var bucketOverflows []types.Event
 
 func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets,
-	postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node, apiConfig csconfig.ApiCredentialsCfg) error {
+	postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node,
+	apiConfig csconfig.ApiCredentialsCfg, hub *cwhub.Hub) error {
 
 	var err error
 	ticker := time.NewTicker(1 * time.Second)
@@ -70,11 +71,20 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
 	var cache []types.RuntimeAlert
 	var cacheMutex sync.Mutex
 
-	scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
+	scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
 	if err != nil {
 		return fmt.Errorf("loading list of installed hub scenarios: %w", err)
 	}
 
+	appsecRules, err := hub.GetInstalledItemNames(cwhub.APPSEC_RULES)
+	if err != nil {
+		return fmt.Errorf("loading list of installed hub appsec rules: %w", err)
+	}
+
+	installedScenariosAndAppsecRules := make([]string, 0, len(scenarios)+len(appsecRules))
+	installedScenariosAndAppsecRules = append(installedScenariosAndAppsecRules, scenarios...)
+	installedScenariosAndAppsecRules = append(installedScenariosAndAppsecRules, appsecRules...)
+
 	apiURL, err := url.Parse(apiConfig.URL)
 	if err != nil {
 		return fmt.Errorf("parsing api url ('%s'): %w", apiConfig.URL, err)
@@ -86,14 +96,27 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
 	password := strfmt.Password(apiConfig.Password)
 
 	Client, err := apiclient.NewClient(&apiclient.Config{
-		MachineID:      apiConfig.Login,
-		Password:       password,
-		Scenarios:      scenarios,
-		UserAgent:      fmt.Sprintf("crowdsec/%s", version.String()),
-		URL:            apiURL,
-		PapiURL:        papiURL,
-		VersionPrefix:  "v1",
-		UpdateScenario: func() ([]string, error) {return cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)},
+		MachineID:     apiConfig.Login,
+		Password:      password,
+		Scenarios:     installedScenariosAndAppsecRules,
+		UserAgent:     fmt.Sprintf("crowdsec/%s", version.String()),
+		URL:           apiURL,
+		PapiURL:       papiURL,
+		VersionPrefix: "v1",
+		UpdateScenario: func() ([]string, error) {
+			scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
+			if err != nil {
+				return nil, err
+			}
+			appsecRules, err := hub.GetInstalledItemNames(cwhub.APPSEC_RULES)
+			if err != nil {
+				return nil, err
+			}
+			ret := make([]string, 0, len(scenarios)+len(appsecRules))
+			ret = append(ret, scenarios...)
+			ret = append(ret, appsecRules...)
+			return ret, nil
+		},
 	})
 	if err != nil {
 		return fmt.Errorf("new client api: %w", err)
@@ -101,7 +124,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
 	authResp, _, err := Client.Auth.AuthenticateWatcher(context.Background(), models.WatcherAuthRequest{
 		MachineID: &apiConfig.Login,
 		Password:  &password,
-		Scenarios: scenarios,
+		Scenarios: installedScenariosAndAppsecRules,
 	})
 	if err != nil {
 		return fmt.Errorf("authenticate watcher (%s): %w", apiConfig.Login, err)
@@ -145,13 +168,6 @@ LOOP:
 			}
 			break LOOP
 		case event := <-overflow:
-			//if the Alert is nil, it's to signal bucket is ready for GC, don't track this
-			if dumpStates && event.Overflow.Alert != nil {
-				if bucketOverflows == nil {
-					bucketOverflows = make([]types.Event, 0)
-				}
-				bucketOverflows = append(bucketOverflows, event)
-			}
 			/*if alert is empty and mapKey is present, the overflow is just to cleanup bucket*/
 			if event.Overflow.Alert == nil && event.Overflow.Mapkey != "" {
 				buckets.Bucket_map.Delete(event.Overflow.Mapkey)
@@ -163,6 +179,14 @@ LOOP:
 				return fmt.Errorf("postoverflow failed : %s", err)
 			}
 			log.Printf("%s", *event.Overflow.Alert.Message)
+			//if the Alert is nil, it's to signal bucket is ready for GC, don't track this
+			//dump after postoveflow processing to avoid missing whitelist info
+			if dumpStates && event.Overflow.Alert != nil {
+				if bucketOverflows == nil {
+					bucketOverflows = make([]types.Event, 0)
+				}
+				bucketOverflows = append(bucketOverflows, event)
+			}
 			if event.Overflow.Whitelisted {
 				log.Printf("[%s] is whitelisted, skip.", *event.Overflow.Alert.Message)
 				continue

cmd/crowdsec/parse.go

@@ -22,6 +22,13 @@ LOOP:
 			if !event.Process {
 				continue
 			}
+			/*Application security engine is going to generate 2 events:
+			- one that is treated as a log and can go to scenarios
+			- another one that will go directly to LAPI*/
+			if event.Type == types.APPSEC {
+				outputEventChan <- event
+				continue
+			}
 			if event.Line.Module == "" {
 				log.Errorf("empty event.Line.Module field, the acquisition module must set it ! : %+v", event.Line)
 				continue

cmd/crowdsec/run_in_svc.go

@@ -1,14 +1,12 @@
-//go:build linux || freebsd || netbsd || openbsd || solaris || !windows
-// +build linux freebsd netbsd openbsd solaris !windows
+//go:build !windows
 
 package main
 
 import (
 	"fmt"
-	"os"
+	"runtime/pprof"
 
 	log "github.com/sirupsen/logrus"
-	"github.com/sirupsen/logrus/hooks/writer"
 
 	"github.com/crowdsecurity/go-cs-lib/trace"
 	"github.com/crowdsecurity/go-cs-lib/version"
@@ -25,15 +23,9 @@ func StartRunSvc() error {
 
 	defer trace.CatchPanic("crowdsec/StartRunSvc")
 
-	// Set a default logger with level=fatal on stderr,
-	// in addition to the one we configure afterwards
-	log.AddHook(&writer.Hook{
-		Writer: os.Stderr,
-		LogLevels: []log.Level{
-			log.PanicLevel,
-			log.FatalLevel,
-		},
-	})
+	//Always try to stop CPU profiling to avoid passing flags around
+	//It's a noop if profiling is not enabled
+	defer pprof.StopCPUProfile()
 
 	if cConfig, err = LoadConfig(flags.ConfigFile, flags.DisableAgent, flags.DisableAPI, false); err != nil {
 		return err
@@ -47,6 +39,7 @@ func StartRunSvc() error {
 	// Enable profiling early
 	if cConfig.Prometheus != nil {
 		var dbClient *database.Client
+
 		var err error
 
 		if cConfig.DbConfig != nil {
@@ -56,8 +49,11 @@ func StartRunSvc() error {
 				return fmt.Errorf("unable to create database client: %s", err)
 			}
 		}
+
 		registerPrometheus(cConfig.Prometheus)
+
 		go servePrometheus(cConfig.Prometheus, dbClient, apiReady, agentReady)
 	}
+
 	return Serve(cConfig, apiReady, agentReady)
 }

cmd/crowdsec/run_in_svc_windows.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"fmt"
+	"runtime/pprof"
 
 	log "github.com/sirupsen/logrus"
 	"golang.org/x/sys/windows/svc"
@@ -19,6 +20,10 @@ func StartRunSvc() error {
 
 	defer trace.CatchPanic("crowdsec/StartRunSvc")
 
+	//Always try to stop CPU profiling to avoid passing flags around
+	//It's a noop if profiling is not enabled
+	defer pprof.StopCPUProfile()
+
 	isRunninginService, err := svc.IsWindowsService()
 	if err != nil {
 		return fmt.Errorf("failed to determine if we are running in windows service mode: %w", err)

cmd/crowdsec/serve.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"os/signal"
+	"runtime/pprof"
 	"syscall"
 	"time"
 
@@ -14,6 +15,7 @@ import (
 	"github.com/crowdsecurity/go-cs-lib/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
@@ -76,7 +78,12 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
 	}
 
 	if !cConfig.DisableAgent {
-		csParsers, err := initCrowdsec(cConfig)
+		hub, err := cwhub.NewHub(cConfig.Hub, nil, false, log.StandardLogger())
+		if err != nil {
+			return nil, fmt.Errorf("while loading hub index: %w", err)
+		}
+
+		csParsers, err := initCrowdsec(cConfig, hub)
 		if err != nil {
 			return nil, fmt.Errorf("unable to init crowdsec: %w", err)
 		}
@@ -93,7 +100,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
 		}
 
 		agentReady := make(chan bool, 1)
-		serveCrowdsec(csParsers, cConfig, agentReady)
+		serveCrowdsec(csParsers, cConfig, hub, agentReady)
 	}
 
 	log.Printf("Reload is finished")
@@ -239,6 +246,10 @@ func HandleSignals(cConfig *csconfig.Config) error {
 
 	exitChan := make(chan error)
 
+	//Always try to stop CPU profiling to avoid passing flags around
+	//It's a noop if profiling is not enabled
+	defer pprof.StopCPUProfile()
+
 	go func() {
 		defer trace.CatchPanic("crowdsec/HandleSignals")
 	Loop:
@@ -342,14 +353,19 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
 	}
 
 	if !cConfig.DisableAgent {
-		csParsers, err := initCrowdsec(cConfig)
+		hub, err := cwhub.NewHub(cConfig.Hub, nil, false, log.StandardLogger())
+		if err != nil {
+			return fmt.Errorf("while loading hub index: %w", err)
+		}
+
+		csParsers, err := initCrowdsec(cConfig, hub)
 		if err != nil {
 			return fmt.Errorf("crowdsec init: %w", err)
 		}
 
 		// if it's just linting, we're done
 		if !flags.TestMode {
-			serveCrowdsec(csParsers, cConfig, agentReady)
+			serveCrowdsec(csParsers, cConfig, hub, agentReady)
 		}
 	} else {
 		agentReady <- true

cmd/crowdsec/win_service.go

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build windows
-// +build windows
 
 package main
 

cmd/crowdsec/win_service_install.go

@@ -3,7 +3,6 @@
 // license that can be found in the LICENSE file.
 
 //go:build windows
-// +build windows
 
 package main
 

cmd/crowdsec/win_service_manage.go

@@ -2,7 +2,7 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// +build windows
+//go:build windows
 
 package main
 

cmd/notification-http/main.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"crypto/tls"
+	"crypto/x509"
 	"fmt"
 	"io"
 	"net/http"
@@ -22,6 +23,10 @@ type PluginConfig struct {
 	SkipTLSVerification bool              `yaml:"skip_tls_verification"`
 	Method              string            `yaml:"method"`
 	LogLevel            *string           `yaml:"log_level"`
+	Client              *http.Client      `yaml:"-"`
+	CertPath            string            `yaml:"cert_path"`
+	KeyPath             string            `yaml:"key_path"`
+	CAPath              string            `yaml:"ca_cert_path"`
 }
 
 type HTTPPlugin struct {
@@ -35,6 +40,64 @@ var logger hclog.Logger = hclog.New(&hclog.LoggerOptions{
 	JSONFormat: true,
 })
 
+func getCertPool(caPath string) (*x509.CertPool, error) {
+	cp, err := x509.SystemCertPool()
+	if err != nil {
+		return nil, fmt.Errorf("unable to load system CA certificates: %w", err)
+	}
+
+	if cp == nil {
+		cp = x509.NewCertPool()
+	}
+
+	if caPath == "" {
+		return cp, nil
+	}
+
+	logger.Info(fmt.Sprintf("Using CA cert '%s'", caPath))
+
+	caCert, err := os.ReadFile(caPath)
+	if err != nil {
+		return nil, fmt.Errorf("unable to load CA certificate '%s': %w", caPath, err)
+	}
+
+	cp.AppendCertsFromPEM(caCert)
+
+	return cp, nil
+}
+
+func getTLSClient(tlsVerify bool, caPath, certPath, keyPath string) (*http.Client, error) {
+	var client *http.Client
+
+	caCertPool, err := getCertPool(caPath)
+	if err != nil {
+		return nil, err
+	}
+
+	tlsConfig := &tls.Config{
+		RootCAs:            caCertPool,
+		InsecureSkipVerify: tlsVerify,
+	}
+
+	if certPath != "" && keyPath != "" {
+		logger.Info(fmt.Sprintf("Using client certificate '%s' and key '%s'", certPath, keyPath))
+
+		cert, err := tls.LoadX509KeyPair(certPath, keyPath)
+		if err != nil {
+			return nil, fmt.Errorf("unable to load client certificate '%s' and key '%s': %w", certPath, keyPath, err)
+		}
+
+		tlsConfig.Certificates = []tls.Certificate{cert}
+	}
+
+	client = &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: tlsConfig,
+		},
+	}
+	return client, err
+}
+
 func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notification) (*protobufs.Empty, error) {
 	if _, ok := s.PluginConfigByName[notification.Name]; !ok {
 		return nil, fmt.Errorf("invalid plugin config name %s", notification.Name)
@@ -46,25 +109,17 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
 	}
 
 	logger.Info(fmt.Sprintf("received signal for %s config", notification.Name))
-	client := http.Client{}
-
-	if cfg.SkipTLSVerification {
-		client.Transport = &http.Transport{
-			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
-		}
-	}
 
 	request, err := http.NewRequest(cfg.Method, cfg.URL, bytes.NewReader([]byte(notification.Text)))
 	if err != nil {
 		return nil, err
 	}
-
 	for headerName, headerValue := range cfg.Headers {
 		logger.Debug(fmt.Sprintf("adding header %s: %s", headerName, headerValue))
 		request.Header.Add(headerName, headerValue)
 	}
 	logger.Debug(fmt.Sprintf("making HTTP %s call to %s with body %s", cfg.Method, cfg.URL, notification.Text))
-	resp, err := client.Do(request)
+	resp, err := cfg.Client.Do(request.WithContext(ctx))
 	if err != nil {
 		logger.Error(fmt.Sprintf("Failed to make HTTP request : %s", err))
 		return nil, err
@@ -89,6 +144,13 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
 func (s *HTTPPlugin) Configure(ctx context.Context, config *protobufs.Config) (*protobufs.Empty, error) {
 	d := PluginConfig{}
 	err := yaml.Unmarshal(config.Config, &d)
+	if err != nil {
+		return nil, err
+	}
+	d.Client, err = getTLSClient(d.SkipTLSVerification, d.CAPath, d.CertPath, d.KeyPath)
+	if err != nil {
+		return nil, err
+	}
 	s.PluginConfigByName[d.Name] = d
 	logger.Debug(fmt.Sprintf("HTTP plugin '%s' use URL '%s'", d.Name, d.URL))
 	return &protobufs.Empty{}, err

cmd/notification-sentinel/main.go

@@ -90,7 +90,7 @@ func (s *SentinelPlugin) Notify(ctx context.Context, notification *protobufs.Not
 	req.Header.Set("x-ms-date", now)
 
 	client := &http.Client{}
-	resp, err := client.Do(req)
+	resp, err := client.Do(req.WithContext(ctx))
 	if err != nil {
 		logger.Error("failed to send request", "error", err)
 		return &protobufs.Empty{}, err

cmd/notification-slack/main.go

@@ -38,10 +38,9 @@ func (n *Notify) Notify(ctx context.Context, notification *protobufs.Notificatio
 	if cfg.LogLevel != nil && *cfg.LogLevel != "" {
 		logger.SetLevel(hclog.LevelFromString(*cfg.LogLevel))
 	}
-
 	logger.Info(fmt.Sprintf("found notify signal for %s config", notification.Name))
 	logger.Debug(fmt.Sprintf("posting to %s webhook, message %s", cfg.Webhook, notification.Text))
-	err := slack.PostWebhook(n.ConfigByName[notification.Name].Webhook, &slack.WebhookMessage{
+	err := slack.PostWebhookContext(ctx, n.ConfigByName[notification.Name].Webhook, &slack.WebhookMessage{
 		Text: notification.Text,
 	})
 	if err != nil {

cmd/notification-splunk/main.go

@@ -65,7 +65,7 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio
 
 	req.Header.Add("Authorization", fmt.Sprintf("Splunk %s", cfg.Token))
 	logger.Debug(fmt.Sprintf("posting event %s to %s", string(data), req.URL))
-	resp, err := s.Client.Do(req)
+	resp, err := s.Client.Do(req.WithContext(ctx))
 	if err != nil {
 		return &protobufs.Empty{}, err
 	}

config/acquis_win.yaml

@@ -10,7 +10,7 @@ labels:
 ---
 ##Firewall
 filenames:
-  - C:\Windows\System32\LogFiles\Firewall\pfirewall.log
+  - C:\Windows\System32\LogFiles\Firewall\*.log
 labels:
   type: windows-firewall
 ---
@@ -28,4 +28,4 @@ use_time_machine: true
 filenames:
   - C:\inetpub\logs\LogFiles\*\*.log
 labels:
-  type: iis
+  type: iis

config/config.yaml

@@ -6,7 +6,6 @@ common:
   log_max_size: 20
   compress_logs: true
   log_max_files: 10
-  working_dir: .
 config_paths:
   config_dir: /etc/crowdsec/
   data_dir: /var/lib/crowdsec/data/

config/config_win.yaml

@@ -3,7 +3,6 @@ common:
   log_media: file
   log_level: info
   log_dir:  C:\ProgramData\CrowdSec\log\
-  working_dir: .
 config_paths:
   config_dir:  C:\ProgramData\CrowdSec\config\
   data_dir:  C:\ProgramData\CrowdSec\data\

config/config_win_no_lapi.yaml

@@ -3,7 +3,6 @@ common:
   log_media: file
   log_level: info
   log_dir:  C:\ProgramData\CrowdSec\log\
-  working_dir: .
 config_paths:
   config_dir:  C:\ProgramData\CrowdSec\config\
   data_dir:  C:\ProgramData\CrowdSec\data\

config/dev.yaml

@@ -2,7 +2,6 @@ common:
   daemonize: true
   log_media: stdout
   log_level: info
-  working_dir: .
 config_paths:
   config_dir: ./config
   data_dir: ./data/   
@@ -34,6 +33,7 @@ api:
   client:
     credentials_path: ./config/local_api_credentials.yaml
   server:
+    console_path: ./config/console.yaml
     #insecure_skip_verify: true
     listen_uri: 127.0.0.1:8081
     profiles_path: ./config/profiles.yaml

config/user.yaml

@@ -3,7 +3,6 @@ common:
   log_media: stdout
   log_level: info
   log_dir: /var/log/
-  working_dir: .
 config_paths:
   config_dir: /etc/crowdsec/
   data_dir: /var/lib/crowdsec/data

debian/postinst

@@ -58,10 +58,10 @@ if [ "$1" = configure ]; then
             db_get crowdsec/capi
             CAPI=$RET
 
-            cscli machines add -a
+            [ -s /etc/crowdsec/local_api_credentials.yaml ] || cscli machines add -a --force --error
 
             if [ "$CAPI" = true ]; then
-                cscli capi register
+                cscli capi register --error
             fi
 
         else
@@ -91,7 +91,7 @@ if [ "$1" = configure ]; then
     systemctl --quiet is-enabled crowdsec || systemctl unmask crowdsec && systemctl enable crowdsec
 
     API=$(cscli config show --key "Config.API.Server")
-    if [ "$API" = "<nil>" ] ; then
+    if [ "$API" = "nil" ] ; then
         LAPI=false
     else
         PORT=$(cscli config show --key "Config.API.Server.ListenURI"|cut -d ":" -f2)
@@ -102,6 +102,13 @@ if [ "$1" = configure ]; then
         echo "Not attempting to start crowdsec, port ${PORT} is already used or lapi was disabled"
         echo "This port is configured through /etc/crowdsec/config.yaml and /etc/crowdsec/local_api_credentials.yaml"
     fi
+
+    echo "Get started with CrowdSec:"
+    echo " * Detailed guides are available in our documentation: https://docs.crowdsec.net"
+    echo " * Configuration items created by the community can be found at the Hub: https://hub.crowdsec.net"
+    echo " * Gain insights into your use of CrowdSec with the help of the console https://app.crowdsec.net"
+
+
 fi
 
-echo "You can always run the configuration again interactively by using '/usr/share/crowdsec/wizard.sh -c"
+echo "You can always run the configuration again interactively by using '/usr/share/crowdsec/wizard.sh -c'"

debian/preinst

@@ -40,4 +40,4 @@ if [ "$1" = upgrade ]; then
    fi   
 fi
 
-echo "You can always run the configuration again interactively by using '/usr/share/crowdsec/wizard.sh -c"
+echo "You can always run the configuration again interactively by using '/usr/share/crowdsec/wizard.sh -c'"

debian/prerm

@@ -1,5 +1,5 @@
 if [ "$1" = "remove" ]; then
-   cscli dashboard remove -f -y || true
+   cscli dashboard remove -f -y --error || echo "Ignore the above error if you never installed the local dashboard."
    systemctl stop crowdsec
    systemctl disable crowdsec
 fi

docker/README.md

@@ -320,10 +320,16 @@ config.yaml) each time the container is run.
 | `PARSERS`               | | Parsers to install, separated by space |
 | `SCENARIOS`             | | Scenarios to install, separated by space |
 | `POSTOVERFLOWS`         | | Postoverflows to install, separated by space |
+| `CONTEXTS`              | | Context files to install, separated by space |
+| `APPSEC_CONFIGS`        | | Appsec configs files to install, separated by space |
+| `APPSEC_RULES`          | | Appsec rules files to install, separated by space |
 | `DISABLE_COLLECTIONS`   | | Collections to remove, separated by space: `-e DISABLE_COLLECTIONS="crowdsecurity/linux crowdsecurity/nginx"` |
 | `DISABLE_PARSERS`       | | Parsers to remove, separated by space |
 | `DISABLE_SCENARIOS`     | | Scenarios to remove, separated by space |
 | `DISABLE_POSTOVERFLOWS` | | Postoverflows to remove, separated by space |
+| `DISABLE_CONTEXTS`      | | Context files to remove, separated by space |
+| `DISABLE_APPSEC_CONFIGS`| | Appsec configs files to remove, separated by space |
+| `DISABLE_APPSEC_RULES`  | | Appsec rules files to remove, separated by space |
 |                         | | |
 | __Log verbosity__       | | |
 | `LEVEL_INFO`            | false | Force INFO level for the container log |

docker/config.yaml

@@ -3,7 +3,6 @@ common:
   log_media: stdout
   log_level: info
   log_dir: /var/log/
-  working_dir: .
 config_paths:
   config_dir: /etc/crowdsec/
   data_dir: /var/lib/crowdsec/data/

docker/docker_start.sh

@@ -101,19 +101,23 @@ register_bouncer() {
 # $2 can be install, remove, upgrade
 # $3 is a list of object names separated by space
 cscli_if_clean() {
+    local itemtype="$1"
+    local action="$2"
+    local objs=$3
+    shift 3
     # loop over all objects
-    for obj in $3; do
-        if cscli "$1" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then
-            echo "Object $1/$obj is tainted, skipping"
+    for obj in $objs; do
+        if cscli "$itemtype" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then
+            echo "Object $itemtype/$obj is tainted, skipping"
         else
 #            # Too verbose? Only show errors if not in debug mode
 #            if [ "$DEBUG" != "true" ]; then
 #                error_only=--error
 #            fi
             error_only=""
-            echo "Running: cscli $error_only $1 $2 \"$obj\""
+            echo "Running: cscli $error_only $itemtype $action \"$obj\" $*"
             # shellcheck disable=SC2086
-            cscli $error_only "$1" "$2" "$obj"
+            cscli $error_only "$itemtype" "$action" "$obj" "$@"
         fi
     done
 }
@@ -174,7 +178,7 @@ if [ ! -e "/etc/crowdsec/local_api_credentials.yaml" ] && [ ! -e "/etc/crowdsec/
         mkdir -p /etc/crowdsec/
         # if you change this, check that it still works
         # under alpine and k8s, with and without tls
-        cp -an /staging/etc/crowdsec/* /etc/crowdsec/
+        rsync -av --ignore-existing /staging/etc/crowdsec/* /etc/crowdsec
     fi
 fi
 
@@ -198,7 +202,7 @@ if isfalse "$DISABLE_LOCAL_API"; then
             # if the db is persistent but the credentials are not, we need to
             # delete the old machine to generate new credentials
             cscli machines delete "$CUSTOM_HOSTNAME" >/dev/null 2>&1 || true
-            cscli machines add "$CUSTOM_HOSTNAME" --auto
+            cscli machines add "$CUSTOM_HOSTNAME" --auto --force
         fi
     fi
 
@@ -296,7 +300,7 @@ fi
 
 conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)'
 
-## Install collections, parsers, scenarios & postoverflows
+## Install hub items
 cscli hub update
 
 cscli_if_clean collections upgrade crowdsecurity/linux
@@ -324,25 +328,55 @@ if [ "$POSTOVERFLOWS" != "" ]; then
     cscli_if_clean postoverflows install "$(difference "$POSTOVERFLOWS" "$DISABLE_POSTOVERFLOWS")"
 fi
 
+if [ "$CONTEXTS" != "" ]; then
+    # shellcheck disable=SC2086
+    cscli_if_clean contexts install "$(difference "$CONTEXTS" "$DISABLE_CONTEXTS")"
+fi
+
+if [ "$APPSEC_CONFIGS" != "" ]; then
+    # shellcheck disable=SC2086
+    cscli_if_clean appsec-configs install "$(difference "$APPSEC_CONFIGS" "$DISABLE_APPSEC_CONFIGS")"
+fi
+
+if [ "$APPSEC_RULES" != "" ]; then
+    # shellcheck disable=SC2086
+    cscli_if_clean appsec-rules install "$(difference "$APPSEC_RULES" "$DISABLE_APPSEC_RULES")"
+fi
+
 ## Remove collections, parsers, scenarios & postoverflows
 if [ "$DISABLE_COLLECTIONS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli_if_clean collections remove "$DISABLE_COLLECTIONS"
+    cscli_if_clean collections remove "$DISABLE_COLLECTIONS" --force
 fi
 
 if [ "$DISABLE_PARSERS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli_if_clean parsers remove "$DISABLE_PARSERS"
+    cscli_if_clean parsers remove "$DISABLE_PARSERS" --force
 fi
 
 if [ "$DISABLE_SCENARIOS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli_if_clean scenarios remove "$DISABLE_SCENARIOS"
+    cscli_if_clean scenarios remove "$DISABLE_SCENARIOS" --force
 fi
 
 if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
     # shellcheck disable=SC2086
-    cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS"
+    cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" --force
+fi
+
+if [ "$DISABLE_CONTEXTS" != "" ]; then
+    # shellcheck disable=SC2086
+    cscli_if_clean contexts remove "$DISABLE_CONTEXTS" --force
+fi
+
+if [ "$DISABLE_APPSEC_CONFIGS" != "" ]; then
+    # shellcheck disable=SC2086
+    cscli_if_clean appsec-configs remove "$DISABLE_APPSEC_CONFIGS" --force
+fi
+
+if [ "$DISABLE_APPSEC_RULES" != "" ]; then
+    # shellcheck disable=SC2086
+    cscli_if_clean appsec-rules remove "$DISABLE_APPSEC_RULES" --force
 fi
 
 ## Register bouncers via env
@@ -354,6 +388,11 @@ for BOUNCER in $(compgen -A variable | grep -i BOUNCER_KEY); do
     fi
 done
 
+if [ "$ENABLE_CONSOLE_MANAGEMENT" != "" ]; then
+    # shellcheck disable=SC2086
+    cscli console enable console_management
+fi
+
 ## Register bouncers via secrets (Swarm only)
 shopt -s nullglob extglob
 for BOUNCER in /run/secrets/@(bouncer_key|BOUNCER_KEY)* ; do

docker/test/Pipfile

@@ -1,6 +1,6 @@
 [packages]
 pytest-dotenv = "0.5.2"
-pytest-xdist = "3.3.1"
+pytest-xdist = "3.5.0"
 pytest-cs = {ref = "0.7.18", git = "https://github.com/crowdsecurity/pytest-cs.git"}
 
 [dev-packages]

docker/test/Pipfile.lock

@@ -1,7 +1,7 @@
 {
     "_meta": {
         "hash": {
-            "sha256": "64085783c9fec3a9eda976b7700b5bad7abd2b7a0f0670fa2209c52f3647be7f"
+            "sha256": "575cb97d0b7fb66caf843191b843724307f7bc39c3c160f22330ba38ee055c80"
         },
         "pipfile-spec": 6,
         "requires": {
@@ -18,198 +18,202 @@
     "default": {
         "certifi": {
             "hashes": [
-                "sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082",
-                "sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"
+                "sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1",
+                "sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==2023.7.22"
+            "version": "==2023.11.17"
         },
         "cffi": {
             "hashes": [
-                "sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5",
-                "sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef",
-                "sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104",
-                "sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426",
-                "sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405",
-                "sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375",
-                "sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a",
-                "sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e",
-                "sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc",
-                "sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf",
-                "sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185",
-                "sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497",
-                "sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3",
-                "sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35",
-                "sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c",
-                "sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83",
-                "sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21",
-                "sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca",
-                "sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984",
-                "sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac",
-                "sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd",
-                "sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee",
-                "sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a",
-                "sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2",
-                "sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192",
-                "sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7",
-                "sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585",
-                "sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f",
-                "sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e",
-                "sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27",
-                "sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b",
-                "sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e",
-                "sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e",
-                "sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d",
-                "sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c",
-                "sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415",
-                "sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82",
-                "sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02",
-                "sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314",
-                "sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325",
-                "sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c",
-                "sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3",
-                "sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914",
-                "sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045",
-                "sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d",
-                "sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9",
-                "sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5",
-                "sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2",
-                "sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c",
-                "sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3",
-                "sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2",
-                "sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8",
-                "sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d",
-                "sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d",
-                "sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9",
-                "sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162",
-                "sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76",
-                "sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4",
-                "sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e",
-                "sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9",
-                "sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6",
-                "sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b",
-                "sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01",
-                "sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"
+                "sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc",
+                "sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a",
+                "sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417",
+                "sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab",
+                "sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520",
+                "sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36",
+                "sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743",
+                "sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8",
+                "sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed",
+                "sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684",
+                "sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56",
+                "sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324",
+                "sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d",
+                "sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235",
+                "sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e",
+                "sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088",
+                "sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000",
+                "sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7",
+                "sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e",
+                "sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673",
+                "sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c",
+                "sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe",
+                "sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2",
+                "sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098",
+                "sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8",
+                "sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a",
+                "sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0",
+                "sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b",
+                "sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896",
+                "sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e",
+                "sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9",
+                "sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2",
+                "sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b",
+                "sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6",
+                "sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404",
+                "sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f",
+                "sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0",
+                "sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4",
+                "sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc",
+                "sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936",
+                "sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba",
+                "sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872",
+                "sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb",
+                "sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614",
+                "sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1",
+                "sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d",
+                "sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969",
+                "sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b",
+                "sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4",
+                "sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627",
+                "sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956",
+                "sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"
             ],
-            "version": "==1.15.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.16.0"
         },
         "charset-normalizer": {
             "hashes": [
-                "sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96",
-                "sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c",
-                "sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710",
-                "sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706",
-                "sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020",
-                "sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252",
-                "sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad",
-                "sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329",
-                "sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a",
-                "sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f",
-                "sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6",
-                "sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4",
-                "sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a",
-                "sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46",
-                "sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2",
-                "sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23",
-                "sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace",
-                "sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd",
-                "sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982",
-                "sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10",
-                "sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2",
-                "sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea",
-                "sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09",
-                "sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5",
-                "sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149",
-                "sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489",
-                "sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9",
-                "sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80",
-                "sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592",
-                "sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3",
-                "sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6",
-                "sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed",
-                "sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c",
-                "sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200",
-                "sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a",
-                "sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e",
-                "sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d",
-                "sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6",
-                "sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623",
-                "sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669",
-                "sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3",
-                "sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa",
-                "sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9",
-                "sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2",
-                "sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f",
-                "sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1",
-                "sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4",
-                "sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a",
-                "sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8",
-                "sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3",
-                "sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029",
-                "sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f",
-                "sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959",
-                "sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22",
-                "sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7",
-                "sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952",
-                "sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346",
-                "sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e",
-                "sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d",
-                "sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299",
-                "sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd",
-                "sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a",
-                "sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3",
-                "sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037",
-                "sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94",
-                "sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c",
-                "sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858",
-                "sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a",
-                "sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449",
-                "sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c",
-                "sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918",
-                "sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1",
-                "sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c",
-                "sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac",
-                "sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"
+                "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027",
+                "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087",
+                "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786",
+                "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8",
+                "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09",
+                "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185",
+                "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574",
+                "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e",
+                "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519",
+                "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898",
+                "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269",
+                "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3",
+                "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f",
+                "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6",
+                "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8",
+                "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a",
+                "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73",
+                "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc",
+                "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714",
+                "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2",
+                "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc",
+                "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce",
+                "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d",
+                "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e",
+                "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6",
+                "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269",
+                "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96",
+                "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d",
+                "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a",
+                "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4",
+                "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77",
+                "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d",
+                "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0",
+                "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed",
+                "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068",
+                "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac",
+                "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25",
+                "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8",
+                "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab",
+                "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26",
+                "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2",
+                "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db",
+                "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f",
+                "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5",
+                "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99",
+                "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c",
+                "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d",
+                "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811",
+                "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa",
+                "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a",
+                "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03",
+                "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b",
+                "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04",
+                "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c",
+                "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001",
+                "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458",
+                "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389",
+                "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99",
+                "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985",
+                "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537",
+                "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238",
+                "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f",
+                "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d",
+                "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796",
+                "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a",
+                "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143",
+                "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8",
+                "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c",
+                "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5",
+                "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5",
+                "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711",
+                "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4",
+                "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6",
+                "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c",
+                "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7",
+                "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4",
+                "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b",
+                "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae",
+                "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12",
+                "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c",
+                "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae",
+                "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8",
+                "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887",
+                "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b",
+                "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4",
+                "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f",
+                "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5",
+                "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33",
+                "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519",
+                "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"
             ],
             "markers": "python_full_version >= '3.7.0'",
-            "version": "==3.2.0"
+            "version": "==3.3.2"
         },
         "cryptography": {
             "hashes": [
-                "sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67",
-                "sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311",
-                "sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8",
-                "sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13",
-                "sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143",
-                "sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f",
-                "sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829",
-                "sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd",
-                "sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397",
-                "sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac",
-                "sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d",
-                "sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a",
-                "sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839",
-                "sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e",
-                "sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6",
-                "sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9",
-                "sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860",
-                "sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca",
-                "sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91",
-                "sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d",
-                "sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714",
-                "sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb",
-                "sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f"
+                "sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960",
+                "sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a",
+                "sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc",
+                "sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a",
+                "sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf",
+                "sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1",
+                "sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39",
+                "sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406",
+                "sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a",
+                "sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a",
+                "sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c",
+                "sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be",
+                "sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15",
+                "sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2",
+                "sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d",
+                "sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157",
+                "sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003",
+                "sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248",
+                "sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a",
+                "sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec",
+                "sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309",
+                "sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7",
+                "sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==41.0.4"
+            "version": "==41.0.7"
         },
         "docker": {
             "hashes": [
-                "sha256:aa6d17830045ba5ef0168d5eaa34d37beeb113948c413affe1d5991fc11f9a20",
-                "sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9"
+                "sha256:12ba681f2777a0ad28ffbcc846a69c31b4dfd9752b47eb425a274ee269c5e14b",
+                "sha256:323736fb92cd9418fc5e7133bc953e11a9da04f4483f828b527db553f1e7e5a3"
             ],
-            "markers": "python_version >= '3.7'",
-            "version": "==6.1.3"
+            "markers": "python_version >= '3.8'",
+            "version": "==7.0.0"
         },
         "execnet": {
             "hashes": [
@@ -221,11 +225,11 @@
         },
         "idna": {
             "hashes": [
-                "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4",
-                "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"
+                "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca",
+                "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"
             ],
             "markers": "python_version >= '3.5'",
-            "version": "==3.4"
+            "version": "==3.6"
         },
         "iniconfig": {
             "hashes": [
@@ -237,11 +241,11 @@
         },
         "packaging": {
             "hashes": [
-                "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61",
-                "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"
+                "sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5",
+                "sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==23.1"
+            "version": "==23.2"
         },
         "pluggy": {
             "hashes": [
@@ -253,23 +257,25 @@
         },
         "psutil": {
             "hashes": [
-                "sha256:104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d",
-                "sha256:3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217",
-                "sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4",
-                "sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c",
-                "sha256:5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f",
-                "sha256:7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da",
-                "sha256:89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4",
-                "sha256:8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42",
-                "sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5",
-                "sha256:acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4",
-                "sha256:b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9",
-                "sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f",
-                "sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30",
-                "sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"
+                "sha256:032f4f2c909818c86cea4fe2cc407f1c0f0cde8e6c6d702b28b8ce0c0d143340",
+                "sha256:0bd41bf2d1463dfa535942b2a8f0e958acf6607ac0be52265ab31f7923bcd5e6",
+                "sha256:1132704b876e58d277168cd729d64750633d5ff0183acf5b3c986b8466cd0284",
+                "sha256:1d4bc4a0148fdd7fd8f38e0498639ae128e64538faa507df25a20f8f7fb2341c",
+                "sha256:3c4747a3e2ead1589e647e64aad601981f01b68f9398ddf94d01e3dc0d1e57c7",
+                "sha256:3f02134e82cfb5d089fddf20bb2e03fd5cd52395321d1c8458a9e58500ff417c",
+                "sha256:44969859757f4d8f2a9bd5b76eba8c3099a2c8cf3992ff62144061e39ba8568e",
+                "sha256:4c03362e280d06bbbfcd52f29acd79c733e0af33d707c54255d21029b8b32ba6",
+                "sha256:5794944462509e49d4d458f4dbfb92c47539e7d8d15c796f141f474010084056",
+                "sha256:b27f8fdb190c8c03914f908a4555159327d7481dac2f01008d483137ef3311a9",
+                "sha256:c727ca5a9b2dd5193b8644b9f0c883d54f1248310023b5ad3e92036c5e2ada68",
+                "sha256:e469990e28f1ad738f65a42dcfc17adaed9d0f325d55047593cb9033a0ab63df",
+                "sha256:ea36cc62e69a13ec52b2f625c27527f6e4479bca2b340b7a452af55b34fcbe2e",
+                "sha256:f37f87e4d73b79e6c5e749440c3113b81d1ee7d26f21c19c47371ddea834f414",
+                "sha256:fe361f743cb3389b8efda21980d93eb55c1f1e3898269bc9a2a1d0bb7b1f6508",
+                "sha256:fe8b7f07948f1304497ce4f4684881250cd859b16d06a1dc4d7941eeb6233bfe"
             ],
-            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
-            "version": "==5.9.5"
+            "markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
+            "version": "==5.9.7"
         },
         "pycparser": {
             "hashes": [
@@ -280,11 +286,11 @@
         },
         "pytest": {
             "hashes": [
-                "sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002",
-                "sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"
+                "sha256:0d009c083ea859a71b76adf7c1d502e4bc170b80a8ef002da5806527b9591fac",
+                "sha256:d989d136982de4e3b29dabcc838ad581c64e8ed52c11fbe86ddebd9da0818cd5"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==7.4.2"
+            "version": "==7.4.3"
         },
         "pytest-cs": {
             "git": "https://github.com/crowdsecurity/pytest-cs.git",
@@ -292,11 +298,11 @@
         },
         "pytest-datadir": {
             "hashes": [
-                "sha256:095f441782b1b907587eca7227fdbae94be43f1c96b4b2cbcc6801a4645be1af",
-                "sha256:9f7a3c4def6ac4cac3cc8181139ab53bd2667231052bd40cb07081748d4420f0"
+                "sha256:1617ed92f9afda0c877e4eac91904b5f779d24ba8f5e438752e3ae39d8d2ee3f",
+                "sha256:34adf361bcc7b37961bbc1dfa8d25a4829e778bab461703c38a5c50ca9c36dc8"
             ],
-            "markers": "python_version >= '3.6'",
-            "version": "==1.4.1"
+            "markers": "python_version >= '3.8'",
+            "version": "==1.5.0"
         },
         "pytest-dotenv": {
             "hashes": [
@@ -308,11 +314,11 @@
         },
         "pytest-xdist": {
             "hashes": [
-                "sha256:d5ee0520eb1b7bcca50a60a518ab7a7707992812c578198f8b44fdfac78e8c93",
-                "sha256:ff9daa7793569e6a68544850fd3927cd257cc03a7ef76c95e86915355e82b5f2"
+                "sha256:cbb36f3d67e0c478baa57fa4edc8843887e0f6cfc42d677530a36d7472b32d8a",
+                "sha256:d075629c7e00b611df89f490a5063944bee7a4362a5ff11c7cc7824a03dfce24"
             ],
             "index": "pypi",
-            "version": "==3.3.1"
+            "version": "==3.5.0"
         },
         "python-dotenv": {
             "hashes": [
@@ -396,35 +402,20 @@
         },
         "urllib3": {
             "hashes": [
-                "sha256:13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594",
-                "sha256:ef16afa8ba34a1f989db38e1dbbe0c302e4289a47856990d0682e374563ce35e"
-            ],
-            "markers": "python_version >= '3.7'",
-            "version": "==2.0.5"
-        },
-        "websocket-client": {
-            "hashes": [
-                "sha256:3aad25d31284266bcfcfd1fd8a743f63282305a364b8d0948a43bd606acc652f",
-                "sha256:6cfc30d051ebabb73a5fa246efdcc14c8fbebbd0330f8984ac3bb6d9edd2ad03"
+                "sha256:55901e917a5896a349ff771be919f8bd99aff50b79fe58fec595eb37bbc56bb3",
+                "sha256:df7aa8afb0148fa78488e7899b2c59b5f4ffcfa82e6c54ccb9dd37c1d7b52d54"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==1.6.3"
+            "version": "==2.1.0"
         }
     },
     "develop": {
         "asttokens": {
             "hashes": [
-                "sha256:2e0171b991b2c959acc6c49318049236844a5da1d65ba2672c4880c1c894834e",
-                "sha256:cf8fc9e61a86461aa9fb161a14a0841a03c405fa829ac6b202670b3495d2ce69"
+                "sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24",
+                "sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0"
             ],
-            "version": "==2.4.0"
-        },
-        "backcall": {
-            "hashes": [
-                "sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e",
-                "sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"
-            ],
-            "version": "==0.2.0"
+            "version": "==2.4.1"
         },
         "decorator": {
             "hashes": [
@@ -436,10 +427,11 @@
         },
         "executing": {
             "hashes": [
-                "sha256:0314a69e37426e3608aada02473b4161d4caf5a4b244d1d0c48072b8fee7bacc",
-                "sha256:19da64c18d2d851112f09c287f8d3dbbdf725ab0e569077efb6cdcbd3497c107"
+                "sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147",
+                "sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc"
             ],
-            "version": "==1.2.0"
+            "markers": "python_version >= '3.5'",
+            "version": "==2.0.1"
         },
         "gnureadline": {
             "hashes": [
@@ -484,19 +476,19 @@
         },
         "ipython": {
             "hashes": [
-                "sha256:2baeb5be6949eeebf532150f81746f8333e2ccce02de1c7eedde3f23ed5e9f1e",
-                "sha256:45a2c3a529296870a97b7de34eda4a31bee16bc7bf954e07d39abe49caf8f887"
+                "sha256:ca6f079bb33457c66e233e4580ebfc4128855b4cf6370dddd73842a9563e8a27",
+                "sha256:e8267419d72d81955ec1177f8a29aaa90ac80ad647499201119e2f05e99aa397"
             ],
             "markers": "python_version >= '3.11'",
-            "version": "==8.15.0"
+            "version": "==8.18.1"
         },
         "jedi": {
             "hashes": [
-                "sha256:bcf9894f1753969cbac8022a8c2eaee06bfa3724e4192470aaffe7eb6272b0c4",
-                "sha256:cb8ce23fbccff0025e9386b5cf85e892f94c9b822378f8da49970471335ac64e"
+                "sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd",
+                "sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0"
             ],
             "markers": "python_version >= '3.6'",
-            "version": "==0.19.0"
+            "version": "==0.19.1"
         },
         "matplotlib-inline": {
             "hashes": [
@@ -516,26 +508,19 @@
         },
         "pexpect": {
             "hashes": [
-                "sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937",
-                "sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"
+                "sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523",
+                "sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"
             ],
             "markers": "sys_platform != 'win32'",
-            "version": "==4.8.0"
-        },
-        "pickleshare": {
-            "hashes": [
-                "sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca",
-                "sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"
-            ],
-            "version": "==0.7.5"
+            "version": "==4.9.0"
         },
         "prompt-toolkit": {
             "hashes": [
-                "sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac",
-                "sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88"
+                "sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d",
+                "sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"
             ],
             "markers": "python_full_version >= '3.7.0'",
-            "version": "==3.0.39"
+            "version": "==3.0.43"
         },
         "ptyprocess": {
             "hashes": [
@@ -553,11 +538,11 @@
         },
         "pygments": {
             "hashes": [
-                "sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692",
-                "sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"
+                "sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c",
+                "sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"
             ],
             "markers": "python_version >= '3.7'",
-            "version": "==2.16.1"
+            "version": "==2.17.2"
         },
         "six": {
             "hashes": [
@@ -569,25 +554,25 @@
         },
         "stack-data": {
             "hashes": [
-                "sha256:32d2dd0376772d01b6cb9fc996f3c8b57a357089dec328ed4b6553d037eaf815",
-                "sha256:cbb2a53eb64e5785878201a97ed7c7b94883f48b87bfb0bbe8b623c74679e4a8"
+                "sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9",
+                "sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695"
             ],
-            "version": "==0.6.2"
+            "version": "==0.6.3"
         },
         "traitlets": {
             "hashes": [
-                "sha256:417745a96681fbb358e723d5346a547521f36e9bd0d50ba7ab368fff5d67aa54",
-                "sha256:f584ea209240466e66e91f3c81aa7d004ba4cf794990b0c775938a1544217cd1"
+                "sha256:f14949d23829023013c47df20b4a76ccd1a85effb786dc060f34de7948361b33",
+                "sha256:fcdaa8ac49c04dfa0ed3ee3384ef6dfdb5d6f3741502be247279407679296772"
             ],
             "markers": "python_version >= '3.8'",
-            "version": "==5.10.0"
+            "version": "==5.14.0"
         },
         "wcwidth": {
             "hashes": [
-                "sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e",
-                "sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"
+                "sha256:f01c104efdf57971bcb756f054dd58ddec5204dd15fa31d6503ea57947d97c02",
+                "sha256:f26ec43d96c8cbfed76a5075dac87680124fa84e0855195a6184da9c187f133c"
             ],
-            "version": "==0.2.6"
+            "version": "==0.2.12"
         }
     }
 }

docker/test/tests/test_agent.py

@@ -13,7 +13,7 @@ def test_no_agent(crowdsec, flavor):
         'DISABLE_AGENT': 'true',
     }
     with crowdsec(flavor=flavor, environment=env) as cs:
-        cs.wait_for_log("*CrowdSec Local API listening on 0.0.0.0:8080*")
+        cs.wait_for_log("*CrowdSec Local API listening on *:8080*")
         cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
         res = cs.cont.exec_run('cscli lapi status')
         assert res.exit_code == 0
@@ -37,7 +37,7 @@ def test_machine_register(crowdsec, flavor, tmp_path_factory):
     with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs:
         cs.wait_for_log([
                 "*Generate local agent credentials*",
-                "*CrowdSec Local API listening on 0.0.0.0:8080*",
+                "*CrowdSec Local API listening on *:8080*",
             ])
         cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
         res = cs.cont.exec_run('cscli lapi status')
@@ -50,7 +50,7 @@ def test_machine_register(crowdsec, flavor, tmp_path_factory):
     with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs:
         cs.wait_for_log([
                 "*Generate local agent credentials*",
-                "*CrowdSec Local API listening on 0.0.0.0:8080*",
+                "*CrowdSec Local API listening on *:8080*",
             ])
         cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
         res = cs.cont.exec_run('cscli lapi status')
@@ -65,7 +65,7 @@ def test_machine_register(crowdsec, flavor, tmp_path_factory):
     with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs:
         cs.wait_for_log([
                 "*Generate local agent credentials*",
-                "*CrowdSec Local API listening on 0.0.0.0:8080*",
+                "*CrowdSec Local API listening on *:8080*",
             ])
         cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
         res = cs.cont.exec_run('cscli lapi status')
@@ -78,7 +78,7 @@ def test_machine_register(crowdsec, flavor, tmp_path_factory):
     with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs:
         cs.wait_for_log([
                 "*Local agent already registered*",
-                "*CrowdSec Local API listening on 0.0.0.0:8080*",
+                "*CrowdSec Local API listening on *:8080*",
             ])
         cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
         res = cs.cont.exec_run('cscli lapi status')