metrics

.github/governance.yml

@@ -42,7 +42,7 @@ issue:
           3. Check [Releases](https://github.com/crowdsecurity/crowdsec/releases/latest) to make sure your agent is on the latest version.
 
     - prefix: kind
-      list: ['feature', 'bug', 'packaging', 'enhancement', 'refactoring']
+      list: ['feature', 'bug', 'packaging', 'enhancement']
       multiple: false
       author_association:
         author: true
@@ -54,7 +54,6 @@ issue:
           @$AUTHOR: There are no 'kind' label on this issue. You need a 'kind' label to start the triage process.
           * `/kind feature`
           * `/kind enhancement`
-          * `/kind refactoring`
           * `/kind bug`
           * `/kind packaging`
 
@@ -66,13 +65,12 @@ pull_request:
   labels:
     - prefix: kind
       multiple: false
-      list: [ 'feature', 'enhancement', 'fix', 'chore', 'dependencies', 'refactoring']
+      list: [ 'feature', 'enhancement', 'fix', 'chore', 'dependencies']
       needs:
         comment: |
           @$AUTHOR: There are no 'kind' label on this PR. You need a 'kind' label to generate the release automatically.
           * `/kind feature`
           * `/kind enhancement`
-          * `/kind refactoring`
           * `/kind fix`
           * `/kind chore`
           * `/kind dependencies`
@@ -83,7 +81,7 @@ pull_request:
             failure: Missing kind label to generate release automatically.
 
     - prefix: area
-      list: [ "agent", "local-api", "cscli", "security", "configuration", "appsec"]
+      list: [ "agent", "local-api", "cscli", "security", "configuration"]
       multiple: true
       needs:
         comment: |
@@ -91,7 +89,6 @@ pull_request:
           * `/area agent`
           * `/area local-api`
           * `/area cscli`
-          * `/area appsec`
           * `/area security`
           * `/area configuration`
 

.github/workflows/bats-hub.yml

@@ -1,4 +1,4 @@
-name: (sub) Bats / Hub
+name: Hub tests
 
 on:
   workflow_call:
@@ -8,13 +8,16 @@ on:
       GIST_BADGES_ID:
         required: true
 
+env:
+  PREFIX_TEST_NAMES_WITH_FILE: true
+
 jobs:
   build:
     strategy:
       matrix:
-        test-file: ["hub-1.bats", "hub-2.bats", "hub-3.bats"]
+        go-version: ["1.20.4"]
 
-    name: "Functional tests"
+    name: "Build + tests"
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
@@ -24,36 +27,49 @@ jobs:
           sudo chmod +w /etc/machine-id
           echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
 
+    - name: "Set up Go ${{ matrix.go-version }}"
+      uses: actions/setup-go@v3
+      with:
+        go-version: ${{ matrix.go-version }}
+
     - name: "Check out CrowdSec repository"
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
       with:
         fetch-depth: 0
         submodules: true
 
-    - name: "Set up Go"
-      uses: actions/setup-go@v5
+    - name: Cache Go modules
+      uses: actions/cache@v3
       with:
-        go-version: "1.22.2"
+        path: |
+          ~/go/pkg/mod
+          ~/.cache/go-build
+          ~/Library/Caches/go-build
+          %LocalAppData%\go-build
+        key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-${{ matrix.go-version }}-go-
 
     - name: "Install bats dependencies"
       env:
         GOBIN: /usr/local/bin
       run: |
-        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
+        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
+        go install github.com/mikefarah/yq/v4@latest
+        go install github.com/cloudflare/cfssl/cmd/cfssl@master
+        go install github.com/cloudflare/cfssl/cmd/cfssljson@master
 
     - name: "Build crowdsec and fixture"
-      run: make bats-clean bats-build bats-fixture BUILD_STATIC=1
+      run: make bats-clean bats-build bats-fixture
 
     - name: "Run hub tests"
-      run: |
-          ./test/bin/generate-hub-tests
-          ./test/run-tests ./test/dyn-bats/${{ matrix.test-file }} --formatter $(pwd)/test/lib/color-formatter
+      run: make bats-test-hub
 
     - name: "Collect hub coverage"
       run: ./test/bin/collect-hub-coverage >> $GITHUB_ENV
 
     - name: "Create Parsers badge"
-      uses: schneegans/dynamic-badges-action@v1.7.0
+      uses: schneegans/dynamic-badges-action@v1.6.0
       if: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'crowdsecurity' }}
       with:
         auth: ${{ secrets.GIST_BADGES_SECRET }}
@@ -64,7 +80,7 @@ jobs:
         color: ${{ env.SCENARIO_BADGE_COLOR }}
 
     - name: "Create Scenarios badge"
-      uses: schneegans/dynamic-badges-action@v1.7.0
+      uses: schneegans/dynamic-badges-action@v1.6.0
       if: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'crowdsecurity' }}
       with:
         auth: ${{ secrets.GIST_BADGES_SECRET }}

.github/workflows/bats-mysql.yml

@@ -1,4 +1,4 @@
-name: (sub) Bats / MySQL
+name: Functional tests (MySQL)
 
 on:
   workflow_call:
@@ -7,9 +7,16 @@ on:
         required: true
         type: string
 
+env:
+  PREFIX_TEST_NAMES_WITH_FILE: true
+
 jobs:
   build:
-    name: "Functional tests"
+    strategy:
+      matrix:
+        go-version: ["1.20.4"]
+
+    name: "Build + tests"
     runs-on: ubuntu-latest
     timeout-minutes: 30
     services:
@@ -27,26 +34,41 @@ jobs:
           sudo chmod +w /etc/machine-id
           echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
 
+    - name: "Set up Go ${{ matrix.go-version }}"
+      uses: actions/setup-go@v3
+      with:
+        go-version: ${{ matrix.go-version }}
+
     - name: "Check out CrowdSec repository"
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
       with:
         fetch-depth: 0
         submodules: true
 
-    - name: "Set up Go"
-      uses: actions/setup-go@v5
+    - name: Cache Go modules
+      uses: actions/cache@v3
       with:
-        go-version: "1.22.2"
+        path: |
+          ~/go/pkg/mod
+          ~/.cache/go-build
+          ~/Library/Caches/go-build
+          %LocalAppData%\go-build
+        key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-${{ matrix.go-version }}-go-
 
     - name: "Install bats dependencies"
       env:
         GOBIN: /usr/local/bin
       run: |
-        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
+        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
+        go install github.com/mikefarah/yq/v4@latest
+        go install github.com/cloudflare/cfssl/cmd/cfssl@master
+        go install github.com/cloudflare/cfssl/cmd/cfssljson@master
 
     - name: "Build crowdsec and fixture"
       run: |
-        make clean bats-build bats-fixture BUILD_STATIC=1
+        make clean bats-build bats-fixture
       env:
         DB_BACKEND: mysql
         MYSQL_HOST: 127.0.0.1
@@ -55,7 +77,7 @@ jobs:
         MYSQL_USER: root
 
     - name: "Run tests"
-      run: ./test/run-tests ./test/bats --formatter $(pwd)/test/lib/color-formatter
+      run: make bats-test
       env:
         DB_BACKEND: mysql
         MYSQL_HOST: 127.0.0.1

.github/workflows/bats-postgres.yml

@@ -1,16 +1,23 @@
-name: (sub) Bats / Postgres
+name: Functional tests (Postgres)
 
 on:
   workflow_call:
 
+env:
+  PREFIX_TEST_NAMES_WITH_FILE: true
+
 jobs:
   build:
-    name: "Functional tests"
+    strategy:
+      matrix:
+        go-version: ["1.20.4"]
+
+    name: "Build + tests"
     runs-on: ubuntu-latest
     timeout-minutes: 30
     services:
       database:
-        image: postgres:16
+        image: postgres:14
         env:
           POSTGRES_PASSWORD: "secret"
         ports:
@@ -23,39 +30,46 @@ jobs:
 
     steps:
 
-    - name: "Install pg_dump v16"
-      # we can remove this when it's released on ubuntu-latest
-      run: |
-          sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
-          wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /etc/apt/trusted.gpg.d/pgdg.asc &>/dev/null
-          sudo apt update
-          sudo apt -qq -y -o=Dpkg::Use-Pty=0 install postgresql-client-16
-
     - name: "Force machineid"
       run: |
           sudo chmod +w /etc/machine-id
           echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
 
+    - name: "Set up Go ${{ matrix.go-version }}"
+      uses: actions/setup-go@v3
+      with:
+        go-version: ${{ matrix.go-version }}
+
     - name: "Check out CrowdSec repository"
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
       with:
         fetch-depth: 0
         submodules: true
 
-    - name: "Set up Go"
-      uses: actions/setup-go@v5
+    - name: Cache Go modules
+      uses: actions/cache@v3
       with:
-        go-version: "1.22.2"
+        path: |
+          ~/go/pkg/mod
+          ~/.cache/go-build
+          ~/Library/Caches/go-build
+          %LocalAppData%\go-build
+        key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-${{ matrix.go-version }}-go-
 
     - name: "Install bats dependencies"
       env:
         GOBIN: /usr/local/bin
       run: |
-        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
+        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
+        go install github.com/mikefarah/yq/v4@latest
+        go install github.com/cloudflare/cfssl/cmd/cfssl@master
+        go install github.com/cloudflare/cfssl/cmd/cfssljson@master
 
     - name: "Build crowdsec and fixture (DB_BACKEND: pgx)"
       run: |
-        make clean bats-build bats-fixture BUILD_STATIC=1
+        make clean bats-build bats-fixture
       env:
         DB_BACKEND: pgx
         PGHOST: 127.0.0.1
@@ -64,7 +78,7 @@ jobs:
         PGUSER: postgres
 
     - name: "Run tests (DB_BACKEND: pgx)"
-      run: ./test/run-tests ./test/bats --formatter $(pwd)/test/lib/color-formatter
+      run: make bats-test
       env:
         DB_BACKEND: pgx
         PGHOST: 127.0.0.1

.github/workflows/bats-sqlite-coverage.yml

@@ -1,14 +1,19 @@
-name: (sub) Bats / sqlite + coverage
+name: Functional tests (sqlite)
 
 on:
   workflow_call:
 
 env:
+  PREFIX_TEST_NAMES_WITH_FILE: true
   TEST_COVERAGE: true
 
 jobs:
   build:
-    name: "Functional tests"
+    strategy:
+      matrix:
+        go-version: ["1.20.4"]
+
+    name: "Build + tests"
     runs-on: ubuntu-latest
     timeout-minutes: 20
 
@@ -19,29 +24,44 @@ jobs:
           sudo chmod +w /etc/machine-id
           echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
 
+    - name: "Set up Go ${{ matrix.go-version }}"
+      uses: actions/setup-go@v3
+      with:
+        go-version: ${{ matrix.go-version }}
+
     - name: "Check out CrowdSec repository"
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
       with:
         fetch-depth: 0
         submodules: true
 
-    - name: "Set up Go"
-      uses: actions/setup-go@v5
+    - name: Cache Go modules
+      uses: actions/cache@v3
       with:
-        go-version: "1.22.2"
+        path: |
+          ~/go/pkg/mod
+          ~/.cache/go-build
+          ~/Library/Caches/go-build
+          %LocalAppData%\go-build
+        key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-${{ matrix.go-version }}-go-
 
     - name: "Install bats dependencies"
       env:
         GOBIN: /usr/local/bin
       run: |
-        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
+        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
+        go install github.com/mikefarah/yq/v4@latest
+        go install github.com/cloudflare/cfssl/cmd/cfssl@master
+        go install github.com/cloudflare/cfssl/cmd/cfssljson@master
 
     - name: "Build crowdsec and fixture"
       run: |
-        make clean bats-build bats-fixture BUILD_STATIC=1
+        make clean bats-build bats-fixture
 
     - name: "Run tests"
-      run: ./test/run-tests ./test/bats --formatter $(pwd)/test/lib/color-formatter
+      run: make bats-test
 
     - name: "Collect coverage data"
       run: |
@@ -77,8 +97,7 @@ jobs:
       if: ${{ always() }}
 
     - name: Upload crowdsec coverage to codecov
-      uses: codecov/codecov-action@v4
+      uses: codecov/codecov-action@v3
       with:
         files: ./coverage-bats.out
         flags: bats
-        token: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/bats.yml

@@ -31,7 +31,7 @@ jobs:
 
   # Jobs for Postgres (and sometimes MySQL) can have failing tests on GitHub
   # CI, but they pass when run on devs' machines or in the release checks. We
-  # disable them here by default. Remove if...false to enable them.
+  # disable them here by default. Remove the if..false to enable them.
 
   mariadb:
     uses: ./.github/workflows/bats-mysql.yml

.github/workflows/cache-cleanup.yaml

@@ -1,35 +0,0 @@
-# https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#managing-caches
-
-name: cleanup caches by a branch
-on:
-  pull_request:
-    types:
-      - closed
-
-jobs:
-  cleanup:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Check out code
-        uses: actions/checkout@v4
-
-      - name: Cleanup
-        run: |
-          gh extension install actions/gh-actions-cache
-
-          REPO=${{ github.repository }}
-          BRANCH="refs/pull/${{ github.event.pull_request.number }}/merge"
-
-          echo "Fetching list of cache key"
-          cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH | cut -f 1 )
-
-          ## Setting this to not fail the workflow while deleting cache keys.
-          set +e
-          echo "Deleting caches..."
-          for cacheKey in $cacheKeysForPR
-          do
-              gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm
-          done
-          echo "Done"
-        env:
-          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/ci-windows-build-msi.yml

@@ -21,26 +21,42 @@ on:
 
 jobs:
   build:
+    strategy:
+      matrix:
+        go-version: ["1.20.4"]
+
     name: Build
     runs-on: windows-2019
 
     steps:
 
+    - name: "Set up Go ${{ matrix.go-version }}"
+      uses: actions/setup-go@v3
+      with:
+        go-version: ${{ matrix.go-version }}
+
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
       with:
         fetch-depth: 0
         submodules: false
 
-    - name: "Set up Go"
-      uses: actions/setup-go@v5
+    - name: Cache Go modules
+      uses: actions/cache@v3
       with:
-        go-version: "1.22.2"
+        path: |
+          ~/go/pkg/mod
+          ~/.cache/go-build
+          ~/Library/Caches/go-build
+          %LocalAppData%\go-build
+        key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-${{ matrix.go-version }}-go-
 
     - name: Build
-      run: make windows_installer BUILD_RE2_WASM=1
+      run: make windows_installer
     - name: Upload MSI
-      uses: actions/upload-artifact@v4
+      uses: actions/upload-artifact@v3
       with:
         path: crowdsec*msi
         name: crowdsec.msi

.github/workflows/ci_release-drafter.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       # Drafts your next Release notes as Pull Requests are merged into "master"
-      - uses: release-drafter/release-drafter@v6
+      - uses: release-drafter/release-drafter@v5
         with:
           config-name: release-drafter.yml
           # (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml

.github/workflows/codeql-analysis.yml

@@ -44,20 +44,11 @@ jobs:
 
     steps:
     - name: Checkout repository
-      uses: actions/checkout@v4
-      with:
-        # required to pick up tags for BUILD_VERSION
-        fetch-depth: 0
-
-    - name: "Set up Go"
-      uses: actions/setup-go@v5
-      with:
-        go-version: "1.22.2"
-        cache-dependency-path: "**/go.sum"
+      uses: actions/checkout@v3
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@v3
+      uses: github/codeql-action/init@v2
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -67,8 +58,8 @@ jobs:
 
     # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
     # If this step fails, then you should remove it and run the build manually (see below)
-    # - name: Autobuild
-    #   uses: github/codeql-action/autobuild@v3
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@v2
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 https://git.io/JvXDl
@@ -77,8 +68,9 @@ jobs:
     #    and modify them (or add more) to build your code if your project
     #    uses a compiled language
 
-    - run: |
-       make clean build BUILD_RE2_WASM=1
+    #- run: |
+    #   make bootstrap
+    #   make release
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@v3
+      uses: github/codeql-action/analyze@v2

.github/workflows/docker-tests.yml

@@ -15,42 +15,71 @@ on:
       - 'README.md'
 
 jobs:
-  test_flavor:
-    strategy:
-      # we could test all the flavors in a single pytest job,
-      # but let's split them (and the image build) in multiple runners for performance
-      matrix:
-        # can be slim, full or debian (no debian slim).
-        flavor: ["slim", "debian"]
-
+  test_docker_image:
     runs-on: ubuntu-latest
     timeout-minutes: 30
     steps:
 
       - name: Check out the repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         with:
           fetch-depth: 0
 
       - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+        uses: docker/setup-buildx-action@v2
         with:
           config: .github/buildkit.toml
 
-      - name: "Build image"
-        uses: docker/build-push-action@v5
+      # - name: "Build flavor: full"
+      #   uses: docker/build-push-action@v4
+      #   with:
+      #     context: .
+      #     file: ./Dockerfile
+      #     tags: crowdsecurity/crowdsec:test
+      #     target: full
+      #     platforms: linux/amd64
+      #     load: true
+      #     cache-from: type=gha
+      #     cache-to: type=gha,mode=min
+
+      - name: "Build flavor: slim"
+        uses: docker/build-push-action@v4
         with:
           context: .
-          file: ./Dockerfile${{ matrix.flavor == 'debian' && '.debian' || '' }}
-          tags: crowdsecurity/crowdsec:test${{ matrix.flavor == 'full' && '' || '-' }}${{ matrix.flavor == 'full' && '' || matrix.flavor }}
-          target: ${{ matrix.flavor == 'debian' && 'full' || matrix.flavor }}
+          file: ./Dockerfile
+          tags: crowdsecurity/crowdsec:test-slim
+          target: slim
+          platforms: linux/amd64
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=min
+
+      - name: "Build flavor: full"
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile
+          tags: crowdsecurity/crowdsec:test
+          target: full
+          platforms: linux/amd64
+          load: true
+          cache-from: type=gha
+          cache-to: type=gha,mode=min
+
+      - name: "Build flavor: full (debian)"
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile.debian
+          tags: crowdsecurity/crowdsec:test-debian
+          target: full
           platforms: linux/amd64
           load: true
           cache-from: type=gha
           cache-to: type=gha,mode=min
 
       - name: "Setup Python"
-        uses: actions/setup-python@v5
+        uses: actions/setup-python@v4
         with:
           python-version: "3.x"
 
@@ -59,15 +88,15 @@ jobs:
           cd docker/test
           python -m pip install --upgrade pipenv wheel
 
-      #- name: "Cache virtualenvs"
-      #  id: cache-pipenv
-      #  uses: actions/cache@v4
-      #  with:
-      #    path: ~/.local/share/virtualenvs
-      #    key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
+      - name: "Cache virtualenvs"
+        id: cache-pipenv
+        uses: actions/cache@v3
+        with:
+          path: ~/.local/share/virtualenvs
+          key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
 
       - name: "Install dependencies"
-        #if: steps.cache-pipenv.outputs.cache-hit != 'true'
+        if: steps.cache-pipenv.outputs.cache-hit != 'true'
         run: |
           cd docker/test
           pipenv install --deploy
@@ -78,10 +107,9 @@ jobs:
       - name: "Run tests"
         env:
           CROWDSEC_TEST_VERSION: test
-          CROWDSEC_TEST_FLAVORS: ${{ matrix.flavor }}
+          CROWDSEC_TEST_FLAVORS: slim,debian
           CROWDSEC_TEST_NETWORK: net-test
           CROWDSEC_TEST_TIMEOUT: 90
-        # running serially to reduce test flakiness
         run: |
           cd docker/test
-          pipenv run pytest -n 1 --durations=0 --color=yes
+          pipenv run pytest -n 2 --durations=0 --color=yes

.github/workflows/go-tests-windows.yml

@@ -20,25 +20,41 @@ env:
 
 jobs:
   build:
+    strategy:
+      matrix:
+        go-version: ["1.20.4"]
+
     name: "Build + tests"
     runs-on: windows-2022
 
     steps:
 
+    - name: "Set up Go ${{ matrix.go-version }}"
+      uses: actions/setup-go@v3
+      with:
+        go-version: ${{ matrix.go-version }}
+
     - name: Check out CrowdSec repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
       with:
         fetch-depth: 0
         submodules: false
 
-    - name: "Set up Go"
-      uses: actions/setup-go@v5
+    - name: Cache Go modules
+      uses: actions/cache@v3
       with:
-        go-version: "1.22.2"
+        path: |
+          ~/go/pkg/mod
+          ~/.cache/go-build
+          ~/Library/Caches/go-build
+          %LocalAppData%\go-build
+        key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-${{ matrix.go-version }}-go-
 
     - name: Build
       run: |
-        make build BUILD_RE2_WASM=1
+        make build
 
     - name: Run tests
       run: |
@@ -48,16 +64,15 @@ jobs:
         cat out.txt | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
 
     - name: Upload unit coverage to Codecov
-      uses: codecov/codecov-action@v4
+      uses: codecov/codecov-action@v3
       with:
         files: coverage.out
         flags: unit-windows
-        token: ${{ secrets.CODECOV_TOKEN }}
 
     - name: golangci-lint
-      uses: golangci/golangci-lint-action@v4
+      uses: golangci/golangci-lint-action@v3
       with:
-        version: v1.57
+        version: v1.51
         args: --issues-exit-code=1 --timeout 10m
         only-new-issues: false
         # the cache is already managed above, enabling it here

.github/workflows/go-tests.yml

@@ -24,18 +24,23 @@ env:
   RICHGO_FORCE_COLOR: 1
   AWS_HOST: localstack
   # these are to mimic aws config
-  AWS_ACCESS_KEY_ID: test
-  AWS_SECRET_ACCESS_KEY: test
+  AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
+  AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
   AWS_REGION: us-east-1
+  KINESIS_INITIALIZE_STREAMS: "stream-1-shard:1,stream-2-shards:2"
   CROWDSEC_FEATURE_DISABLE_HTTP_RETRY_BACKOFF: true
 
 jobs:
   build:
+    strategy:
+      matrix:
+        go-version: ["1.20.4"]
+
     name: "Build + tests"
     runs-on: ubuntu-latest
     services:
       localstack:
-        image: localstack/localstack:3.0
+        image: localstack/localstack:1.3.0
         ports:
         - 4566:4566  # Localstack exposes all services on the same port
         env:
@@ -44,7 +49,7 @@ jobs:
           KINESIS_ERROR_PROBABILITY: ""
           DOCKER_HOST: unix:///var/run/docker.sock
           KINESIS_INITIALIZE_STREAMS: ${{ env.KINESIS_INITIALIZE_STREAMS }}
-          LOCALSTACK_HOST: ${{ env.AWS_HOST }}  # Required so that resource urls are provided properly
+          HOSTNAME_EXTERNAL: ${{ env.AWS_HOST }}  # Required so that resource urls are provided properly
           # e.g sqs url will get localhost if we don't set this env to map our service
         options: >-
           --name=localstack
@@ -53,7 +58,7 @@ jobs:
           --health-timeout=5s
           --health-retries=3
       zoo1:
-        image: confluentinc/cp-zookeeper:7.4.3
+        image: confluentinc/cp-zookeeper:7.3.0
         ports:
           - "2181:2181"
         env:
@@ -103,62 +108,49 @@ jobs:
           --health-timeout 10s
           --health-retries 5
 
-      loki:
-        image: grafana/loki:2.9.1
-        ports:
-          - "3100:3100"
-        options: >-
-          --name=loki1
-          --health-cmd "wget -q -O - http://localhost:3100/ready | grep 'ready'"
-          --health-interval 30s
-          --health-timeout 10s
-          --health-retries 5
-          --health-start-period 30s
-
     steps:
 
+    - name: "Set up Go ${{ matrix.go-version }}"
+      uses: actions/setup-go@v3
+      with:
+        go-version: ${{ matrix.go-version }}
+
     - name: Check out CrowdSec repository
-      uses: actions/checkout@v4
+      uses: actions/checkout@v3
       with:
         fetch-depth: 0
         submodules: false
 
-    - name: "Set up Go"
-      uses: actions/setup-go@v5
+    - name: Cache Go modules
+      uses: actions/cache@v3
       with:
-        go-version: "1.22.2"
+        path: |
+          ~/go/pkg/mod
+          ~/.cache/go-build
+          ~/Library/Caches/go-build
+          %LocalAppData%\go-build
+        key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: |
+          ${{ runner.os }}-${{ matrix.go-version }}-go-
 
-    - name: Create localstack streams
+    - name: Build and run tests
       run: |
-          aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-1-shard --shard-count 1
-          aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-2-shards --shard-count 2
-
-    - name: Build and run tests, static
-      run: |
-        sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev
         go install github.com/ory/go-acc@v0.2.8
         go install github.com/kyoh86/richgo@v0.3.10
         set -o pipefail
-        make build BUILD_STATIC=1
-        make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
-
-    - name: Run tests again, dynamic
-      run: |
-        make clean build
-        set -o pipefail
-        make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
+        make build
+        make go-acc | richgo testfilter
 
     - name: Upload unit coverage to Codecov
-      uses: codecov/codecov-action@v4
+      uses: codecov/codecov-action@v3
       with:
         files: coverage.out
         flags: unit-linux
-        token: ${{ secrets.CODECOV_TOKEN }}
 
     - name: golangci-lint
-      uses: golangci/golangci-lint-action@v4
+      uses: golangci/golangci-lint-action@v3
       with:
-        version: v1.57
+        version: v1.51
         args: --issues-exit-code=1 --timeout 10m
         only-new-issues: false
         # the cache is already managed above, enabling it here

.github/workflows/governance-bot.yaml

@@ -23,7 +23,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       # Semantic versioning, lock to different version: v2, v2.0 or a commit hash.
-      - uses: BirthdayResearch/oss-governance-bot@v4
+      - uses: BirthdayResearch/oss-governance-bot@v3
         with:
           # You can use a PAT to post a comment/label/status so that it shows up as a user instead of github-actions
           github-token: ${{secrets.GITHUB_TOKEN}} # optional, default to '${{ github.token }}'

.github/workflows/publish-docker-master.yml

@@ -1,47 +0,0 @@
-name: (push-master) Publish latest Docker images
-
-on:
-  push:
-    branches: [ master ]
-    paths:
-      - 'pkg/**'
-      - 'cmd/**'
-      - 'mk/**'
-      - 'docker/docker_start.sh'
-      - 'docker/config.yaml'
-      - '.github/workflows/publish-docker-master.yml'
-      - '.github/workflows/publish-docker.yml'
-      - 'Dockerfile'
-      - 'Dockerfile.debian'
-      - 'go.mod'
-      - 'go.sum'
-      - 'Makefile'
-
-jobs:
-  dev-alpine:
-    uses: ./.github/workflows/publish-docker.yml
-    with:
-      platform: linux/amd64
-      crowdsec_version: ""
-      image_version: dev
-      latest: false
-      push: true
-      slim: false
-      debian: false
-    secrets:
-      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-
-  dev-debian:
-    uses: ./.github/workflows/publish-docker.yml
-    with:
-      platform: linux/amd64
-      crowdsec_version: ""
-      image_version: dev
-      latest: false
-      push: true
-      slim: false
-      debian: true
-    secrets:
-      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}

.github/workflows/publish-docker-release.yml

@@ -1,48 +0,0 @@
-name: (manual) Publish Docker images
-
-on:
-  workflow_dispatch:
-    inputs:
-      image_version:
-        description: Docker Image version (base tag, i.e. v1.6.0-2)
-        required: true
-      crowdsec_version:
-        description: Crowdsec version (BUILD_VERSION)
-        required: true
-      latest:
-        description: Overwrite latest (and slim) tags?
-        default: false
-        required: true
-      push:
-        description: Really push?
-        default: false
-        required: true
-
-jobs:
-  alpine:
-    uses: ./.github/workflows/publish-docker.yml
-    secrets:
-      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-    with:
-      image_version: ${{ github.event.inputs.image_version }}
-      crowdsec_version: ${{ github.event.inputs.crowdsec_version }}
-      latest: ${{ github.event.inputs.latest == 'true' }}
-      push: ${{ github.event.inputs.push == 'true' }}
-      slim: true
-      debian: false
-      platform: "linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6"
-
-  debian:
-    uses: ./.github/workflows/publish-docker.yml
-    secrets:
-      DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
-      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
-    with:
-      image_version: ${{ github.event.inputs.image_version }}
-      crowdsec_version: ${{ github.event.inputs.crowdsec_version }}
-      latest: ${{ github.event.inputs.latest == 'true' }}
-      push: ${{ github.event.inputs.push == 'true' }}
-      slim: false
-      debian: true
-      platform: "linux/amd64,linux/386,linux/arm64"

.github/workflows/publish-docker.yml

@@ -1,125 +0,0 @@
-name: (sub) Publish Docker images
-
-on:
-  workflow_call:
-    secrets:
-      DOCKER_USERNAME:
-        required: true
-      DOCKER_PASSWORD:
-        required: true
-    inputs:
-      platform:
-        required: true
-        type: string
-      image_version:
-        required: true
-        type: string
-      crowdsec_version:
-        required: true
-        type: string
-      latest:
-        required: true
-        type: boolean
-      push:
-        required: true
-        type: boolean
-      slim:
-        required: true
-        type: boolean
-      debian:
-        required: true
-        type: boolean
-
-jobs:
-  push_to_registry:
-    name: Push Docker image to registries
-    runs-on: ubuntu-latest
-    steps:
-
-      - name: Check out the repo
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
-
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
-        with:
-          config: .github/buildkit.toml
-
-      - name: Login to DockerHub
-        uses: docker/login-action@v3
-        with:
-          username: ${{ secrets.DOCKER_USERNAME }}
-          password: ${{ secrets.DOCKER_PASSWORD }}
-
-      - name: Login to GitHub Container Registry
-        uses: docker/login-action@v3
-        with:
-          registry: ghcr.io
-          username: ${{ github.repository_owner }}
-          password: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Prepare (slim)
-        if: ${{ inputs.slim }}
-        id: slim
-        run: |
-          DOCKERHUB_IMAGE=${{ secrets.DOCKER_USERNAME }}/crowdsec
-          GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
-          VERSION=${{ inputs.image_version }}
-          DEBIAN=${{ inputs.debian && '-debian' || '' }}
-          TAGS="${DOCKERHUB_IMAGE}:${VERSION}-slim${DEBIAN},${GHCR_IMAGE}:${VERSION}-slim${DEBIAN}"
-          if [[ ${{ inputs.latest }} == true ]]; then
-            TAGS=$TAGS,${DOCKERHUB_IMAGE}:slim${DEBIAN},${GHCR_IMAGE}:slim${DEBIAN}
-          fi
-          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
-          echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
-
-      - name: Prepare (full)
-        id: full
-        run: |
-          DOCKERHUB_IMAGE=${{ secrets.DOCKER_USERNAME }}/crowdsec
-          GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
-          VERSION=${{ inputs.image_version }}
-          DEBIAN=${{ inputs.debian && '-debian' || '' }}
-          TAGS="${DOCKERHUB_IMAGE}:${VERSION}${DEBIAN},${GHCR_IMAGE}:${VERSION}${DEBIAN}"
-          if [[ ${{ inputs.latest }} == true ]]; then
-            TAGS=$TAGS,${DOCKERHUB_IMAGE}:latest${DEBIAN},${GHCR_IMAGE}:latest${DEBIAN}
-          fi
-          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
-          echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
-
-      - name: Build and push image (slim)
-        if: ${{ inputs.slim }}
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ./Dockerfile${{ inputs.debian && '.debian' || '' }}
-          push: ${{ inputs.push }}
-          tags: ${{ steps.slim.outputs.tags }}
-          target: slim
-          platforms: ${{ inputs.platform }}
-          labels: |
-            org.opencontainers.image.source=${{ github.event.repository.html_url }}
-            org.opencontainers.image.created=${{ steps.slim.outputs.created }}
-            org.opencontainers.image.revision=${{ github.sha }}
-          build-args: |
-            BUILD_VERSION=${{ inputs.crowdsec_version }}
-
-      - name: Build and push image (full)
-        uses: docker/build-push-action@v5
-        with:
-          context: .
-          file: ./Dockerfile${{ inputs.debian && '.debian' || '' }}
-          push: ${{ inputs.push }}
-          tags: ${{ steps.full.outputs.tags }}
-          target: full
-          platforms: ${{ inputs.platform }}
-          labels: |
-            org.opencontainers.image.source=${{ github.event.repository.html_url }}
-            org.opencontainers.image.created=${{ steps.full.outputs.created }}
-            org.opencontainers.image.revision=${{ github.sha }}
-          build-args: |
-            BUILD_VERSION=${{ inputs.crowdsec_version }}

.github/workflows/publish-tarball-release.yml

@@ -1,40 +0,0 @@
-# .github/workflows/build-docker-image.yml
-name: Release
-
-on:
-  release:
-    types:
-      - prereleased
-
-permissions:
-  # Use write for: hub release edit
-  contents: write
-
-jobs:
-  build:
-    name: Build and upload binary package
-    runs-on: ubuntu-latest
-    steps:
-
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v4
-        with:
-          fetch-depth: 0
-          submodules: false
-
-      - name: "Set up Go"
-        uses: actions/setup-go@v5
-        with:
-          go-version: "1.22.2"
-
-      - name: Build the binaries
-        run: |
-          sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev
-          make vendor release BUILD_STATIC=1
-
-      - name: Upload to release
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: |
-          tag_name="${GITHUB_REF##*/}"
-          gh release upload "$tag_name" crowdsec-release.tgz vendor.tgz *-vendor.tar.xz

.github/workflows/publish_docker-image_on_master-debian.yml

@@ -0,0 +1,70 @@
+name: Publish Debian Docker image on Push to Master
+
+on:
+  push:
+    branches: [ master ]
+    paths:
+      - 'pkg/**'
+      - 'cmd/**'
+      - 'plugins/**'
+      - 'docker/docker_start.sh'
+      - 'docker/config.yaml'
+      - '.github/workflows/publish_docker-image_on_master-debian.yml'
+      - 'Dockerfile.debian'
+      - 'go.mod'
+      - 'go.sum'
+      - 'Makefile'
+
+jobs:
+  push_to_registry:
+    name: Push Debian Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Check out the repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Prepare
+        id: prep
+        run: |
+          DOCKER_IMAGE=crowdsecurity/crowdsec
+          GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
+          VERSION=dev-debian
+          TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}"
+          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
+          echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          config: .github/buildkit.toml
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push full image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile.debian
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.prep.outputs.tags }}
+          platforms: linux/amd64
+          labels: |
+            org.opencontainers.image.source=${{ github.event.repository.html_url }}
+            org.opencontainers.image.created=${{ steps.prep.outputs.created }}
+            org.opencontainers.image.revision=${{ github.sha }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=min

.github/workflows/publish_docker-image_on_master.yml

@@ -0,0 +1,70 @@
+name: Publish Docker image on Push to Master
+
+on:
+  push:
+    branches: [ master ]
+    paths:
+      - 'pkg/**'
+      - 'cmd/**'
+      - 'plugins/**'
+      - 'docker/docker_start.sh'
+      - 'docker/config.yaml'
+      - '.github/workflows/publish_docker-image_on_master.yml'
+      - 'Dockerfile'
+      - 'go.mod'
+      - 'go.sum'
+      - 'Makefile'
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: Check out the repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Prepare
+        id: prep
+        run: |
+          DOCKER_IMAGE=crowdsecurity/crowdsec
+          GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
+          VERSION=dev
+          TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}"
+          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
+          echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          config: .github/buildkit.toml
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push full image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.prep.outputs.tags }}
+          platforms: linux/amd64
+          labels: |
+            org.opencontainers.image.source=${{ github.event.repository.html_url }}
+            org.opencontainers.image.created=${{ steps.prep.outputs.created }}
+            org.opencontainers.image.revision=${{ github.sha }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=min

.github/workflows/release_publish-package.yml

@@ -0,0 +1,54 @@
+# .github/workflows/build-docker-image.yml
+name: build
+
+on:
+  release:
+    types:
+      - prereleased
+
+permissions:
+  # Use write for: hub release edit
+  contents: write
+
+jobs:
+  build:
+    strategy:
+      matrix:
+        go-version: ["1.20.4"]
+
+    name: Build and upload binary package
+    runs-on: ubuntu-latest
+    steps:
+
+      - name: "Set up Go ${{ matrix.go-version }}"
+        uses: actions/setup-go@v3
+        with:
+          go-version: ${{ matrix.go-version }}
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+          submodules: false
+
+      - name: Cache Go modules
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/go/pkg/mod
+            ~/.cache/go-build
+            ~/Library/Caches/go-build
+            %LocalAppData%\go-build
+          key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-${{ matrix.go-version }}-go-
+
+      - name: Build the binaries
+        run: make release
+
+      - name: Upload to release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          tag_name="${GITHUB_REF##*/}"
+          hub release edit -a crowdsec-release.tgz -m "" "$tag_name"

.github/workflows/release_publish_docker-image-debian.yml

@@ -0,0 +1,61 @@
+name: Publish Docker Debian image
+
+on:
+  release:
+    types:
+      - released
+      - prereleased
+  workflow_dispatch:
+
+jobs:
+  push_to_registry:
+    name: Push Docker debian image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Prepare
+        id: prep
+        run: |
+          DOCKER_IMAGE=crowdsecurity/crowdsec
+          VERSION=bullseye
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION=${GITHUB_REF#refs/tags/}
+          elif [[ $GITHUB_REF == refs/heads/* ]]; then
+            VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -E 's#/+#-#g')
+          elif [[ $GITHUB_REF == refs/pull/* ]]; then
+            VERSION=pr-${{ github.event.number }}
+          fi
+          TAGS="${DOCKER_IMAGE}:${VERSION}-debian"
+          if [[ "${{ github.event.action }}" == "released" ]]; then
+            TAGS=$TAGS,${DOCKER_IMAGE}:latest-debian
+          fi
+          echo "version=${VERSION}" >> $GITHUB_OUTPUT
+          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
+          echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          config: .github/buildkit.toml
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+      - name: Build and push
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile.debian
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.prep.outputs.tags }}
+          platforms: linux/amd64,linux/arm64,linux/386
+          labels: |
+            org.opencontainers.image.source=${{ github.event.repository.html_url }}
+            org.opencontainers.image.created=${{ steps.prep.outputs.created }}
+            org.opencontainers.image.revision=${{ github.sha }}

.github/workflows/release_publish_docker-image.yml

@@ -0,0 +1,86 @@
+name: Publish Docker image
+
+on:
+  release:
+    types:
+      - released
+      - prereleased
+
+jobs:
+  push_to_registry:
+    name: Push Docker image to Docker Hub
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out the repo
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+      - name: Prepare
+        id: prep
+        run: |
+          DOCKER_IMAGE=crowdsecurity/crowdsec
+          GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
+          VERSION=edge
+          if [[ $GITHUB_REF == refs/tags/* ]]; then
+            VERSION=${GITHUB_REF#refs/tags/}
+          elif [[ $GITHUB_REF == refs/heads/* ]]; then
+            VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -E 's#/+#-#g')
+          elif [[ $GITHUB_REF == refs/pull/* ]]; then
+            VERSION=pr-${{ github.event.number }}
+          fi
+          TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}"
+          TAGS_SLIM="${DOCKER_IMAGE}:${VERSION}-slim"
+          if [[ ${{ github.event.action }} == released ]]; then
+            TAGS=$TAGS,${DOCKER_IMAGE}:latest,${GHCR_IMAGE}:latest
+            TAGS_SLIM=$TAGS_SLIM,${DOCKER_IMAGE}:slim
+          fi
+          echo "version=${VERSION}" >> $GITHUB_OUTPUT
+          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
+          echo "tags_slim=${TAGS_SLIM}" >> $GITHUB_OUTPUT
+          echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v2
+        with:
+          config: .github/buildkit.toml
+
+      - name: Login to DockerHub
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.DOCKER_USERNAME }}
+          password: ${{ secrets.DOCKER_PASSWORD }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@v2
+        with:
+          registry: ghcr.io
+          username: ${{ github.repository_owner }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Build and push slim image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.prep.outputs.tags_slim }}
+          target: slim
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6,linux/386
+          labels: |
+            org.opencontainers.image.source=${{ github.event.repository.html_url }}
+            org.opencontainers.image.created=${{ steps.prep.outputs.created }}
+            org.opencontainers.image.revision=${{ github.sha }}
+
+      - name: Build and push full image
+        uses: docker/build-push-action@v4
+        with:
+          context: .
+          file: ./Dockerfile
+          push: ${{ github.event_name != 'pull_request' }}
+          tags: ${{ steps.prep.outputs.tags }}
+          platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6,linux/386
+          labels: |
+            org.opencontainers.image.source=${{ github.event.repository.html_url }}
+            org.opencontainers.image.created=${{ steps.prep.outputs.created }}
+            org.opencontainers.image.revision=${{ github.sha }}

.github/workflows/update_docker_hub_doc.yml

@@ -1,4 +1,4 @@
-name: (push-master) Update Docker Hub README
+name: Update Docker Hub README
 
 on:
   push:
@@ -13,7 +13,7 @@ jobs:
     steps:
       -
         name: Check out the repo
-        uses: actions/checkout@v4
+        uses: actions/checkout@v3
         if: ${{ github.repository_owner == 'crowdsecurity' }}
       -
         name: Update docker hub README

.gitignore

@@ -6,10 +6,7 @@
 *.dylib
 *~
 .pc
-
-# IDEs
 .vscode
-.idea
 
 # If vendor is included, allow prebuilt (wasm?) libraries.
 !vendor/**/*.so
@@ -18,9 +15,6 @@
 *.test
 *.cover
 
-# Test dependencies
-test/tools/*
-
 # VMs used for dev/test
 
 .vagrant
@@ -36,15 +30,17 @@ test/coverage/*
 *.swp
 *.swo
 
-# Dependencies are not vendored by default, but a tarball is created by "make vendor"
-# and provided in the release. Used by gentoo, etc.
-vendor/
-vendor.tgz
+# Dependency directories (remove the comment below to include it)
+# vendor/
 
 # crowdsec binaries
 cmd/crowdsec-cli/cscli
 cmd/crowdsec/crowdsec
-cmd/notification-*/notification-*
+plugins/notifications/http/notification-http
+plugins/notifications/slack/notification-slack
+plugins/notifications/splunk/notification-splunk
+plugins/notifications/email/notification-email
+plugins/notifications/dummy/notification-dummy
 
 # Test cache (downloaded files)
 .cache

.gitmodules

@@ -14,3 +14,7 @@
 [submodule "tests/lib/bats-mock"]
 	path = test/lib/bats-mock
 	url = https://github.com/crowdsecurity/bats-mock.git
+[submodule "coraza"]
+	path = coraza
+	url = http://github.com/buixor/coraza 
+	branch = testing

.golangci.yml

@@ -1,66 +1,38 @@
 # https://github.com/golangci/golangci-lint/blob/master/.golangci.reference.yml
 
+run:
+  skip-dirs:
+    - pkg/time/rate
+  skip-files:
+    - pkg/database/ent/generate.go
+    - pkg/yamlpatch/merge.go
+    - pkg/yamlpatch/merge_test.go
+
 linters-settings:
-  cyclop:
-    # lower this after refactoring
-    max-complexity: 48
-
-  gci:
-    sections:
-     - standard
-     - default
-     - prefix(github.com/crowdsecurity)
-     - prefix(github.com/crowdsecurity/crowdsec)
-
-  gomoddirectives:
-    replace-allow-list:
-      - golang.org/x/time/rate
-
-  gocognit:
-    # lower this after refactoring
-    min-complexity: 145
-
   gocyclo:
-    # lower this after refactoring
-    min-complexity: 48
+    min-complexity: 30
 
   funlen:
     # Checks the number of lines in a function.
     # If lower than 0, disable the check.
     # Default: 60
-    # lower this after refactoring
-    lines: 437
+    lines: -1
     # Checks the number of statements in a function.
     # If lower than 0, disable the check.
     # Default: 40
-    # lower this after refactoring
-    statements: 122
+    statements: -1
 
   govet:
-    enable-all: true
-    disable:
-     - reflectvaluecompare
-     - fieldalignment
+    check-shadowing: true
 
   lll:
-    # lower this after refactoring
-    line-length: 2607
-
-  maintidx:
-    # raise this after refactoring
-    under: 11
+    line-length: 140
 
   misspell:
     locale: US
 
-  nestif:
-    # lower this after refactoring
-    min-complexity: 28
-
-  nlreturn:
-    block-size: 5
-
   nolintlint:
+    allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
     allow-unused: false # report any unused nolint directives
     require-explanation: false # don't require an explanation for nolint directives
     require-specific: false # don't require nolint directives to be specific about which linter is being skipped
@@ -68,183 +40,104 @@ linters-settings:
   interfacebloat:
     max: 12
 
-  depguard:
-    rules:
-      wrap:
-        deny:
-          - pkg: "github.com/pkg/errors"
-            desc: "errors.Wrap() is deprecated in favor of fmt.Errorf()"
-        files:
-          - "!**/pkg/database/*.go"
-          - "!**/pkg/exprhelpers/*.go"
-          - "!**/pkg/acquisition/modules/appsec/appsec.go"
-          - "!**/pkg/acquisition/modules/loki/internal/lokiclient/loki_client.go"
-          - "!**/pkg/apiserver/controllers/v1/errors.go"
-      yaml:
-        files:
-          - "!**/pkg/acquisition/acquisition.go"
-          - "!**/pkg/acquisition/acquisition_test.go"
-          - "!**/pkg/acquisition/modules/appsec/appsec.go"
-          - "!**/pkg/acquisition/modules/cloudwatch/cloudwatch.go"
-          - "!**/pkg/acquisition/modules/docker/docker.go"
-          - "!**/pkg/acquisition/modules/file/file.go"
-          - "!**/pkg/acquisition/modules/journalctl/journalctl.go"
-          - "!**/pkg/acquisition/modules/kafka/kafka.go"
-          - "!**/pkg/acquisition/modules/kinesis/kinesis.go"
-          - "!**/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go"
-          - "!**/pkg/acquisition/modules/loki/loki.go"
-          - "!**/pkg/acquisition/modules/loki/timestamp_test.go"
-          - "!**/pkg/acquisition/modules/s3/s3.go"
-          - "!**/pkg/acquisition/modules/syslog/syslog.go"
-          - "!**/pkg/acquisition/modules/wineventlog/wineventlog_windows.go"
-          - "!**/pkg/appsec/appsec.go"
-          - "!**/pkg/appsec/loader.go"
-          - "!**/pkg/csplugin/broker.go"
-          - "!**/pkg/csplugin/broker_test.go"
-          - "!**/pkg/dumps/bucket_dump.go"
-          - "!**/pkg/dumps/parser_dump.go"
-          - "!**/pkg/hubtest/coverage.go"
-          - "!**/pkg/hubtest/hubtest_item.go"
-          - "!**/pkg/hubtest/parser_assert.go"
-          - "!**/pkg/hubtest/scenario_assert.go"
-          - "!**/pkg/leakybucket/buckets_test.go"
-          - "!**/pkg/leakybucket/manager_load.go"
-          - "!**/pkg/metabase/metabase.go"
-          - "!**/pkg/parser/node.go"
-          - "!**/pkg/parser/node_test.go"
-          - "!**/pkg/parser/parsing_test.go"
-          - "!**/pkg/parser/stage.go"
-        deny:
-          - pkg: "gopkg.in/yaml.v2"
-            desc: "yaml.v2 is deprecated for new code in favor of yaml.v3"
-
-  wsl:
-    # Allow blocks to end with comments
-    allow-trailing-comment: true
-
 linters:
   enable-all: true
   disable:
     #
     # DEPRECATED by golangi-lint
     #
-    - deadcode
-    - exhaustivestruct
-    - golint
-    - ifshort
-    - interfacer
-    - maligned
-    - nosnakecase
-    - scopelint
-    - structcheck
-    - varcheck
-
-    #
-    # Disabled until fixed for go 1.22
-    #
-
-    - copyloopvar         # copyloopvar is a linter detects places where loop variables are copied
-    - intrange            # intrange is a linter to find places where for loops could make use of an integer range.
+    - deadcode          # The owner seems to have abandoned the linter.  Replaced by unused.
+    - exhaustivestruct  # The owner seems to have abandoned the linter. Replaced by exhaustruct.
+    - golint            # Golint differs from gofmt. Gofmt reformats Go source code, whereas golint prints out style mistakes
+    - ifshort           # Checks that your code uses short syntax for if-statements whenever possible
+    - interfacer        # Linter that suggests narrower interface types
+    - maligned          # Tool to detect Go structs that would take less memory if their fields were sorted
+    - nosnakecase       # nosnakecase is a linter that detects snake case of variable naming and function name.
+    - scopelint         # Scopelint checks for unpinned variables in go programs
+    - structcheck       # The owner seems to have abandoned the linter.  Replaced by unused.
+    - varcheck          # The owner seems to have abandoned the linter.  Replaced by unused.
 
     #
     # Enabled
     #
 
     # - asasalint           # check for pass []any as any in variadic func(...any)
-    # - asciicheck          # checks that all code identifiers does not have non-ASCII symbols in the name
+    # - asciicheck          # Simple linter to check that your code does not contain non-ASCII identifiers
     # - bidichk             # Checks for dangerous unicode character sequences
-    # - bodyclose           # checks whether HTTP response body is closed successfully
-    # - cyclop              # checks function and package cyclomatic complexity
     # - decorder            # check declaration order and count of types, constants, variables and functions
     # - depguard            # Go linter that checks if package imports are in a list of acceptable packages
     # - dupword             # checks for duplicate words in the source code
     # - durationcheck       # check for two durations multiplied together
-    # - errcheck            # errcheck is a program for checking for unchecked errors in Go code. These unchecked errors can be critical bugs in some cases
-    # - errorlint           # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13.
-    # - execinquery         # execinquery is a linter about query string checker in Query function which reads your Go src files and warning it finds
+    # - errcheck            # Errcheck is a program for checking for unchecked errors in go programs. These unchecked errors can be critical bugs in some cases
     # - exportloopref       # checks for pointers to enclosing loop variables
     # - funlen              # Tool for detection of long functions
     # - ginkgolinter        # enforces standards of using ginkgo and gomega
-    # - gocheckcompilerdirectives # Checks that go compiler directive comments (//go:) are valid.
     # - gochecknoinits      # Checks that no init functions are present in Go code
-    # - gochecksumtype      # Run exhaustiveness checks on Go "sum types"
-    # - gocognit            # Computes and checks the cognitive complexity of functions
     # - gocritic            # Provides diagnostics that check for bugs, performance and style issues.
-    # - gocyclo             # Computes and checks the cyclomatic complexity of functions
     # - goheader            # Checks is file header matches to pattern
     # - gomoddirectives     # Manage the use of 'replace', 'retract', and 'excludes' directives in go.mod.
     # - gomodguard          # Allow and block list linter for direct Go module dependencies. This is different from depguard where there are different block types for example version constraints and module recommendations.
     # - goprintffuncname    # Checks that printf-like functions are named with `f` at the end
-    # - gosimple            # (megacheck): Linter for Go source code that specializes in simplifying code
-    # - gosmopolitan        # Report certain i18n/l10n anti-patterns in your Go codebase
-    # - govet               # (vet, vetshadow): Vet examines Go source code and reports suspicious constructs. It is roughly the same as 'go vet' and uses its passes.
-    # - grouper             # Analyze expression groups.
+    # - gosimple            # (megacheck): Linter for Go source code that specializes in simplifying a code
+    # - govet               # (vet, vetshadow): Vet examines Go source code and reports suspicious constructs, such as Printf calls whose arguments do not align with the format string
+    # - grouper             # An analyzer to analyze expression groups.
     # - importas            # Enforces consistent import aliases
     # - ineffassign         # Detects when assignments to existing variables are not used
     # - interfacebloat      # A linter that checks the number of methods inside an interface.
-    # - lll                 # Reports long lines
-    # - loggercheck         # (logrlint): Checks key value pairs for common logger libraries (kitlog,klog,logr,zap).
     # - logrlint            # Check logr arguments.
-    # - maintidx            # maintidx measures the maintainability index of each function.
     # - makezero            # Finds slice declarations with non-zero initial length
-    # - mirror              # reports wrong mirror patterns of bytes/strings usage
-    # - misspell            # Finds commonly misspelled English words
-    # - nakedret            # Checks that functions with naked returns are not longer than a maximum size (can be zero).
-    # - nestif              # Reports deeply nested if statements
+    # - misspell            # Finds commonly misspelled English words in comments
     # - nilerr              # Finds the code that returns nil even if it checks that the error is not nil.
     # - nolintlint          # Reports ill-formed or insufficient nolint directives
-    # - nonamedreturns      # Reports all named returns
-    # - nosprintfhostport   # Checks for misuse of Sprintf to construct a host with port in a URL.
-    # - perfsprint          # Checks that fmt.Sprintf can be replaced with a faster alternative.
     # - predeclared         # find code that shadows one of Go's predeclared identifiers
     # - reassign            # Checks that package variables are not reassigned
-    # - rowserrcheck        # checks whether Rows.Err of rows is checked successfully
-    # - sloglint            # ensure consistent code style when using log/slog
-    # - spancheck           # Checks for mistakes with OpenTelemetry/Census spans.
-    # - sqlclosecheck       # Checks that sql.Rows, sql.Stmt, sqlx.NamedStmt, pgx.Query are closed.
-    # - staticcheck         # (megacheck): It's a set of rules from staticcheck. It's not the same thing as the staticcheck binary. The author of staticcheck doesn't support or approve the use of staticcheck as a library inside golangci-lint.
-    # - tenv                # tenv is analyzer that detects using os.Setenv instead of t.Setenv since Go1.17
+    # - rowserrcheck        # checks whether Err of rows is checked successfully
+    # - sqlclosecheck       # Checks that sql.Rows and sql.Stmt are closed.
+    # - staticcheck         # (megacheck): Staticcheck is a go vet on steroids, applying a ton of static analysis checks
     # - testableexamples    # linter checks if examples are testable (have an expected output)
-    # - testifylint         # Checks usage of github.com/stretchr/testify.
+    # - tenv                # tenv is analyzer that detects using os.Setenv instead of t.Setenv since Go1.17
     # - tparallel           # tparallel detects inappropriate usage of t.Parallel() method in your Go test codes
+    # - typecheck           # Like the front-end of a Go compiler, parses and type-checks Go code
     # - unconvert           # Remove unnecessary type conversions
     # - unused              # (megacheck): Checks Go code for unused constants, variables, functions and types
     # - usestdlibvars       # A linter that detect the possibility to use variables/constants from the Go standard library.
-    # - wastedassign        # Finds wasted assignment statements
-    # - zerologlint         # Detects the wrong usage of `zerolog` that a user forgets to dispatch with `Send` or `Msg`
 
     #
     # Recommended? (easy)
     #
 
     - dogsled               # Checks assignments with too many blank identifiers (e.g. x, _, _, _, := f())
-    - errchkjson            # Checks types passed to the json encoding functions. Reports unsupported types and reports occations, where the check for the returned error can be omitted.
+    - errchkjson            # Checks types passed to the json encoding functions. Reports unsupported types and optionally reports occations, where the check for the returned error can be omitted.
+    - errorlint             # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13.
     - exhaustive            # check exhaustiveness of enum switch statements
     - gci                   # Gci control golang package import order and make it always deterministic.
     - godot                 # Check if comments end in a period
     - gofmt                 # Gofmt checks whether code was gofmt-ed. By default this tool runs with -s option to check for code simplification
-    - goimports             # Check import statements are formatted according to the 'goimport' command. Reformat imports in autofix mode.
+    - goimports             # In addition to fixing imports, goimports also formats your code in the same style as gofmt.
     - gosec                 # (gas): Inspects source code for security problems
-    - inamedparam           # reports interfaces with unnamed method parameters
+    - lll                   # Reports long lines
     - musttag               # enforce field tags in (un)marshaled structs
+    - nakedret              # Finds naked returns in functions greater than a specified function length
+    - nonamedreturns        # Reports all named returns
+    - nosprintfhostport     # Checks for misuse of Sprintf to construct a host with port in a URL.
     - promlinter            # Check Prometheus metrics naming via promlint
-    - protogetter           # Reports direct reads from proto message fields when getters should be used
     - revive                # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint.
-    - tagalign              # check that struct tags are well aligned
-    - thelper               # thelper detects tests helpers which is not start with t.Helper() method.
+    - thelper               # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers
+    - wastedassign          # wastedassign finds wasted assignment statements.
     - wrapcheck             # Checks that errors returned from external packages are wrapped
 
     #
     # Recommended? (requires some work)
     #
 
+    - bodyclose             # checks whether HTTP response body is closed successfully
     - containedctx          # containedctx is a linter that detects struct contained context.Context field
-    - contextcheck          # check whether the function uses a non-inherited context
+    - contextcheck          # check the function whether use a non-inherited context
     - errname               # Checks that sentinel errors are prefixed with the `Err` and error types are suffixed with the `Error`.
     - gomnd                 # An analyzer to detect magic numbers.
     - ireturn               # Accept Interfaces, Return Concrete Types
     - nilnil                # Checks that there is no simultaneous return of `nil` error and an invalid value.
-    - noctx                 # Finds sending http request without context.Context
+    - noctx                 # noctx finds sending http request without context.Context
     - unparam               # Reports unused function parameters
 
     #
@@ -253,25 +146,31 @@ linters:
 
     - gofumpt               # Gofumpt checks whether code was gofumpt-ed.
     - nlreturn              # nlreturn checks for a new line before return and branch statements to increase code clarity
-    - whitespace            # Whitespace is a linter that checks for unnecessary newlines at the start and end of functions, if, for, etc.
-    - wsl                   # add or remove empty lines
+    - whitespace            # Tool for detection of leading and trailing whitespace
+    - wsl                   # Whitespace Linter - Forces you to use empty lines!
 
     #
     # Well intended, but not ready for this
     #
+    - cyclop                # checks function and package cyclomatic complexity
     - dupl                  # Tool for code clone detection
     - forcetypeassert       # finds forced type assertions
+    - gocognit              # Computes and checks the cognitive complexity of functions
+    - gocyclo               # Computes and checks the cyclomatic complexity of functions
     - godox                 # Tool for detection of FIXME, TODO and other comment keywords
-    - goerr113              # Go linter to check the errors handling expressions
-    - paralleltest          # Detects missing usage of t.Parallel() method in your Go test
+    - goerr113              # Golang linter to check the errors handling expressions
+    - maintidx              # maintidx measures the maintainability index of each function.
+    - nestif                # Reports deeply nested if statements
+    - paralleltest          # paralleltest detects missing usage of t.Parallel() method in your Go test
     - testpackage           # linter that makes you use a separate _test package
 
     #
     # Too strict / too many false positives (for now?)
     #
+    - execinquery           # execinquery is a linter about query string checker in Query function which reads your Go src files and warning it finds
     - exhaustruct           # Checks if all structure fields are initialized
     - forbidigo             # Forbids identifiers
-    - gochecknoglobals      # Check that no global variables exist.
+    - gochecknoglobals      # check that no global variables exist
     - goconst               # Finds repeated strings that could be replaced by a constant
     - stylecheck            # Stylecheck is a replacement for golint
     - tagliatelle           # Checks the struct tags.
@@ -288,30 +187,29 @@ issues:
   # “Look, that’s why there’s rules, understand? So that you think before you
   # break ‘em.” ― Terry Pratchett
 
-  exclude-dirs:
-    - pkg/time/rate
-
-  exclude-files:
-    - pkg/yamlpatch/merge.go
-    - pkg/yamlpatch/merge_test.go
-
-  exclude-generated-strict: true
-
   max-issues-per-linter: 0
-  max-same-issues: 0
+  max-same-issues: 10
   exclude-rules:
-
-    # Won't fix:
+    - path: go.mod
+      text: "replacement are not allowed: golang.org/x/time/rate"
 
     # `err` is often shadowed, we may continue to do it
     - linters:
         - govet
       text: "shadow: declaration of \"err\" shadows declaration"
 
+    #
+    # errcheck
+    #
+
     - linters:
         - errcheck
       text: "Error return value of `.*` is not checked"
 
+    #
+    # gocritic
+    #
+
     - linters:
         - gocritic
       text: "ifElseChain: rewrite if-else to switch statement"
@@ -328,55 +226,6 @@ issues:
         - gocritic
       text: "commentFormatting: put a space between `//` and comment text"
 
-    # Will fix, trivial - just beware of merge conflicts
-
     - linters:
-        - perfsprint
-      text: "fmt.Sprintf can be replaced .*"
-
-    - linters:
-        - perfsprint
-      text: "fmt.Errorf can be replaced with errors.New"
-
-    #
-    # Will fix, easy but some neurons required
-    #
-
-    - linters:
-        - errorlint
-      text: "non-wrapping format verb for fmt.Errorf. Use `%w` to format errors"
-
-    - linters:
-        - errorlint
-      text: "type assertion on error will fail on wrapped errors. Use errors.As to check for specific errors"
-
-    - linters:
-        - errorlint
-      text: "type switch on error will fail on wrapped errors. Use errors.As to check for specific errors"
-
-    - linters:
-        - errorlint
-      text: "type assertion on error will fail on wrapped errors. Use errors.Is to check for specific errors"
-
-    - linters:
-        - errorlint
-      text: "comparing with .* will fail on wrapped errors. Use errors.Is to check for a specific error"
-
-    - linters:
-        - errorlint
-      text: "switch on an error will fail on wrapped errors. Use errors.Is to check for specific errors"
-
-    - linters:
-        - nosprintfhostport
-      text: "host:port in url should be constructed with net.JoinHostPort and not directly with fmt.Sprintf"
-
-    # https://github.com/timakin/bodyclose
-    - linters:
-        - bodyclose
-      text: "response body must be closed"
-
-    # named/naked returns are evil, with a single exception
-    # https://go.dev/wiki/CodeReviewComments#named-result-parameters
-    - linters:
-        - nonamedreturns
-      text: "named return .* with type .* found"
+        - staticcheck
+      text: "x509.ParseCRL has been deprecated since Go 1.19: Use ParseRevocationList instead"

Dockerfile

@@ -1,63 +1,69 @@
 # vim: set ft=dockerfile:
-FROM golang:1.22.2-alpine3.18 AS build
+ARG GOVERSION=1.20.4
 
-ARG BUILD_VERSION
+FROM golang:${GOVERSION}-alpine AS build
 
 WORKDIR /go/src/crowdsec
 
-# We like to choose the release of re2 to use, and Alpine does not ship a static version anyway.
+COPY . .
+
+# Alpine does not ship a static version of re2, we can build it ourselves
+# Later versions require 'abseil', which is likewise not available in its static form
 ENV RE2_VERSION=2023-03-01
-ENV BUILD_VERSION=${BUILD_VERSION}
 
 # wizard.sh requires GNU coreutils
-RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils pkgconfig && \
+RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils icu-static re2-dev pkgconfig && \
     wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \
     tar -xzf ${RE2_VERSION}.tar.gz && \
     cd re2-${RE2_VERSION} && \
+    make && \
     make install && \
     echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
-    go install github.com/mikefarah/yq/v4@v4.43.1
-
-COPY . .
-
-RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
+    cd - && \
+    make clean release DOCKER_BUILD=1 && \
     cd crowdsec-v* && \
     ./wizard.sh --docker-mode && \
     cd - >/dev/null && \
     cscli hub update && \
     cscli collections install crowdsecurity/linux && \
-    cscli parsers install crowdsecurity/whitelists
+    cscli parsers install crowdsecurity/whitelists && \
+    go install github.com/mikefarah/yq/v4@v4.31.2
 
     # In case we need to remove agents here..
     # cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete
 
 FROM alpine:latest as slim
 
-RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata bash rsync && \
+RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata bash && \
     mkdir -p /staging/etc/crowdsec && \
     mkdir -p /staging/etc/crowdsec/acquis.d && \
     mkdir -p /staging/var/lib/crowdsec && \
     mkdir -p /var/lib/crowdsec/data
 
-COPY --from=build /go/bin/yq /usr/local/bin/crowdsec /usr/local/bin/cscli /usr/local/bin/
+COPY --from=build /go/bin/yq /usr/local/bin/yq
 COPY --from=build /etc/crowdsec /staging/etc/crowdsec
+COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
+COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
 COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
 COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
-COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec
 RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml
 
-ENTRYPOINT /bin/bash /docker_start.sh
+ENTRYPOINT /bin/bash docker_start.sh
 
-FROM slim as full
+FROM slim as plugins
 
 # Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
 # The files are here for reference, as users will need to mount a new version to be actually able to use notifications
-COPY --from=build \
-    /go/src/crowdsec/cmd/notification-email/email.yaml \
-    /go/src/crowdsec/cmd/notification-http/http.yaml \
-    /go/src/crowdsec/cmd/notification-slack/slack.yaml \
-    /go/src/crowdsec/cmd/notification-splunk/splunk.yaml \
-    /go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml \
-    /staging/etc/crowdsec/notifications/
-
+COPY --from=build /go/src/crowdsec/plugins/notifications/email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
+COPY --from=build /go/src/crowdsec/plugins/notifications/http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
+COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
+COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
 COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
+
+FROM slim as geoip
+
+COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec
+
+FROM plugins as full
+
+COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec

Dockerfile.debian

@@ -1,42 +1,32 @@
 # vim: set ft=dockerfile:
-FROM golang:1.22.2-bookworm AS build
+ARG GOVERSION=1.20.4
 
-ARG BUILD_VERSION
+FROM golang:${GOVERSION}-bullseye AS build
 
 WORKDIR /go/src/crowdsec
 
+COPY . .
+
 ENV DEBIAN_FRONTEND=noninteractive
 ENV DEBCONF_NOWARNINGS="yes"
 
-# We like to choose the release of re2 to use, the debian version is usually older.
-ENV RE2_VERSION=2023-03-01
-ENV BUILD_VERSION=${BUILD_VERSION}
-
 # wizard.sh requires GNU coreutils
 RUN apt-get update && \
-    apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \
-    wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \
-    tar -xzf ${RE2_VERSION}.tar.gz && \
-    cd re2-${RE2_VERSION} && \
-    make && \
-    make install && \
+    apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata libre2-dev && \
     echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
-    go install github.com/mikefarah/yq/v4@v4.43.1
-
-COPY . .
-
-RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
+    make clean release DOCKER_BUILD=1 && \
     cd crowdsec-v* && \
     ./wizard.sh --docker-mode && \
     cd - >/dev/null && \
     cscli hub update && \
     cscli collections install crowdsecurity/linux && \
-    cscli parsers install crowdsecurity/whitelists
+    cscli parsers install crowdsecurity/whitelists && \
+    go install github.com/mikefarah/yq/v4@v4.31.2
 
     # In case we need to remove agents here..
     # cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete
 
-FROM debian:bookworm-slim as slim
+FROM debian:bullseye-slim as slim
 
 ENV DEBIAN_FRONTEND=noninteractive
 ENV DEBCONF_NOWARNINGS="yes"
@@ -48,15 +38,19 @@ RUN apt-get update && \
     iproute2 \
     ca-certificates \
     bash \
-    tzdata \
-    rsync && \
+    tzdata && \
     mkdir -p /staging/etc/crowdsec && \
     mkdir -p /staging/etc/crowdsec/acquis.d && \
     mkdir -p /staging/var/lib/crowdsec && \
     mkdir -p /var/lib/crowdsec/data
 
-COPY --from=build /go/bin/yq /usr/local/bin/crowdsec /usr/local/bin/cscli /usr/local/bin/
+RUN echo "deb http://deb.debian.org/debian bullseye-backports main" >> /etc/apt/sources.list \
+    && apt-get update && apt-get install -t bullseye-backports -y libsystemd0
+
+COPY --from=build /go/bin/yq /usr/local/bin/yq
 COPY --from=build /etc/crowdsec /staging/etc/crowdsec
+COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
+COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
 COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
 COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
 RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml && \
@@ -68,14 +62,10 @@ FROM slim as plugins
 
 # Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
 # The files are here for reference, as users will need to mount a new version to be actually able to use notifications
-COPY --from=build \
-    /go/src/crowdsec/cmd/notification-email/email.yaml \
-    /go/src/crowdsec/cmd/notification-http/http.yaml \
-    /go/src/crowdsec/cmd/notification-slack/slack.yaml \
-    /go/src/crowdsec/cmd/notification-splunk/splunk.yaml \
-    /go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml \
-    /staging/etc/crowdsec/notifications/
-
+COPY --from=build /go/src/crowdsec/plugins/notifications/email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
+COPY --from=build /go/src/crowdsec/plugins/notifications/http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
+COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
+COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
 COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
 
 FROM slim as geoip

Makefile

@@ -1,78 +1,41 @@
 include mk/platform.mk
-include mk/gmsl
 
-# By default, this build requires the C++ re2 library to be installed.
-#
-# Debian/Ubuntu: apt install libre2-dev
-# Fedora/CentOS: dnf install re2-devel
-# FreeBSD:       pkg install re2
-# Alpine:        apk add re2-dev
-# Windows:       choco install re2
-# MacOS:         brew install re2
-
-# To build without re2, run "make BUILD_RE2_WASM=1"
-# The WASM version is slower and introduces a short delay when starting a process
-# (including cscli) so it is not recommended for production use.
-BUILD_RE2_WASM ?= 0
-
-# To build static binaries, run "make BUILD_STATIC=1".
-# On some platforms, this requires additional packages
-# (e.g. glibc-static and libstdc++-static on fedora, centos.. which are on the powertools/crb repository).
-# If the static build fails at the link stage, it might be because the static library is not provided
-# for your distribution (look for libre2.a). See the Dockerfile for an example of how to build it.
-BUILD_STATIC ?= 0
-
-# List of plugins to build
-PLUGINS ?= $(patsubst ./cmd/notification-%,%,$(wildcard ./cmd/notification-*))
-
-# Can be overriden, if you can deal with the consequences
 BUILD_REQUIRE_GO_MAJOR ?= 1
-BUILD_REQUIRE_GO_MINOR ?= 21
+BUILD_REQUIRE_GO_MINOR ?= 20
 
-#--------------------------------------
-
-GO = go
-GOTEST = $(GO) test
+GOCMD = go
+GOTEST = $(GOCMD) test
 
 BUILD_CODENAME ?= alphaga
 
 CROWDSEC_FOLDER = ./cmd/crowdsec
 CSCLI_FOLDER = ./cmd/crowdsec-cli/
-PLUGINS_DIR_PREFIX = ./cmd/notification-
+
+PLUGINS ?= $(patsubst ./plugins/notifications/%,%,$(wildcard ./plugins/notifications/*))
+PLUGINS_DIR = ./plugins/notifications
 
 CROWDSEC_BIN = crowdsec$(EXT)
 CSCLI_BIN = cscli$(EXT)
 
-# semver comparison to select the hub branch requires the version to start with "v"
-ifneq ($(call substr,$(BUILD_VERSION),1,1),v)
-    $(error BUILD_VERSION "$(BUILD_VERSION)" should start with "v")
-endif
-
 # Directory for the release files
 RELDIR = crowdsec-$(BUILD_VERSION)
 
 GO_MODULE_NAME = github.com/crowdsecurity/crowdsec
 
-# Check if a given value is considered truthy and returns "0" or "1".
-# A truthy value is one of the following: "1", "yes", or "true", case-insensitive.
-#
-# Usage:
-# ifeq ($(call bool,$(FOO)),1)
-# $(info Let's foo)
-# endif
-bool = $(if $(filter $(call lc, $1),1 yes true),1,0)
+# see if we have libre2-dev installed for C++ optimizations
+RE2_CHECK := $(shell pkg-config --libs re2 2>/dev/null)
 
 #--------------------------------------
 #
-# Define MAKE_FLAGS and LD_OPTS for the sub-makefiles in cmd/
+# Define MAKE_FLAGS and LD_OPTS for the sub-makefiles in cmd/ and plugins/
 #
 
 MAKE_FLAGS = --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
 
 LD_OPTS_VARS= \
--X 'github.com/crowdsecurity/go-cs-lib/version.Version=$(BUILD_VERSION)' \
--X 'github.com/crowdsecurity/go-cs-lib/version.BuildDate=$(BUILD_TIMESTAMP)' \
--X 'github.com/crowdsecurity/go-cs-lib/version.Tag=$(BUILD_TAG)' \
+-X 'github.com/crowdsecurity/go-cs-lib/pkg/version.Version=$(BUILD_VERSION)' \
+-X 'github.com/crowdsecurity/go-cs-lib/pkg/version.BuildDate=$(BUILD_TIMESTAMP)' \
+-X 'github.com/crowdsecurity/go-cs-lib/pkg/version.Tag=$(BUILD_TAG)' \
 -X '$(GO_MODULE_NAME)/pkg/cwversion.Codename=$(BUILD_CODENAME)' \
 -X '$(GO_MODULE_NAME)/pkg/csconfig.defaultConfigDir=$(DEFAULT_CONFIGDIR)' \
 -X '$(GO_MODULE_NAME)/pkg/csconfig.defaultDataDir=$(DEFAULT_DATADIR)'
@@ -83,107 +46,48 @@ endif
 
 GO_TAGS := netgo,osusergo,sqlite_omit_load_extension
 
-# this will be used by Go in the make target, some distributions require it
-export PKG_CONFIG_PATH:=/usr/local/lib/pkgconfig:$(PKG_CONFIG_PATH)
-
-ifeq ($(call bool,$(BUILD_RE2_WASM)),0)
-ifeq ($(PKG_CONFIG),)
-  $(error "pkg-config is not available. Please install pkg-config.")
-endif
-
-ifeq ($(RE2_CHECK),)
-RE2_FAIL := "libre2-dev is not installed, please install it or set BUILD_RE2_WASM=1 to use the WebAssembly version"
-else
+ifneq (,$(RE2_CHECK))
 # += adds a space that we don't want
 GO_TAGS := $(GO_TAGS),re2_cgo
 LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.Libre2=C++'
 endif
-endif
 
-# Build static to avoid the runtime dependency on libre2.so
-ifeq ($(call bool,$(BUILD_STATIC)),1)
-BUILD_TYPE = static
-EXTLDFLAGS := -extldflags '-static'
-else
-BUILD_TYPE = dynamic
-EXTLDFLAGS :=
-endif
+export LD_OPTS=-ldflags "-s -w -extldflags '-static' $(LD_OPTS_VARS)" \
+	-trimpath -tags $(GO_TAGS)
 
-# Build with debug symbols, and disable optimizations + inlining, to use Delve
-ifeq ($(call bool,$(DEBUG)),1)
-STRIP_SYMBOLS :=
-DISABLE_OPTIMIZATION := -gcflags "-N -l"
-else
-STRIP_SYMBOLS := -s -w
-DISABLE_OPTIMIZATION :=
-endif
-
-export LD_OPTS=-ldflags "$(STRIP_SYMBOLS) $(EXTLDFLAGS) $(LD_OPTS_VARS)" \
-	-trimpath -tags $(GO_TAGS) $(DISABLE_OPTIMIZATION)
-
-ifeq ($(call bool,$(TEST_COVERAGE)),1)
+ifneq (,$(TEST_COVERAGE))
 LD_OPTS += -cover
 endif
 
 #--------------------------------------
 
 .PHONY: build
-build: pre-build goversion crowdsec cscli plugins  ## Build crowdsec, cscli and plugins
+build: pre-build goversion crowdsec cscli plugins
 
 .PHONY: pre-build
-pre-build:  ## Sanity checks and build information
-	$(info Building $(BUILD_VERSION) ($(BUILD_TAG)) $(BUILD_TYPE) for $(GOOS)/$(GOARCH))
-
-ifneq (,$(RE2_FAIL))
-	$(error $(RE2_FAIL))
+pre-build:
+ifdef BUILD_STATIC
+	$(warning WARNING: The BUILD_STATIC variable is deprecated and has no effect. Builds are static by default since v1.5.0.)
 endif
-
+	$(info Building $(BUILD_VERSION) ($(BUILD_TAG)) for $(GOOS)/$(GOARCH))
 ifneq (,$(RE2_CHECK))
 	$(info Using C++ regexp library)
 else
 	$(info Fallback to WebAssembly regexp library. To use the C++ version, make sure you have installed libre2-dev and pkg-config.)
 endif
-
-ifeq ($(call bool,$(DEBUG)),1)
-	$(info Building with debug symbols and disabled optimizations)
-endif
-
-ifeq ($(call bool,$(TEST_COVERAGE)),1)
-	$(info Test coverage collection enabled)
-endif
-
-# intentional, empty line
 	$(info )
 
 .PHONY: all
-all: clean test build  ## Clean, test and build (requires localstack)
+all: clean test build
 
 .PHONY: plugins
-plugins:  ## Build notification plugins
+plugins:
 	@$(foreach plugin,$(PLUGINS), \
-		$(MAKE) -C $(PLUGINS_DIR_PREFIX)$(plugin) build $(MAKE_FLAGS); \
+		$(MAKE) -C $(PLUGINS_DIR)/$(plugin) build $(MAKE_FLAGS); \
 	)
 
-# same as "$(MAKE) -f debian/rules clean" but without the dependency on debhelper
-.PHONY: clean-debian
-clean-debian:
-	@$(RM) -r debian/crowdsec
-	@$(RM) -r debian/crowdsec
-	@$(RM) -r debian/files
-	@$(RM) -r debian/.debhelper
-	@$(RM) -r debian/*.substvars
-	@$(RM) -r debian/*-stamp
-
-.PHONY: clean-rpm
-clean-rpm:
-	@$(RM) -r rpm/BUILD
-	@$(RM) -r rpm/BUILDROOT
-	@$(RM) -r rpm/RPMS
-	@$(RM) -r rpm/SOURCES/*.tar.gz
-	@$(RM) -r rpm/SRPMS
-
 .PHONY: clean
-clean: clean-debian clean-rpm testclean  ## Remove build artifacts
+clean: testclean
 	@$(MAKE) -C $(CROWDSEC_FOLDER) clean $(MAKE_FLAGS)
 	@$(MAKE) -C $(CSCLI_FOLDER) clean $(MAKE_FLAGS)
 	@$(RM) $(CROWDSEC_BIN) $(WIN_IGNORE_ERR)
@@ -191,65 +95,55 @@ clean: clean-debian clean-rpm testclean  ## Remove build artifacts
 	@$(RM) *.log $(WIN_IGNORE_ERR)
 	@$(RM) crowdsec-release.tgz $(WIN_IGNORE_ERR)
 	@$(foreach plugin,$(PLUGINS), \
-		$(MAKE) -C $(PLUGINS_DIR_PREFIX)$(plugin) clean $(MAKE_FLAGS); \
+		$(MAKE) -C $(PLUGINS_DIR)/$(plugin) clean $(MAKE_FLAGS); \
 	)
 
 .PHONY: cscli
-cscli: goversion  ## Build cscli
+cscli: goversion
 	@$(MAKE) -C $(CSCLI_FOLDER) build $(MAKE_FLAGS)
 
 .PHONY: crowdsec
-crowdsec: goversion  ## Build crowdsec
+crowdsec: goversion
 	@$(MAKE) -C $(CROWDSEC_FOLDER) build $(MAKE_FLAGS)
 
-.PHONY: generate
-generate:  ## Generate code for the database and APIs
-	$(GO) generate ./pkg/database/ent
-	$(GO) generate ./pkg/models
-
 .PHONY: testclean
-testclean: bats-clean  ## Remove test artifacts
+testclean: bats-clean
 	@$(RM) pkg/apiserver/ent $(WIN_IGNORE_ERR)
 	@$(RM) pkg/cwhub/hubdir $(WIN_IGNORE_ERR)
 	@$(RM) pkg/cwhub/install $(WIN_IGNORE_ERR)
 	@$(RM) pkg/types/example.txt $(WIN_IGNORE_ERR)
 
-# for the tests with localstack
 export AWS_ENDPOINT_FORCE=http://localhost:4566
-export AWS_ACCESS_KEY_ID=test
-export AWS_SECRET_ACCESS_KEY=test
+export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
+export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
 
 testenv:
 	@echo 'NOTE: You need Docker, docker-compose and run "make localstack" in a separate shell ("make localstack-stop" to terminate it)'
 
 .PHONY: test
-test: testenv goversion  ## Run unit tests with localstack
+test: testenv goversion
 	$(GOTEST) $(LD_OPTS) ./...
 
 .PHONY: go-acc
-go-acc: testenv goversion  ## Run unit tests with localstack + coverage
-	go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS)
+go-acc: testenv goversion
+	go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS) | \
+		sed 's/ *coverage:.*of statements in.*//'
 
-# mock AWS services
 .PHONY: localstack
-localstack:  ## Run localstack containers (required for unit testing)
+localstack:
 	docker-compose -f test/localstack/docker-compose.yml up
 
 .PHONY: localstack-stop
-localstack-stop:  ## Stop localstack containers
+localstack-stop:
 	docker-compose -f test/localstack/docker-compose.yml down
 
-# build vendor.tgz to be distributed with the release
 .PHONY: vendor
-vendor: vendor-remove  ## CI only - vendor dependencies and archive them for packaging
-	$(GO) mod vendor
-	tar czf vendor.tgz vendor
-	tar --create --auto-compress --file=$(RELDIR)-vendor.tar.xz vendor
-
-# remove vendor directories and vendor.tgz
-.PHONY: vendor-remove
-vendor-remove:  ## Remove vendor dependencies and archives
-	$(RM) vendor vendor.tgz *-vendor.tar.xz
+vendor:
+	@echo "Vendoring dependencies"
+	@$(GOCMD) mod vendor
+	@$(foreach plugin,$(PLUGINS), \
+		$(MAKE) -C $(PLUGINS_DIR)/$(plugin) vendor $(MAKE_FLAGS); \
+	)
 
 .PHONY: package
 package:
@@ -260,9 +154,9 @@ package:
 	@$(CP) $(CSCLI_FOLDER)/$(CSCLI_BIN) $(RELDIR)/cmd/crowdsec-cli
 
 	@$(foreach plugin,$(PLUGINS), \
-		$(MKDIR) $(RELDIR)/$(PLUGINS_DIR_PREFIX)$(plugin); \
-		$(CP) $(PLUGINS_DIR_PREFIX)$(plugin)/notification-$(plugin)$(EXT) $(RELDIR)/$(PLUGINS_DIR_PREFIX)$(plugin); \
-		$(CP) $(PLUGINS_DIR_PREFIX)$(plugin)/$(plugin).yaml $(RELDIR)/$(PLUGINS_DIR_PREFIX)$(plugin)/; \
+		$(MKDIR) $(RELDIR)/$(PLUGINS_DIR)/$(plugin); \
+		$(CP) $(PLUGINS_DIR)/$(plugin)/notification-$(plugin)$(EXT) $(RELDIR)/$(PLUGINS_DIR)/$(plugin); \
+		$(CP) $(PLUGINS_DIR)/$(plugin)/$(plugin).yaml $(RELDIR)/$(PLUGINS_DIR)/$(plugin)/; \
 	)
 
 	@$(CPR) ./config $(RELDIR)
@@ -281,14 +175,14 @@ else
 endif
 
 .PHONY: release
-release: check_release build package  ## Build a release tarball
+release: check_release build package
 
 .PHONY: windows_installer
-windows_installer: build  ## Windows - build the installer
+windows_installer: build
 	@.\make_installer.ps1 -version $(BUILD_VERSION)
 
 .PHONY: chocolatey
-chocolatey: windows_installer  ## Windows - build the chocolatey package
+chocolatey: windows_installer
 	@.\make_chocolatey.ps1 -version $(BUILD_VERSION)
 
 # Include test/bats.mk only if it exists
@@ -301,4 +195,3 @@ include test/bats.mk
 endif
 
 include mk/goversion.mk
-include mk/help.mk

azure-pipelines.yml

@@ -15,13 +15,19 @@ pool:
 stages:
   - stage: Build
     jobs:
-      - job: Build
+      - job:
         displayName: "Build"
         steps:
-          - task: GoTool@0
-            displayName: "Install Go"
+          - task: DotNetCoreCLI@2
+            displayName: "Install SignClient"
             inputs:
-                version: '1.22.2'
+              command: 'custom'
+              custom: 'tool'
+              arguments: 'install --global SignClient --version 1.3.155'
+          - task: GoTool@0
+            displayName: "Install Go 1.20"
+            inputs:
+                version: '1.20.4'
 
           - pwsh: |
               choco install -y make
@@ -32,15 +38,25 @@ stages:
               pwsh: true
               #we are not calling make windows_installer because we want to sign the binaries before they are added to the MSI
               script: |
-                make build BUILD_RE2_WASM=1
+                make build
+          - task: AzureKeyVault@2
+            inputs:
+              azureSubscription: 'Azure subscription 1(8a93ab40-7e99-445e-ad47-0f6a3e2ef546)'
+              KeyVaultName: 'CodeSigningSecrets'
+              SecretsFilter: 'CodeSigningUser,CodeSigningPassword'
+              RunAsPreJob: false
+
+          - task: DownloadSecureFile@1
+            inputs:
+              secureFile: appsettings.json
 
+          - pwsh: |
+              SignClient.exe Sign --name "crowdsec-binaries" `
+              --input "**/*.exe" --config (Join-Path -Path $(Agent.TempDirectory) -ChildPath "appsettings.json") `
+              --user $(CodeSigningUser) --secret '$(CodeSigningPassword)'
+            displayName: "Sign Crowdsec binaries + plugins"
           - pwsh: |
               $build_version=$env:BUILD_SOURCEBRANCHNAME
-              #Override the version if it's set in the pipeline
-              if ( ${env:USERBUILDVERSION} -ne "")
-              {
-                $build_version = ${env:USERBUILDVERSION}
-              }
               if ($build_version.StartsWith("v"))
               {
                 $build_version = $build_version.Substring(1)
@@ -53,112 +69,35 @@ stages:
             displayName: GetCrowdsecVersion
             name: GetCrowdsecVersion
           - pwsh: |
-              Get-ChildItem -Path .\cmd -Directory | ForEach-Object {
-                $dirName = $_.Name
-                Get-ChildItem -Path .\cmd\$dirName -File -Filter '*.exe' | ForEach-Object {
-                  $fileName = $_.Name
-                  $destDir = Join-Path $(Build.ArtifactStagingDirectory) cmd\$dirName
-                  New-Item -ItemType Directory -Path $destDir -Force
-                  Copy-Item -Path .\cmd\$dirName\$fileName -Destination $destDir
-                }
-              }
-            displayName: "Copy binaries to staging directory"
-          - task: PublishPipelineArtifact@1
-            inputs:
-              targetPath: '$(Build.ArtifactStagingDirectory)'
-              artifact: 'unsigned_binaries'
-            displayName: "Upload binaries artifact"
-
-  - stage: Sign
-    dependsOn: Build
-    variables: 
-      - group: 'FOSS Build Variables'
-      - name: BuildVersion
-        value: $[ stageDependencies.Build.Build.outputs['GetCrowdsecVersion.BuildVersion'] ]
-    condition: succeeded()
-    jobs:
-      - job: Sign
-        displayName: "Sign"
-        steps:
-          - download: current
-            artifact: unsigned_binaries
-            displayName: "Download binaries artifact"
-          - task: CopyFiles@2
-            inputs:
-              SourceFolder: '$(Pipeline.Workspace)/unsigned_binaries'
-              TargetFolder: '$(Build.SourcesDirectory)'
-            displayName: "Copy binaries to workspace"
-          - task: DotNetCoreCLI@2
-            displayName: "Install SignTool tool"
-            inputs:
-              command: 'custom'
-              custom: 'tool'
-              arguments: install --global sign --version 0.9.0-beta.23127.3
-          - task: AzureKeyVault@2
-            displayName: "Get signing parameters"
-            inputs:
-              azureSubscription: "Azure subscription"
-              KeyVaultName: "$(KeyVaultName)"
-              SecretsFilter: "TenantId,ClientId,ClientSecret,Certificate,KeyVaultUrl"
-          - pwsh: |
-              sign code azure-key-vault `
-              "**/*.exe" `
-              --base-directory "$(Build.SourcesDirectory)/cmd/" `
-              --publisher-name "CrowdSec" `
-              --description "CrowdSec" `
-              --description-url "https://github.com/crowdsecurity/crowdsec" `
-              --azure-key-vault-tenant-id "$(TenantId)" `
-              --azure-key-vault-client-id "$(ClientId)" `
-              --azure-key-vault-client-secret "$(ClientSecret)" `
-              --azure-key-vault-certificate "$(Certificate)" `
-              --azure-key-vault-url "$(KeyVaultUrl)"
-            displayName: "Sign crowdsec binaries"
-          - pwsh: |
-              .\make_installer.ps1 -version '$(BuildVersion)'
+              .\make_installer.ps1 -version '$(GetCrowdsecVersion.BuildVersion)'
             displayName: "Build Crowdsec MSI"
             name: BuildMSI
-          - pwsh: |
-              .\make_chocolatey.ps1 -version '$(BuildVersion)'
-            displayName: "Build Chocolatey nupkg"
-          - pwsh: |
-              sign code azure-key-vault `
-              "*.msi" `
-              --base-directory "$(Build.SourcesDirectory)" `
-              --publisher-name "CrowdSec" `
-              --description "CrowdSec" `
-              --description-url "https://github.com/crowdsecurity/crowdsec" `
-              --azure-key-vault-tenant-id "$(TenantId)" `
-              --azure-key-vault-client-id "$(ClientId)" `
-              --azure-key-vault-client-secret "$(ClientSecret)" `
-              --azure-key-vault-certificate "$(Certificate)" `
-              --azure-key-vault-url "$(KeyVaultUrl)"
-            displayName: "Sign MSI package"
-          - pwsh: |
-              sign code azure-key-vault `
-              "*.nupkg" `
-              --base-directory "$(Build.SourcesDirectory)" `
-              --publisher-name "CrowdSec" `
-              --description "CrowdSec" `
-              --description-url "https://github.com/crowdsecurity/crowdsec" `
-              --azure-key-vault-tenant-id "$(TenantId)" `
-              --azure-key-vault-client-id "$(ClientId)" `
-              --azure-key-vault-client-secret "$(ClientSecret)" `
-              --azure-key-vault-certificate "$(Certificate)" `
-              --azure-key-vault-url "$(KeyVaultUrl)"
-            displayName: "Sign nuget package"
-          - task: PublishPipelineArtifact@1
-            inputs:
-              targetPath: '$(Build.SourcesDirectory)/crowdsec_$(BuildVersion).msi'
-              artifact: 'signed_msi_package'
-            displayName: "Upload signed MSI artifact"
-          - task: PublishPipelineArtifact@1
-            inputs:
-              targetPath: '$(Build.SourcesDirectory)/crowdsec.$(BuildVersion).nupkg'
-              artifact: 'signed_nuget_package'
-            displayName: "Upload signed nuget artifact"
 
+          - pwsh: |
+              .\make_chocolatey.ps1 -version '$(GetCrowdsecVersion.BuildVersion)'
+            displayName: "Build Chocolatey nupkg"
+
+          - pwsh: |
+              SignClient.exe Sign --name "crowdsec-msi" `
+              --input "*.msi" --config (Join-Path -Path $(Agent.TempDirectory) -ChildPath "appsettings.json") `
+              --user $(CodeSigningUser) --secret '$(CodeSigningPassword)'
+            displayName: "Sign Crowdsec MSI"
+
+          - task: PublishBuildArtifacts@1
+            inputs:
+              PathtoPublish: '$(Build.Repository.LocalPath)\\crowdsec_$(GetCrowdsecVersion.BuildVersion).msi'
+              ArtifactName: 'crowdsec.msi'
+              publishLocation: 'Container'
+            displayName: "Upload MSI artifact"
+          
+          - task: PublishBuildArtifacts@1
+            inputs:
+              PathtoPublish: '$(Build.Repository.LocalPath)\\windows\\Chocolatey\\crowdsec\\crowdsec.$(GetCrowdsecVersion.BuildVersion).nupkg'
+              ArtifactName: 'crowdsec.nupkg'
+              publishLocation: 'Container'
+            displayName: "Upload nupkg artifact"
   - stage: Publish
-    dependsOn: Sign
+    dependsOn: Build
     jobs:
       - deployment: "Publish"
         displayName: "Publish to GitHub"
@@ -180,7 +119,8 @@ stages:
                     assetUploadMode: 'replace'
                     addChangeLog: false
                     isPreRelease: true #we force prerelease because the pipeline is invoked on tag creation, which happens when we do a prerelease
+                    #the .. is an ugly hack, but I can't find the var that gives D:\a\1 ...
                     assets: |
-                      $(Pipeline.Workspace)/signed_msi_package/*.msi
-                      $(Pipeline.Workspace)/signed_nuget_package/*.nupkg
+                      $(Build.ArtifactStagingDirectory)\..\crowdsec.msi/*.msi
+                      $(Build.ArtifactStagingDirectory)\..\crowdsec.nupkg/*.nupkg
                   condition: ne(variables['GetLatestPrelease.LatestPreRelease'], '')

cmd/crowdsec-cli/Makefile

@@ -4,8 +4,10 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-GO = go
-GOBUILD = $(GO) build
+# Go parameters
+GOCMD = go
+GOBUILD = $(GOCMD) build
+GOTEST = $(GOCMD) test
 
 BINARY_NAME = cscli$(EXT)
 PREFIX ?= "/"
@@ -15,7 +17,7 @@ BIN_PREFIX = $(PREFIX)"/usr/local/bin/"
 all: clean build
 
 build: clean
-	$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
+	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
 
 .PHONY: install
 install: install-conf install-bin

cmd/crowdsec-cli/alerts.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/csv"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"net/url"
 	"os"
@@ -12,16 +11,17 @@ import (
 	"strconv"
 	"strings"
 	"text/template"
+	"time"
 
 	"github.com/fatih/color"
 	"github.com/go-openapi/strfmt"
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v3"
+	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/version"
+	"github.com/crowdsecurity/go-cs-lib/pkg/version"
 
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/models"
@@ -30,46 +30,83 @@ import (
 
 func DecisionsFromAlert(alert *models.Alert) string {
 	ret := ""
-	decMap := make(map[string]int)
-
+	var decMap = make(map[string]int)
 	for _, decision := range alert.Decisions {
 		k := *decision.Type
 		if *decision.Simulated {
 			k = fmt.Sprintf("(simul)%s", k)
 		}
-
 		v := decMap[k]
 		decMap[k] = v + 1
 	}
-
 	for k, v := range decMap {
 		if len(ret) > 0 {
 			ret += " "
 		}
-
 		ret += fmt.Sprintf("%s:%d", k, v)
 	}
-
 	return ret
 }
 
-func (cli *cliAlerts) alertsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
-	switch cli.cfg().Cscli.Output {
-	case "raw":
+func DateFromAlert(alert *models.Alert) string {
+	ts, err := time.Parse(time.RFC3339, alert.CreatedAt)
+	if err != nil {
+		log.Infof("while parsing %s with %s : %s", alert.CreatedAt, time.RFC3339, err)
+		return alert.CreatedAt
+	}
+	return ts.Format(time.RFC822)
+}
+
+func SourceFromAlert(alert *models.Alert) string {
+
+	//more than one item, just number and scope
+	if len(alert.Decisions) > 1 {
+		return fmt.Sprintf("%d %ss (%s)", len(alert.Decisions), *alert.Decisions[0].Scope, *alert.Decisions[0].Origin)
+	}
+
+	//fallback on single decision information
+	if len(alert.Decisions) == 1 {
+		return fmt.Sprintf("%s:%s", *alert.Decisions[0].Scope, *alert.Decisions[0].Value)
+	}
+
+	//try to compose a human friendly version
+	if *alert.Source.Value != "" && *alert.Source.Scope != "" {
+		scope := ""
+		scope = fmt.Sprintf("%s:%s", *alert.Source.Scope, *alert.Source.Value)
+		extra := ""
+		if alert.Source.Cn != "" {
+			extra = alert.Source.Cn
+		}
+		if alert.Source.AsNumber != "" {
+			extra += fmt.Sprintf("/%s", alert.Source.AsNumber)
+		}
+		if alert.Source.AsName != "" {
+			extra += fmt.Sprintf("/%s", alert.Source.AsName)
+		}
+
+		if extra != "" {
+			scope += " (" + extra + ")"
+		}
+		return scope
+	}
+	return ""
+}
+
+func AlertsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
+
+	if csConfig.Cscli.Output == "raw" {
 		csvwriter := csv.NewWriter(os.Stdout)
 		header := []string{"id", "scope", "value", "reason", "country", "as", "decisions", "created_at"}
-
 		if printMachine {
 			header = append(header, "machine")
 		}
-
-		if err := csvwriter.Write(header); err != nil {
+		err := csvwriter.Write(header)
+		if err != nil {
 			return err
 		}
-
 		for _, alertItem := range *alerts {
 			row := []string{
-				strconv.FormatInt(alertItem.ID, 10),
+				fmt.Sprintf("%d", alertItem.ID),
 				*alertItem.Source.Scope,
 				*alertItem.Source.Value,
 				*alertItem.Scenario,
@@ -81,32 +118,22 @@ func (cli *cliAlerts) alertsToTable(alerts *models.GetAlertsResponse, printMachi
 			if printMachine {
 				row = append(row, alertItem.MachineID)
 			}
-
-			if err := csvwriter.Write(row); err != nil {
+			err := csvwriter.Write(row)
+			if err != nil {
 				return err
 			}
 		}
-
 		csvwriter.Flush()
-	case "json":
-		if *alerts == nil {
-			// avoid returning "null" in json
-			// could be cleaner if we used slice of alerts directly
-			fmt.Println("[]")
-			return nil
-		}
-
+	} else if csConfig.Cscli.Output == "json" {
 		x, _ := json.MarshalIndent(alerts, "", " ")
-		fmt.Print(string(x))
-	case "human":
+		fmt.Printf("%s", string(x))
+	} else if csConfig.Cscli.Output == "human" {
 		if len(*alerts) == 0 {
 			fmt.Println("No active alerts")
 			return nil
 		}
-
 		alertsTable(color.Output, alerts, printMachine)
 	}
-
 	return nil
 }
 
@@ -128,109 +155,93 @@ var alertTemplate = `
 
 `
 
-func (cli *cliAlerts) displayOneAlert(alert *models.Alert, withDetail bool) error {
-	tmpl, err := template.New("alert").Parse(alertTemplate)
-	if err != nil {
-		return err
-	}
 
-	if err = tmpl.Execute(os.Stdout, alert); err != nil {
-		return err
-	}
-
-	alertDecisionsTable(color.Output, alert)
-
-	if len(alert.Meta) > 0 {
-		fmt.Printf("\n - Context  :\n")
-		sort.Slice(alert.Meta, func(i, j int) bool {
-			return alert.Meta[i].Key < alert.Meta[j].Key
-		})
-
-		table := newTable(color.Output)
-		table.SetRowLines(false)
-		table.SetHeaders("Key", "Value")
-
-		for _, meta := range alert.Meta {
-			var valSlice []string
-			if err := json.Unmarshal([]byte(meta.Value), &valSlice); err != nil {
-				return fmt.Errorf("unknown context value type '%s': %w", meta.Value, err)
-			}
-
-			for _, value := range valSlice {
-				table.AddRow(
-					meta.Key,
-					value,
-				)
-			}
+func DisplayOneAlert(alert *models.Alert, withDetail bool) error {
+	if csConfig.Cscli.Output == "human" {
+		tmpl, err := template.New("alert").Parse(alertTemplate)
+		if err != nil {
+			return err
+		}
+		err = tmpl.Execute(os.Stdout, alert)
+		if err != nil {
+			return err
 		}
 
-		table.Render()
-	}
+		alertDecisionsTable(color.Output, alert)
 
-	if withDetail {
-		fmt.Printf("\n - Events  :\n")
+		if len(alert.Meta) > 0 {
+			fmt.Printf("\n - Context  :\n")
+			sort.Slice(alert.Meta, func(i, j int) bool {
+				return alert.Meta[i].Key < alert.Meta[j].Key
+			})
+			table := newTable(color.Output)
+			table.SetRowLines(false)
+			table.SetHeaders("Key", "Value")
+			for _, meta := range alert.Meta {
+				var valSlice []string
+				if err := json.Unmarshal([]byte(meta.Value), &valSlice); err != nil {
+					return fmt.Errorf("unknown context value type '%s' : %s", meta.Value, err)
+				}
+				for _, value := range valSlice {
+					table.AddRow(
+						meta.Key,
+						value,
+					)
+				}
+			}
+			table.Render()
+		}
 
-		for _, event := range alert.Events {
-			alertEventTable(color.Output, event)
+		if withDetail {
+			fmt.Printf("\n - Events  :\n")
+			for _, event := range alert.Events {
+				alertEventTable(color.Output, event)
+			}
 		}
 	}
-
 	return nil
 }
 
-type cliAlerts struct {
-	client *apiclient.ApiClient
-	cfg    configGetter
-}
-
-func NewCLIAlerts(getconfig configGetter) *cliAlerts {
-	return &cliAlerts{
-		cfg: getconfig,
-	}
-}
-
-func (cli *cliAlerts) NewCommand() *cobra.Command {
-	cmd := &cobra.Command{
+func NewAlertsCmd() *cobra.Command {
+	var cmdAlerts = &cobra.Command{
 		Use:               "alerts [action]",
 		Short:             "Manage alerts",
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
-		Aliases:           []string{"alert"},
-		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
-			cfg := cli.cfg()
-			if err := cfg.LoadAPIClient(); err != nil {
-				return fmt.Errorf("loading api client: %w", err)
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			var err error
+			if err := csConfig.LoadAPIClient(); err != nil {
+				return errors.Wrap(err, "loading api client")
 			}
-			apiURL, err := url.Parse(cfg.API.Client.Credentials.URL)
+			apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
 			if err != nil {
-				return fmt.Errorf("parsing api url %s: %w", apiURL, err)
+				return errors.Wrapf(err, "parsing api url %s", apiURL)
 			}
-
-			cli.client, err = apiclient.NewClient(&apiclient.Config{
-				MachineID:     cfg.API.Client.Credentials.Login,
-				Password:      strfmt.Password(cfg.API.Client.Credentials.Password),
+			Client, err = apiclient.NewClient(&apiclient.Config{
+				MachineID:     csConfig.API.Client.Credentials.Login,
+				Password:      strfmt.Password(csConfig.API.Client.Credentials.Password),
 				UserAgent:     fmt.Sprintf("crowdsec/%s", version.String()),
 				URL:           apiURL,
 				VersionPrefix: "v1",
 			})
-			if err != nil {
-				return fmt.Errorf("new api client: %w", err)
-			}
 
+			if err != nil {
+				return errors.Wrap(err, "new api client")
+			}
 			return nil
 		},
 	}
 
-	cmd.AddCommand(cli.NewListCmd())
-	cmd.AddCommand(cli.NewInspectCmd())
-	cmd.AddCommand(cli.NewFlushCmd())
-	cmd.AddCommand(cli.NewDeleteCmd())
+	cmdAlerts.AddCommand(NewAlertsListCmd())
+	cmdAlerts.AddCommand(NewAlertsInspectCmd())
+	cmdAlerts.AddCommand(NewAlertsFlushCmd())
+	cmdAlerts.AddCommand(NewAlertsDeleteCmd())
 
-	return cmd
+	return cmdAlerts
 }
 
-func (cli *cliAlerts) NewListCmd() *cobra.Command {
-	alertListFilter := apiclient.AlertsListOpts{
+func NewAlertsListCmd() *cobra.Command {
+	var alertListFilter = apiclient.AlertsListOpts{
 		ScopeEquals:    new(string),
 		ValueEquals:    new(string),
 		ScenarioEquals: new(string),
@@ -242,24 +253,21 @@ func (cli *cliAlerts) NewListCmd() *cobra.Command {
 		IncludeCAPI:    new(bool),
 		OriginEquals:   new(string),
 	}
-
-	limit := new(int)
+	var limit = new(int)
 	contained := new(bool)
-
 	var printMachine bool
-
-	cmd := &cobra.Command{
+	var cmdAlertsList = &cobra.Command{
 		Use:   "list [filters]",
 		Short: "List alerts",
 		Example: `cscli alerts list
 cscli alerts list --ip 1.2.3.4
 cscli alerts list --range 1.2.3.0/24
-cscli alerts list --origin lists
 cscli alerts list -s crowdsecurity/ssh-bf
 cscli alerts list --type ban`,
-		Long:              `List alerts with optional filters`,
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, _ []string) error {
+		RunE: func(cmd *cobra.Command, args []string) error {
+			var err error
+
 			if err := manageCliDecisionAlerts(alertListFilter.IPEquals, alertListFilter.RangeEquals,
 				alertListFilter.ScopeEquals, alertListFilter.ValueEquals); err != nil {
 				printHelp(cmd)
@@ -325,56 +333,50 @@ cscli alerts list --type ban`,
 				alertListFilter.Contains = new(bool)
 			}
 
-			alerts, _, err := cli.client.Alerts.List(context.Background(), alertListFilter)
+			alerts, _, err := Client.Alerts.List(context.Background(), alertListFilter)
 			if err != nil {
-				return fmt.Errorf("unable to list alerts: %w", err)
+				return fmt.Errorf("unable to list alerts: %v", err)
 			}
 
-			if err = cli.alertsToTable(alerts, printMachine); err != nil {
-				return fmt.Errorf("unable to list alerts: %w", err)
+			err = AlertsToTable(alerts, printMachine)
+			if err != nil {
+				return fmt.Errorf("unable to list alerts: %v", err)
 			}
 
 			return nil
 		},
 	}
+	cmdAlertsList.Flags().SortFlags = false
+	cmdAlertsList.Flags().BoolVarP(alertListFilter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
+	cmdAlertsList.Flags().StringVar(alertListFilter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
+	cmdAlertsList.Flags().StringVar(alertListFilter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
+	cmdAlertsList.Flags().StringVarP(alertListFilter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
+	cmdAlertsList.Flags().StringVarP(alertListFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
+	cmdAlertsList.Flags().StringVarP(alertListFilter.RangeEquals, "range", "r", "", "restrict to alerts from this range (shorthand for --scope range --value <RANGE/X>)")
+	cmdAlertsList.Flags().StringVar(alertListFilter.TypeEquals, "type", "", "restrict to alerts with given decision type (ie. ban, captcha)")
+	cmdAlertsList.Flags().StringVar(alertListFilter.ScopeEquals, "scope", "", "restrict to alerts of this scope (ie. ip,range)")
+	cmdAlertsList.Flags().StringVarP(alertListFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
+	cmdAlertsList.Flags().StringVar(alertListFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
+	cmdAlertsList.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
+	cmdAlertsList.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that sent alerts")
+	cmdAlertsList.Flags().IntVarP(limit, "limit", "l", 50, "limit size of alerts list table (0 to view all alerts)")
 
-	flags := cmd.Flags()
-	flags.SortFlags = false
-	flags.BoolVarP(alertListFilter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
-	flags.StringVar(alertListFilter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
-	flags.StringVar(alertListFilter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
-	flags.StringVarP(alertListFilter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
-	flags.StringVarP(alertListFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
-	flags.StringVarP(alertListFilter.RangeEquals, "range", "r", "", "restrict to alerts from this range (shorthand for --scope range --value <RANGE/X>)")
-	flags.StringVar(alertListFilter.TypeEquals, "type", "", "restrict to alerts with given decision type (ie. ban, captcha)")
-	flags.StringVar(alertListFilter.ScopeEquals, "scope", "", "restrict to alerts of this scope (ie. ip,range)")
-	flags.StringVarP(alertListFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
-	flags.StringVar(alertListFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
-	flags.BoolVar(contained, "contained", false, "query decisions contained by range")
-	flags.BoolVarP(&printMachine, "machine", "m", false, "print machines that sent alerts")
-	flags.IntVarP(limit, "limit", "l", 50, "limit size of alerts list table (0 to view all alerts)")
-
-	return cmd
+	return cmdAlertsList
 }
 
-func (cli *cliAlerts) NewDeleteCmd() *cobra.Command {
-	var (
-		ActiveDecision *bool
-		AlertDeleteAll bool
-		delAlertByID   string
-	)
-
-	alertDeleteFilter := apiclient.AlertsDeleteOpts{
+func NewAlertsDeleteCmd() *cobra.Command {
+	var ActiveDecision *bool
+	var AlertDeleteAll bool
+	var delAlertByID string
+	contained := new(bool)
+	var alertDeleteFilter = apiclient.AlertsDeleteOpts{
 		ScopeEquals:    new(string),
 		ValueEquals:    new(string),
 		ScenarioEquals: new(string),
 		IPEquals:       new(string),
 		RangeEquals:    new(string),
 	}
-
-	contained := new(bool)
-
-	cmd := &cobra.Command{
+	var cmdAlertsDelete = &cobra.Command{
 		Use: "delete [filters] [--all]",
 		Short: `Delete alerts
 /!\ This command can be use only on the same machine than the local API.`,
@@ -384,7 +386,7 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`,
 		DisableAutoGenTag: true,
 		Aliases:           []string{"remove"},
 		Args:              cobra.ExactArgs(0),
-		PreRunE: func(cmd *cobra.Command, _ []string) error {
+		PreRunE: func(cmd *cobra.Command, args []string) error {
 			if AlertDeleteAll {
 				return nil
 			}
@@ -392,16 +394,16 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`,
 				*alertDeleteFilter.ScenarioEquals == "" && *alertDeleteFilter.IPEquals == "" &&
 				*alertDeleteFilter.RangeEquals == "" && delAlertByID == "" {
 				_ = cmd.Usage()
-				return errors.New("at least one filter or --all must be specified")
+				return fmt.Errorf("at least one filter or --all must be specified")
 			}
 
 			return nil
 		},
-		RunE: func(cmd *cobra.Command, _ []string) error {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			var err error
 
 			if !AlertDeleteAll {
-				if err = manageCliDecisionAlerts(alertDeleteFilter.IPEquals, alertDeleteFilter.RangeEquals,
+				if err := manageCliDecisionAlerts(alertDeleteFilter.IPEquals, alertDeleteFilter.RangeEquals,
 					alertDeleteFilter.ScopeEquals, alertDeleteFilter.ValueEquals); err != nil {
 					printHelp(cmd)
 					return err
@@ -437,14 +439,14 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`,
 
 			var alerts *models.DeleteAlertsResponse
 			if delAlertByID == "" {
-				alerts, _, err = cli.client.Alerts.Delete(context.Background(), alertDeleteFilter)
+				alerts, _, err = Client.Alerts.Delete(context.Background(), alertDeleteFilter)
 				if err != nil {
-					return fmt.Errorf("unable to delete alerts: %w", err)
+					return fmt.Errorf("unable to delete alerts : %v", err)
 				}
 			} else {
-				alerts, _, err = cli.client.Alerts.DeleteOne(context.Background(), delAlertByID)
+				alerts, _, err = Client.Alerts.DeleteOne(context.Background(), delAlertByID)
 				if err != nil {
-					return fmt.Errorf("unable to delete alert: %w", err)
+					return fmt.Errorf("unable to delete alert: %v", err)
 				}
 			}
 			log.Infof("%s alert(s) deleted", alerts.NbDeleted)
@@ -452,99 +454,90 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`,
 			return nil
 		},
 	}
-
-	flags := cmd.Flags()
-	flags.SortFlags = false
-	flags.StringVar(alertDeleteFilter.ScopeEquals, "scope", "", "the scope (ie. ip,range)")
-	flags.StringVarP(alertDeleteFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
-	flags.StringVarP(alertDeleteFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
-	flags.StringVarP(alertDeleteFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
-	flags.StringVarP(alertDeleteFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
-	flags.StringVar(&delAlertByID, "id", "", "alert ID")
-	flags.BoolVarP(&AlertDeleteAll, "all", "a", false, "delete all alerts")
-	flags.BoolVar(contained, "contained", false, "query decisions contained by range")
-
-	return cmd
+	cmdAlertsDelete.Flags().SortFlags = false
+	cmdAlertsDelete.Flags().StringVar(alertDeleteFilter.ScopeEquals, "scope", "", "the scope (ie. ip,range)")
+	cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
+	cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
+	cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
+	cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
+	cmdAlertsDelete.Flags().StringVar(&delAlertByID, "id", "", "alert ID")
+	cmdAlertsDelete.Flags().BoolVarP(&AlertDeleteAll, "all", "a", false, "delete all alerts")
+	cmdAlertsDelete.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
+	return cmdAlertsDelete
 }
 
-func (cli *cliAlerts) NewInspectCmd() *cobra.Command {
+func NewAlertsInspectCmd() *cobra.Command {
 	var details bool
-
-	cmd := &cobra.Command{
+	var cmdAlertsInspect = &cobra.Command{
 		Use:               `inspect "alert_id"`,
 		Short:             `Show info about an alert`,
 		Example:           `cscli alerts inspect 123`,
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			cfg := cli.cfg()
 			if len(args) == 0 {
 				printHelp(cmd)
-				return errors.New("missing alert_id")
+				return fmt.Errorf("missing alert_id")
 			}
 			for _, alertID := range args {
 				id, err := strconv.Atoi(alertID)
 				if err != nil {
 					return fmt.Errorf("bad alert id %s", alertID)
 				}
-				alert, _, err := cli.client.Alerts.GetByID(context.Background(), id)
+				alert, _, err := Client.Alerts.GetByID(context.Background(), id)
 				if err != nil {
-					return fmt.Errorf("can't find alert with id %s: %w", alertID, err)
+					return fmt.Errorf("can't find alert with id %s: %s", alertID, err)
 				}
-				switch cfg.Cscli.Output {
+				switch csConfig.Cscli.Output {
 				case "human":
-					if err := cli.displayOneAlert(alert, details); err != nil {
+					if err := DisplayOneAlert(alert, details); err != nil {
 						continue
 					}
 				case "json":
 					data, err := json.MarshalIndent(alert, "", "  ")
 					if err != nil {
-						return fmt.Errorf("unable to marshal alert with id %s: %w", alertID, err)
+						return fmt.Errorf("unable to marshal alert with id %s: %s", alertID, err)
 					}
 					fmt.Printf("%s\n", string(data))
 				case "raw":
 					data, err := yaml.Marshal(alert)
 					if err != nil {
-						return fmt.Errorf("unable to marshal alert with id %s: %w", alertID, err)
+						return fmt.Errorf("unable to marshal alert with id %s: %s", alertID, err)
 					}
-					fmt.Println(string(data))
+					fmt.Printf("%s\n", string(data))
 				}
 			}
 
 			return nil
 		},
 	}
+	cmdAlertsInspect.Flags().SortFlags = false
+	cmdAlertsInspect.Flags().BoolVarP(&details, "details", "d", false, "show alerts with events")
 
-	cmd.Flags().SortFlags = false
-	cmd.Flags().BoolVarP(&details, "details", "d", false, "show alerts with events")
-
-	return cmd
+	return cmdAlertsInspect
 }
 
-func (cli *cliAlerts) NewFlushCmd() *cobra.Command {
-	var (
-		maxItems int
-		maxAge   string
-	)
-
-	cmd := &cobra.Command{
+func NewAlertsFlushCmd() *cobra.Command {
+	var maxItems int
+	var maxAge string
+	var cmdAlertsFlush = &cobra.Command{
 		Use: `flush`,
 		Short: `Flush alerts
 /!\ This command can be used only on the same machine than the local API`,
 		Example:           `cscli alerts flush --max-items 1000 --max-age 7d`,
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			cfg := cli.cfg()
-			if err := require.LAPI(cfg); err != nil {
-				return err
+		RunE: func(cmd *cobra.Command, args []string) error {
+			var err error
+			if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
+				return fmt.Errorf("local API is disabled, please run this command on the local API machine")
 			}
-			db, err := database.NewClient(cfg.DbConfig)
+			dbClient, err = database.NewClient(csConfig.DbConfig)
 			if err != nil {
-				return fmt.Errorf("unable to create new database client: %w", err)
+				return fmt.Errorf("unable to create new database client: %s", err)
 			}
 			log.Info("Flushing alerts. !! This may take a long time !!")
-			err = db.FlushAlerts(maxAge, maxItems)
+			err = dbClient.FlushAlerts(maxAge, maxItems)
 			if err != nil {
-				return fmt.Errorf("unable to flush alerts: %w", err)
+				return fmt.Errorf("unable to flush alerts: %s", err)
 			}
 			log.Info("Alerts flushed")
 
@@ -552,9 +545,9 @@ func (cli *cliAlerts) NewFlushCmd() *cobra.Command {
 		},
 	}
 
-	cmd.Flags().SortFlags = false
-	cmd.Flags().IntVar(&maxItems, "max-items", 5000, "Maximum number of alert items to keep in the database")
-	cmd.Flags().StringVar(&maxAge, "max-age", "7d", "Maximum age of alert items to keep in the database")
+	cmdAlertsFlush.Flags().SortFlags = false
+	cmdAlertsFlush.Flags().IntVar(&maxItems, "max-items", 5000, "Maximum number of alert items to keep in the database")
+	cmdAlertsFlush.Flags().StringVar(&maxAge, "max-age", "7d", "Maximum age of alert items to keep in the database")
 
-	return cmd
+	return cmdAlertsFlush
 }

cmd/crowdsec-cli/bouncers.go

@@ -3,318 +3,217 @@ package main
 import (
 	"encoding/csv"
 	"encoding/json"
-	"errors"
 	"fmt"
-	"os"
-	"slices"
+	"io"
 	"strings"
 	"time"
 
-	"github.com/AlecAivazis/survey/v2"
 	"github.com/fatih/color"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"golang.org/x/exp/slices"
 
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
-func askYesNo(message string, defaultAnswer bool) (bool, error) {
-	var answer bool
-
-	prompt := &survey.Confirm{
-		Message: message,
-		Default: defaultAnswer,
+func getBouncers(out io.Writer, dbClient *database.Client) error {
+	bouncers, err := dbClient.ListBouncers()
+	if err != nil {
+		return fmt.Errorf("unable to list bouncers: %s", err)
 	}
-
-	if err := survey.AskOne(prompt, &answer); err != nil {
-		return defaultAnswer, err
-	}
-
-	return answer, nil
-}
-
-type cliBouncers struct {
-	db  *database.Client
-	cfg configGetter
-}
-
-func NewCLIBouncers(cfg configGetter) *cliBouncers {
-	return &cliBouncers{
-		cfg: cfg,
-	}
-}
-
-func (cli *cliBouncers) NewCommand() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:   "bouncers [action]",
-		Short: "Manage bouncers [requires local API]",
-		Long: `To list/add/delete/prune bouncers.
-Note: This command requires database direct access, so is intended to be run on Local API/master.
-`,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"bouncer"},
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
-			var err error
-
-			cfg := cli.cfg()
-
-			if err = require.LAPI(cfg); err != nil {
-				return err
+	if csConfig.Cscli.Output == "human" {
+		getBouncersTable(out, bouncers)
+	} else if csConfig.Cscli.Output == "json" {
+		enc := json.NewEncoder(out)
+		enc.SetIndent("", "  ")
+		if err := enc.Encode(bouncers); err != nil {
+			return fmt.Errorf("failed to unmarshal: %w", err)
+		}
+		return nil
+	} else if csConfig.Cscli.Output == "raw" {
+		csvwriter := csv.NewWriter(out)
+		err := csvwriter.Write([]string{"name", "ip", "revoked", "last_pull", "type", "version", "auth_type"})
+		if err != nil {
+			return fmt.Errorf("failed to write raw header: %w", err)
+		}
+		for _, b := range bouncers {
+			var revoked string
+			if !b.Revoked {
+				revoked = "validated"
+			} else {
+				revoked = "pending"
 			}
-
-			cli.db, err = database.NewClient(cfg.DbConfig)
+			err := csvwriter.Write([]string{b.Name, b.IPAddress, revoked, b.LastPull.Format(time.RFC3339), b.Type, b.Version, b.AuthType})
 			if err != nil {
-				return fmt.Errorf("can't connect to the database: %w", err)
+				return fmt.Errorf("failed to write raw: %w", err)
 			}
+		}
+		csvwriter.Flush()
+	}
+	return nil
+}
 
+func NewBouncersListCmd() *cobra.Command {
+	cmdBouncersList := &cobra.Command{
+		Use:               "list",
+		Short:             "List bouncers",
+		Long:              `List bouncers`,
+		Example:           `cscli bouncers list`,
+		Args:              cobra.ExactArgs(0),
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, arg []string) error {
+			err := getBouncers(color.Output, dbClient)
+			if err != nil {
+				return fmt.Errorf("unable to list bouncers: %s", err)
+			}
 			return nil
 		},
 	}
 
-	cmd.AddCommand(cli.newListCmd())
-	cmd.AddCommand(cli.newAddCmd())
-	cmd.AddCommand(cli.newDeleteCmd())
-	cmd.AddCommand(cli.newPruneCmd())
-
-	return cmd
+	return cmdBouncersList
 }
 
-func (cli *cliBouncers) list() error {
-	out := color.Output
+func runBouncersAdd(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
 
-	bouncers, err := cli.db.ListBouncers()
+	keyLength, err := flags.GetInt("length")
 	if err != nil {
-		return fmt.Errorf("unable to list bouncers: %w", err)
+		return err
 	}
 
-	switch cli.cfg().Cscli.Output {
-	case "human":
-		getBouncersTable(out, bouncers)
-	case "json":
-		enc := json.NewEncoder(out)
-		enc.SetIndent("", "  ")
-
-		if err := enc.Encode(bouncers); err != nil {
-			return fmt.Errorf("failed to marshal: %w", err)
-		}
-
-		return nil
-	case "raw":
-		csvwriter := csv.NewWriter(out)
-
-		if err := csvwriter.Write([]string{"name", "ip", "revoked", "last_pull", "type", "version", "auth_type"}); err != nil {
-			return fmt.Errorf("failed to write raw header: %w", err)
-		}
-
-		for _, b := range bouncers {
-			valid := "validated"
-			if b.Revoked {
-				valid = "pending"
-			}
-
-			if err := csvwriter.Write([]string{b.Name, b.IPAddress, valid, b.LastPull.Format(time.RFC3339), b.Type, b.Version, b.AuthType}); err != nil {
-				return fmt.Errorf("failed to write raw: %w", err)
-			}
-		}
-
-		csvwriter.Flush()
+	key, err := flags.GetString("key")
+	if err != nil {
+		return err
 	}
 
-	return nil
-}
+	keyName := args[0]
+	var apiKey string
 
-func (cli *cliBouncers) newListCmd() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:               "list",
-		Short:             "list all bouncers within the database",
-		Example:           `cscli bouncers list`,
-		Args:              cobra.ExactArgs(0),
-		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			return cli.list()
-		},
+	if keyName == "" {
+		return fmt.Errorf("please provide a name for the api key")
 	}
-
-	return cmd
-}
-
-func (cli *cliBouncers) add(bouncerName string, key string) error {
-	var err error
-
-	keyLength := 32
-
+	apiKey = key
 	if key == "" {
-		key, err = middlewares.GenerateAPIKey(keyLength)
-		if err != nil {
-			return fmt.Errorf("unable to generate api key: %w", err)
-		}
+		apiKey, err = middlewares.GenerateAPIKey(keyLength)
 	}
-
-	_, err = cli.db.CreateBouncer(bouncerName, "", middlewares.HashSHA512(key), types.ApiKeyAuthType)
 	if err != nil {
-		return fmt.Errorf("unable to create bouncer: %w", err)
+		return fmt.Errorf("unable to generate api key: %s", err)
+	}
+	_, err = dbClient.CreateBouncer(keyName, "", middlewares.HashSHA512(apiKey), types.ApiKeyAuthType)
+	if err != nil {
+		return fmt.Errorf("unable to create bouncer: %s", err)
 	}
 
-	switch cli.cfg().Cscli.Output {
-	case "human":
-		fmt.Printf("API key for '%s':\n\n", bouncerName)
-		fmt.Printf("   %s\n\n", key)
+	if csConfig.Cscli.Output == "human" {
+		fmt.Printf("Api key for '%s':\n\n", keyName)
+		fmt.Printf("   %s\n\n", apiKey)
 		fmt.Print("Please keep this key since you will not be able to retrieve it!\n")
-	case "raw":
-		fmt.Print(key)
-	case "json":
-		j, err := json.Marshal(key)
+	} else if csConfig.Cscli.Output == "raw" {
+		fmt.Printf("%s", apiKey)
+	} else if csConfig.Cscli.Output == "json" {
+		j, err := json.Marshal(apiKey)
 		if err != nil {
-			return errors.New("unable to marshal api key")
+			return fmt.Errorf("unable to marshal api key")
 		}
-
-		fmt.Print(string(j))
+		fmt.Printf("%s", string(j))
 	}
 
 	return nil
 }
 
-func (cli *cliBouncers) newAddCmd() *cobra.Command {
-	var key string
-
-	cmd := &cobra.Command{
-		Use:   "add MyBouncerName",
-		Short: "add a single bouncer to the database",
+func NewBouncersAddCmd() *cobra.Command {
+	cmdBouncersAdd := &cobra.Command{
+		Use:   "add MyBouncerName [--length 16]",
+		Short: "add bouncer",
+		Long:  `add bouncer`,
 		Example: `cscli bouncers add MyBouncerName
-cscli bouncers add MyBouncerName --key <random-key>`,
+cscli bouncers add MyBouncerName -l 24
+cscli bouncers add MyBouncerName -k <random-key>`,
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, args []string) error {
-			return cli.add(args[0], key)
-		},
+		RunE:              runBouncersAdd,
 	}
 
-	flags := cmd.Flags()
-	flags.StringP("length", "l", "", "length of the api key")
-	_ = flags.MarkDeprecated("length", "use --key instead")
-	flags.StringVarP(&key, "key", "k", "", "api key for the bouncer")
+	flags := cmdBouncersAdd.Flags()
 
-	return cmd
+	flags.IntP("length", "l", 16, "length of the api key")
+	flags.StringP("key", "k", "", "api key for the bouncer")
+
+	return cmdBouncersAdd
 }
 
-func (cli *cliBouncers) deleteValid(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	bouncers, err := cli.db.ListBouncers()
-	if err != nil {
-		cobra.CompError("unable to list bouncers " + err.Error())
-	}
-
-	ret := []string{}
-
-	for _, bouncer := range bouncers {
-		if strings.Contains(bouncer.Name, toComplete) && !slices.Contains(args, bouncer.Name) {
-			ret = append(ret, bouncer.Name)
-		}
-	}
-
-	return ret, cobra.ShellCompDirectiveNoFileComp
-}
-
-func (cli *cliBouncers) delete(bouncers []string) error {
-	for _, bouncerID := range bouncers {
-		err := cli.db.DeleteBouncer(bouncerID)
+func runBouncersDelete(cmd *cobra.Command, args []string) error {
+	for _, bouncerID := range args {
+		err := dbClient.DeleteBouncer(bouncerID)
 		if err != nil {
-			return fmt.Errorf("unable to delete bouncer '%s': %w", bouncerID, err)
+			return fmt.Errorf("unable to delete bouncer '%s': %s", bouncerID, err)
 		}
-
 		log.Infof("bouncer '%s' deleted successfully", bouncerID)
 	}
 
 	return nil
 }
 
-func (cli *cliBouncers) newDeleteCmd() *cobra.Command {
-	cmd := &cobra.Command{
+func NewBouncersDeleteCmd() *cobra.Command {
+	cmdBouncersDelete := &cobra.Command{
 		Use:               "delete MyBouncerName",
-		Short:             "delete bouncer(s) from the database",
+		Short:             "delete bouncer",
 		Args:              cobra.MinimumNArgs(1),
 		Aliases:           []string{"remove"},
 		DisableAutoGenTag: true,
-		ValidArgsFunction: cli.deleteValid,
-		RunE: func(_ *cobra.Command, args []string) error {
-			return cli.delete(args)
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			var err error
+			dbClient, err = getDBClient()
+			if err != nil {
+				cobra.CompError("unable to create new database client: " + err.Error())
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+			bouncers, err := dbClient.ListBouncers()
+			if err != nil {
+				cobra.CompError("unable to list bouncers " + err.Error())
+			}
+			ret := make([]string, 0)
+			for _, bouncer := range bouncers {
+				if strings.Contains(bouncer.Name, toComplete) && !slices.Contains(args, bouncer.Name) {
+					ret = append(ret, bouncer.Name)
+				}
+			}
+			return ret, cobra.ShellCompDirectiveNoFileComp
 		},
+		RunE: runBouncersDelete,
 	}
 
-	return cmd
+	return cmdBouncersDelete
 }
 
-func (cli *cliBouncers) prune(duration time.Duration, force bool) error {
-	if duration < 2*time.Minute {
-		if yes, err := askYesNo(
-				"The duration you provided is less than 2 minutes. " +
-				"This may remove active bouncers. Continue?", false); err != nil {
-			return err
-		} else if !yes {
-			fmt.Println("User aborted prune. No changes were made.")
-			return nil
-		}
-	}
-
-	bouncers, err := cli.db.QueryBouncersLastPulltimeLT(time.Now().UTC().Add(-duration))
-	if err != nil {
-		return fmt.Errorf("unable to query bouncers: %w", err)
-	}
-
-	if len(bouncers) == 0 {
-		fmt.Println("No bouncers to prune.")
-		return nil
-	}
-
-	getBouncersTable(color.Output, bouncers)
-
-	if !force {
-		if yes, err := askYesNo(
-				"You are about to PERMANENTLY remove the above bouncers from the database. " +
-				"These will NOT be recoverable. Continue?", false); err != nil {
-			return err
-		} else if !yes {
-			fmt.Println("User aborted prune. No changes were made.")
-			return nil
-		}
-	}
-
-	deleted, err := cli.db.BulkDeleteBouncers(bouncers)
-	if err != nil {
-		return fmt.Errorf("unable to prune bouncers: %w", err)
-	}
-
-	fmt.Fprintf(os.Stderr, "Successfully deleted %d bouncers\n", deleted)
-
-	return nil
-}
-
-func (cli *cliBouncers) newPruneCmd() *cobra.Command {
-	var (
-		duration time.Duration
-		force    bool
-	)
-
-	const defaultDuration = 60 * time.Minute
-
-	cmd := &cobra.Command{
-		Use:               "prune",
-		Short:             "prune multiple bouncers from the database",
-		Args:              cobra.NoArgs,
+func NewBouncersCmd() *cobra.Command {
+	var cmdBouncers = &cobra.Command{
+		Use:   "bouncers [action]",
+		Short: "Manage bouncers [requires local API]",
+		Long: `To list/add/delete bouncers.
+Note: This command requires database direct access, so is intended to be run on Local API/master.
+`,
+		Args:              cobra.MinimumNArgs(1),
+		Aliases:           []string{"bouncer"},
 		DisableAutoGenTag: true,
-		Example: `cscli bouncers prune -d 45m
-cscli bouncers prune -d 45m --force`,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			return cli.prune(duration, force)
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			var err error
+			if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
+				return fmt.Errorf("local API is disabled, please run this command on the local API machine")
+			}
+			dbClient, err = database.NewClient(csConfig.DbConfig)
+			if err != nil {
+				return fmt.Errorf("unable to create new database client: %s", err)
+			}
+			return nil
 		},
 	}
 
-	flags := cmd.Flags()
-	flags.DurationVarP(&duration, "duration", "d", defaultDuration, "duration of time since last pull")
-	flags.BoolVar(&force, "force", false, "force prune without asking for confirmation")
+	cmdBouncers.AddCommand(NewBouncersListCmd())
+	cmdBouncers.AddCommand(NewBouncersAddCmd())
+	cmdBouncers.AddCommand(NewBouncersDeleteCmd())
 
-	return cmd
+	return cmdBouncers
 }

cmd/crowdsec-cli/bouncers_table.go

@@ -5,9 +5,9 @@ import (
 	"time"
 
 	"github.com/aquasecurity/table"
+	"github.com/enescakir/emoji"
 
 	"github.com/crowdsecurity/crowdsec/pkg/database/ent"
-	"github.com/crowdsecurity/crowdsec/pkg/emoji"
 )
 
 func getBouncersTable(out io.Writer, bouncers []*ent.Bouncer) {
@@ -17,9 +17,11 @@ func getBouncersTable(out io.Writer, bouncers []*ent.Bouncer) {
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	for _, b := range bouncers {
-		revoked := emoji.CheckMark
-		if b.Revoked {
-			revoked = emoji.Prohibited
+		var revoked string
+		if !b.Revoked {
+			revoked = emoji.CheckMark.String()
+		} else {
+			revoked = emoji.Prohibited.String()
 		}
 
 		t.AddRow(b.Name, b.IPAddress, revoked, b.LastPull.Format(time.RFC3339), b.Type, b.Version, b.AuthType)

cmd/crowdsec-cli/capi.go

@@ -2,221 +2,187 @@ package main
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"net/url"
 	"os"
 
-	"github.com/go-openapi/strfmt"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v3"
+	"github.com/crowdsecurity/go-cs-lib/pkg/version"
 
-	"github.com/crowdsecurity/go-cs-lib/version"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/crowdsecurity/crowdsec/pkg/fflag"
 	"github.com/crowdsecurity/crowdsec/pkg/models"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/go-openapi/strfmt"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v2"
 )
 
-const (
-	CAPIBaseURL   = "https://api.crowdsec.net/"
-	CAPIURLPrefix = "v3"
-)
+const CAPIBaseURL string = "https://api.crowdsec.net/"
+const CAPIURLPrefix = "v3"
 
-type cliCapi struct {
-	cfg configGetter
-}
-
-func NewCLICapi(cfg configGetter) *cliCapi {
-	return &cliCapi{
-		cfg: cfg,
-	}
-}
-
-func (cli *cliCapi) NewCommand() *cobra.Command {
-	cmd := &cobra.Command{
+func NewCapiCmd() *cobra.Command {
+	var cmdCapi = &cobra.Command{
 		Use:               "capi [action]",
 		Short:             "Manage interaction with Central API (CAPI)",
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
-			cfg := cli.cfg()
-			if err := require.LAPI(cfg); err != nil {
-				return err
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
+				return errors.Wrap(err, "Local API is disabled, please run this command on the local API machine")
 			}
-
-			if err := require.CAPI(cfg); err != nil {
-				return err
+			if csConfig.API.Server.OnlineClient == nil {
+				log.Fatalf("no configuration for Central API in '%s'", *csConfig.FilePath)
 			}
 
 			return nil
 		},
 	}
 
-	cmd.AddCommand(cli.newRegisterCmd())
-	cmd.AddCommand(cli.newStatusCmd())
+	cmdCapi.AddCommand(NewCapiRegisterCmd())
+	cmdCapi.AddCommand(NewCapiStatusCmd())
 
-	return cmd
+	return cmdCapi
 }
 
-func (cli *cliCapi) register(capiUserPrefix string, outputFile string) error {
-	cfg := cli.cfg()
+func NewCapiRegisterCmd() *cobra.Command {
+	var capiUserPrefix string
+	var outputFile string
 
-	capiUser, err := generateID(capiUserPrefix)
-	if err != nil {
-		return fmt.Errorf("unable to generate machine id: %w", err)
-	}
-
-	password := strfmt.Password(generatePassword(passwordLength))
-
-	apiurl, err := url.Parse(types.CAPIBaseURL)
-	if err != nil {
-		return fmt.Errorf("unable to parse api url %s: %w", types.CAPIBaseURL, err)
-	}
-
-	_, err = apiclient.RegisterClient(&apiclient.Config{
-		MachineID:     capiUser,
-		Password:      password,
-		UserAgent:     fmt.Sprintf("crowdsec/%s", version.String()),
-		URL:           apiurl,
-		VersionPrefix: CAPIURLPrefix,
-	}, nil)
-	if err != nil {
-		return fmt.Errorf("api client register ('%s'): %w", types.CAPIBaseURL, err)
-	}
-
-	log.Infof("Successfully registered to Central API (CAPI)")
-
-	var dumpFile string
-
-	switch {
-	case outputFile != "":
-		dumpFile = outputFile
-	case cfg.API.Server.OnlineClient.CredentialsFilePath != "":
-		dumpFile = cfg.API.Server.OnlineClient.CredentialsFilePath
-	default:
-		dumpFile = ""
-	}
-
-	apiCfg := csconfig.ApiCredentialsCfg{
-		Login:    capiUser,
-		Password: password.String(),
-		URL:      types.CAPIBaseURL,
-	}
-
-	apiConfigDump, err := yaml.Marshal(apiCfg)
-	if err != nil {
-		return fmt.Errorf("unable to marshal api credentials: %w", err)
-	}
-
-	if dumpFile != "" {
-		err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
-		if err != nil {
-			return fmt.Errorf("write api credentials in '%s' failed: %w", dumpFile, err)
-		}
-
-		log.Infof("Central API credentials written to '%s'", dumpFile)
-	} else {
-		fmt.Println(string(apiConfigDump))
-	}
-
-	log.Warning(ReloadMessage())
-
-	return nil
-}
-
-func (cli *cliCapi) newRegisterCmd() *cobra.Command {
-	var (
-		capiUserPrefix string
-		outputFile     string
-	)
-
-	cmd := &cobra.Command{
+	var cmdCapiRegister = &cobra.Command{
 		Use:               "register",
 		Short:             "Register to Central API (CAPI)",
 		Args:              cobra.MinimumNArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			return cli.register(capiUserPrefix, outputFile)
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+			capiUser, err := generateID(capiUserPrefix)
+			if err != nil {
+				log.Fatalf("unable to generate machine id: %s", err)
+			}
+			password := strfmt.Password(generatePassword(passwordLength))
+			apiurl, err := url.Parse(types.CAPIBaseURL)
+			if err != nil {
+				log.Fatalf("unable to parse api url %s : %s", types.CAPIBaseURL, err)
+			}
+			_, err = apiclient.RegisterClient(&apiclient.Config{
+				MachineID:     capiUser,
+				Password:      password,
+				UserAgent:     fmt.Sprintf("crowdsec/%s", version.String()),
+				URL:           apiurl,
+				VersionPrefix: CAPIURLPrefix,
+			}, nil)
+
+			if err != nil {
+				log.Fatalf("api client register ('%s'): %s", types.CAPIBaseURL, err)
+			}
+			log.Printf("Successfully registered to Central API (CAPI)")
+
+			var dumpFile string
+
+			if outputFile != "" {
+				dumpFile = outputFile
+			} else if csConfig.API.Server.OnlineClient.CredentialsFilePath != "" {
+				dumpFile = csConfig.API.Server.OnlineClient.CredentialsFilePath
+			} else {
+				dumpFile = ""
+			}
+			apiCfg := csconfig.ApiCredentialsCfg{
+				Login:    capiUser,
+				Password: password.String(),
+				URL:      types.CAPIBaseURL,
+			}
+			if fflag.PapiClient.IsEnabled() {
+				apiCfg.PapiURL = types.PAPIBaseURL
+			}
+			apiConfigDump, err := yaml.Marshal(apiCfg)
+			if err != nil {
+				log.Fatalf("unable to marshal api credentials: %s", err)
+			}
+			if dumpFile != "" {
+				err = os.WriteFile(dumpFile, apiConfigDump, 0600)
+				if err != nil {
+					log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
+				}
+				log.Printf("Central API credentials dumped to '%s'", dumpFile)
+			} else {
+				fmt.Printf("%s\n", string(apiConfigDump))
+			}
+
+			log.Warning(ReloadMessage())
 		},
 	}
-
-	cmd.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
-	cmd.Flags().StringVar(&capiUserPrefix, "schmilblick", "", "set a schmilblick (use in tests only)")
-
-	if err := cmd.Flags().MarkHidden("schmilblick"); err != nil {
+	cmdCapiRegister.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
+	cmdCapiRegister.Flags().StringVar(&capiUserPrefix, "schmilblick", "", "set a schmilblick (use in tests only)")
+	if err := cmdCapiRegister.Flags().MarkHidden("schmilblick"); err != nil {
 		log.Fatalf("failed to hide flag: %s", err)
 	}
 
-	return cmd
+	return cmdCapiRegister
 }
 
-func (cli *cliCapi) status() error {
-	cfg := cli.cfg()
-
-	if err := require.CAPIRegistered(cfg); err != nil {
-		return err
-	}
-
-	password := strfmt.Password(cfg.API.Server.OnlineClient.Credentials.Password)
-
-	apiurl, err := url.Parse(cfg.API.Server.OnlineClient.Credentials.URL)
-	if err != nil {
-		return fmt.Errorf("parsing api url ('%s'): %w", cfg.API.Server.OnlineClient.Credentials.URL, err)
-	}
-
-	hub, err := require.Hub(cfg, nil, nil)
-	if err != nil {
-		return err
-	}
-
-	scenarios, err := hub.GetInstalledNamesByType(cwhub.SCENARIOS)
-	if err != nil {
-		return fmt.Errorf("failed to get scenarios: %w", err)
-	}
-
-	if len(scenarios) == 0 {
-		return errors.New("no scenarios installed, abort")
-	}
-
-	Client, err = apiclient.NewDefaultClient(apiurl, CAPIURLPrefix, fmt.Sprintf("crowdsec/%s", version.String()), nil)
-	if err != nil {
-		return fmt.Errorf("init default client: %w", err)
-	}
-
-	t := models.WatcherAuthRequest{
-		MachineID: &cfg.API.Server.OnlineClient.Credentials.Login,
-		Password:  &password,
-		Scenarios: scenarios,
-	}
-
-	log.Infof("Loaded credentials from %s", cfg.API.Server.OnlineClient.CredentialsFilePath)
-	log.Infof("Trying to authenticate with username %s on %s", cfg.API.Server.OnlineClient.Credentials.Login, apiurl)
-
-	_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
-	if err != nil {
-		return fmt.Errorf("failed to authenticate to Central API (CAPI): %w", err)
-	}
-
-	log.Info("You can successfully interact with Central API (CAPI)")
-
-	return nil
-}
-
-func (cli *cliCapi) newStatusCmd() *cobra.Command {
-	cmd := &cobra.Command{
+func NewCapiStatusCmd() *cobra.Command {
+	var cmdCapiStatus = &cobra.Command{
 		Use:               "status",
 		Short:             "Check status with the Central API (CAPI)",
 		Args:              cobra.MinimumNArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			return cli.status()
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+			if csConfig.API.Server == nil {
+				log.Fatalln("There is no configuration on 'api.server:'")
+			}
+			if csConfig.API.Server.OnlineClient == nil {
+				log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
+			}
+
+			if csConfig.API.Server.OnlineClient.Credentials == nil {
+				log.Fatalf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
+			}
+
+			password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
+			apiurl, err := url.Parse(csConfig.API.Server.OnlineClient.Credentials.URL)
+			if err != nil {
+				log.Fatalf("parsing api url ('%s'): %s", csConfig.API.Server.OnlineClient.Credentials.URL, err)
+			}
+
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatal(err)
+			}
+
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+				log.Info("Run 'sudo cscli hub update' to get the hub index")
+				log.Fatalf("Failed to load hub index : %s", err)
+			}
+			scenarios, err := cwhub.GetInstalledScenariosAsString()
+			if err != nil {
+				log.Fatalf("failed to get scenarios : %s", err)
+			}
+			if len(scenarios) == 0 {
+				log.Fatalf("no scenarios installed, abort")
+			}
+
+			Client, err = apiclient.NewDefaultClient(apiurl, CAPIURLPrefix, fmt.Sprintf("crowdsec/%s", version.String()), nil)
+			if err != nil {
+				log.Fatalf("init default client: %s", err)
+			}
+			t := models.WatcherAuthRequest{
+				MachineID: &csConfig.API.Server.OnlineClient.Credentials.Login,
+				Password:  &password,
+				Scenarios: scenarios,
+			}
+			log.Infof("Loaded credentials from %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
+			log.Infof("Trying to authenticate with username %s on %s", csConfig.API.Server.OnlineClient.Credentials.Login, apiurl)
+			_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
+			if err != nil {
+				log.Fatalf("Failed to authenticate to Central API (CAPI) : %s", err)
+			}
+			log.Infof("You can successfully interact with Central API (CAPI)")
 		},
 	}
 
-	return cmd
+	return cmdCapiStatus
 }

cmd/crowdsec-cli/collections.go

@@ -0,0 +1,183 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/fatih/color"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func NewCollectionsCmd() *cobra.Command {
+	var cmdCollections = &cobra.Command{
+		Use:   "collections [action]",
+		Short: "Manage collections from hub",
+		Long:  `Install/Remove/Upgrade/Inspect collections from the CrowdSec Hub.`,
+		/*TBD fix help*/
+		Args:              cobra.MinimumNArgs(1),
+		Aliases:           []string{"collection"},
+		DisableAutoGenTag: true,
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatal(err)
+			}
+			if csConfig.Hub == nil {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+
+			if err := cwhub.SetHubBranch(); err != nil {
+				return fmt.Errorf("error while setting hub branch: %s", err)
+			}
+
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+				log.Info("Run 'sudo cscli hub update' to get the hub index")
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+
+			return nil
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			if cmd.Name() == "inspect" || cmd.Name() == "list" {
+				return
+			}
+			log.Infof(ReloadMessage())
+		},
+	}
+
+	var ignoreError bool
+	var cmdCollectionsInstall = &cobra.Command{
+		Use:     "install collection",
+		Short:   "Install given collection(s)",
+		Long:    `Fetch and install given collection(s) from hub`,
+		Example: `cscli collections install crowdsec/xxx crowdsec/xyz`,
+		Args:    cobra.MinimumNArgs(1),
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compAllItems(cwhub.COLLECTIONS, args, toComplete)
+		},
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			for _, name := range args {
+				t := cwhub.GetItem(cwhub.COLLECTIONS, name)
+				if t == nil {
+					nearestItem, score := GetDistance(cwhub.COLLECTIONS, name)
+					Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError)
+					continue
+				}
+				if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
+					if !ignoreError {
+						log.Fatalf("Error while installing '%s': %s", name, err)
+					}
+					log.Errorf("Error while installing '%s': %s", name, err)
+				}
+			}
+		},
+	}
+	cmdCollectionsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
+	cmdCollectionsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
+	cmdCollectionsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple collections")
+	cmdCollections.AddCommand(cmdCollectionsInstall)
+
+	var cmdCollectionsRemove = &cobra.Command{
+		Use:               "remove collection",
+		Short:             "Remove given collection(s)",
+		Long:              `Remove given collection(s) from hub`,
+		Example:           `cscli collections remove crowdsec/xxx crowdsec/xyz`,
+		Aliases:           []string{"delete"},
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			if all {
+				cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, forceAction)
+				return
+			}
+
+			if len(args) == 0 {
+				log.Fatal("Specify at least one collection to remove or '--all' flag.")
+			}
+
+			for _, name := range args {
+				if !forceAction {
+					item := cwhub.GetItem(cwhub.COLLECTIONS, name)
+					if item == nil {
+						log.Fatalf("unable to retrieve: %s\n", name)
+					}
+					if len(item.BelongsToCollections) > 0 {
+						log.Warningf("%s belongs to other collections :\n%s\n", name, item.BelongsToCollections)
+						log.Printf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection\n", name)
+						continue
+					}
+				}
+				cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, forceAction)
+			}
+		},
+	}
+	cmdCollectionsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
+	cmdCollectionsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
+	cmdCollectionsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the collections")
+	cmdCollections.AddCommand(cmdCollectionsRemove)
+
+	var cmdCollectionsUpgrade = &cobra.Command{
+		Use:               "upgrade collection",
+		Short:             "Upgrade given collection(s)",
+		Long:              `Fetch and upgrade given collection(s) from hub`,
+		Example:           `cscli collections upgrade crowdsec/xxx crowdsec/xyz`,
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			if all {
+				cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
+			} else {
+				if len(args) == 0 {
+					log.Fatalf("no target collection to upgrade")
+				}
+				for _, name := range args {
+					cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, forceAction)
+				}
+			}
+		},
+	}
+	cmdCollectionsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the collections")
+	cmdCollectionsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
+	cmdCollections.AddCommand(cmdCollectionsUpgrade)
+
+	var cmdCollectionsInspect = &cobra.Command{
+		Use:               "inspect collection",
+		Short:             "Inspect given collection",
+		Long:              `Inspect given collection`,
+		Example:           `cscli collections inspect crowdsec/xxx crowdsec/xyz`,
+		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			for _, name := range args {
+				InspectItem(name, cwhub.COLLECTIONS)
+			}
+		},
+	}
+	cmdCollectionsInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
+	cmdCollections.AddCommand(cmdCollectionsInspect)
+
+	var cmdCollectionsList = &cobra.Command{
+		Use:               "list collection [-a]",
+		Short:             "List all collections",
+		Long:              `List all collections`,
+		Example:           `cscli collections list`,
+		Args:              cobra.ExactArgs(0),
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all)
+		},
+	}
+	cmdCollectionsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
+	cmdCollections.AddCommand(cmdCollectionsList)
+
+	return cmdCollections
+}

cmd/crowdsec-cli/completion.go

@@ -7,7 +7,8 @@ import (
 )
 
 func NewCompletionCmd() *cobra.Command {
-	completionCmd := &cobra.Command{
+
+	var completionCmd = &cobra.Command{
 		Use:   "completion [bash|zsh|powershell|fish]",
 		Short: "Generate completion script",
 		Long: `To load completions:
@@ -81,6 +82,5 @@ func NewCompletionCmd() *cobra.Command {
 			}
 		},
 	}
-
 	return completionCmd
 }

cmd/crowdsec-cli/config.go

@@ -4,29 +4,19 @@ import (
 	"github.com/spf13/cobra"
 )
 
-type cliConfig struct {
-	cfg configGetter
-}
-
-func NewCLIConfig(cfg configGetter) *cliConfig {
-	return &cliConfig{
-		cfg: cfg,
-	}
-}
-
-func (cli *cliConfig) NewCommand() *cobra.Command {
-	cmd := &cobra.Command{
+func NewConfigCmd() *cobra.Command {
+	cmdConfig := &cobra.Command{
 		Use:               "config [command]",
 		Short:             "Allows to view current config",
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
 	}
 
-	cmd.AddCommand(cli.newShowCmd())
-	cmd.AddCommand(cli.newShowYAMLCmd())
-	cmd.AddCommand(cli.newBackupCmd())
-	cmd.AddCommand(cli.newRestoreCmd())
-	cmd.AddCommand(cli.newFeatureFlagsCmd())
+	cmdConfig.AddCommand(NewConfigShowCmd())
+	cmdConfig.AddCommand(NewConfigShowYAMLCmd())
+	cmdConfig.AddCommand(NewConfigBackupCmd())
+	cmdConfig.AddCommand(NewConfigRestoreCmd())
+	cmdConfig.AddCommand(NewConfigFeatureFlagsCmd())
 
-	return cmd
+	return cmdConfig
 }

cmd/crowdsec-cli/config_backup.go

@@ -1,99 +1,19 @@
 package main
 
 import (
-	"encoding/json"
-	"errors"
 	"fmt"
 	"os"
 	"path/filepath"
 
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-func (cli *cliConfig) backupHub(dirPath string) error {
-	hub, err := require.Hub(cli.cfg(), nil, nil)
-	if err != nil {
-		return err
-	}
-
-	for _, itemType := range cwhub.ItemTypes {
-		clog := log.WithFields(log.Fields{
-			"type": itemType,
-		})
-
-		itemMap := hub.GetItemMap(itemType)
-		if itemMap == nil {
-			clog.Infof("No %s to backup.", itemType)
-			continue
-		}
-
-		itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itemType)
-		if err = os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
-			return fmt.Errorf("error while creating %s: %w", itemDirectory, err)
-		}
-
-		upstreamParsers := []string{}
-
-		for k, v := range itemMap {
-			clog = clog.WithFields(log.Fields{
-				"file": v.Name,
-			})
-			if !v.State.Installed { // only backup installed ones
-				clog.Debugf("[%s]: not installed", k)
-				continue
-			}
-
-			// for the local/tainted ones, we back up the full file
-			if v.State.Tainted || v.State.IsLocal() || !v.State.UpToDate {
-				// we need to backup stages for parsers
-				if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS {
-					fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
-					if err = os.MkdirAll(fstagedir, os.ModePerm); err != nil {
-						return fmt.Errorf("error while creating stage dir %s: %w", fstagedir, err)
-					}
-				}
-
-				clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.State.IsLocal(), v.State.UpToDate)
-
-				tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
-				if err = CopyFile(v.State.LocalPath, tfile); err != nil {
-					return fmt.Errorf("failed copy %s %s to %s: %w", itemType, v.State.LocalPath, tfile, err)
-				}
-
-				clog.Infof("local/tainted saved %s to %s", v.State.LocalPath, tfile)
-
-				continue
-			}
-
-			clog.Debugf("[%s]: from hub, just backup name (up-to-date:%t)", k, v.State.UpToDate)
-			clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.State.UpToDate)
-			upstreamParsers = append(upstreamParsers, v.Name)
-		}
-		// write the upstream items
-		upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
-
-		upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
-		if err != nil {
-			return fmt.Errorf("failed marshaling upstream parsers: %w", err)
-		}
-
-		err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0o644)
-		if err != nil {
-			return fmt.Errorf("unable to write to %s %s: %w", itemType, upstreamParsersFname, err)
-		}
-
-		clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
-	}
-
-	return nil
-}
-
-/*
-	Backup crowdsec configurations to directory <dirPath>:
+/* Backup crowdsec configurations to directory <dirPath> :
 
 - Main config (config.yaml)
 - Profiles config (profiles.yaml)
@@ -101,33 +21,30 @@ func (cli *cliConfig) backupHub(dirPath string) error {
 - Backup of API credentials (local API and online API)
 - List of scenarios, parsers, postoverflows and collections that are up-to-date
 - Tainted/local/out-of-date scenarios, parsers, postoverflows and collections
-- Acquisition files (acquis.yaml, acquis.d/*.yaml)
 */
-func (cli *cliConfig) backup(dirPath string) error {
+func backupConfigToDirectory(dirPath string) error {
 	var err error
 
-	cfg := cli.cfg()
-
 	if dirPath == "" {
-		return errors.New("directory path can't be empty")
+		return fmt.Errorf("directory path can't be empty")
 	}
 
 	log.Infof("Starting configuration backup")
 
 	/*if parent directory doesn't exist, bail out. create final dir with Mkdir*/
 	parentDir := filepath.Dir(dirPath)
-	if _, err = os.Stat(parentDir); err != nil {
-		return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err)
+	if _, err := os.Stat(parentDir); err != nil {
+		return errors.Wrapf(err, "while checking parent directory %s existence", parentDir)
 	}
 
 	if err = os.Mkdir(dirPath, 0o700); err != nil {
-		return fmt.Errorf("while creating %s: %w", dirPath, err)
+		return errors.Wrapf(err, "while creating %s", dirPath)
 	}
 
-	if cfg.ConfigPaths.SimulationFilePath != "" {
+	if csConfig.ConfigPaths.SimulationFilePath != "" {
 		backupSimulation := filepath.Join(dirPath, "simulation.yaml")
-		if err = CopyFile(cfg.ConfigPaths.SimulationFilePath, backupSimulation); err != nil {
-			return fmt.Errorf("failed copy %s to %s: %w", cfg.ConfigPaths.SimulationFilePath, backupSimulation, err)
+		if err = types.CopyFile(csConfig.ConfigPaths.SimulationFilePath, backupSimulation); err != nil {
+			return errors.Wrapf(err, "failed copy %s to %s", csConfig.ConfigPaths.SimulationFilePath, backupSimulation)
 		}
 
 		log.Infof("Saved simulation to %s", backupSimulation)
@@ -137,32 +54,32 @@ func (cli *cliConfig) backup(dirPath string) error {
 	   - backup AcquisitionFilePath
 	   - backup the other files of acquisition directory
 	*/
-	if cfg.Crowdsec != nil && cfg.Crowdsec.AcquisitionFilePath != "" {
+	if csConfig.Crowdsec != nil && csConfig.Crowdsec.AcquisitionFilePath != "" {
 		backupAcquisition := filepath.Join(dirPath, "acquis.yaml")
-		if err = CopyFile(cfg.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil {
-			return fmt.Errorf("failed copy %s to %s: %w", cfg.Crowdsec.AcquisitionFilePath, backupAcquisition, err)
+		if err = types.CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil {
+			return fmt.Errorf("failed copy %s to %s : %s", csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition, err)
 		}
 	}
 
 	acquisBackupDir := filepath.Join(dirPath, "acquis")
 	if err = os.Mkdir(acquisBackupDir, 0o700); err != nil {
-		return fmt.Errorf("error while creating %s: %w", acquisBackupDir, err)
+		return fmt.Errorf("error while creating %s : %s", acquisBackupDir, err)
 	}
 
-	if cfg.Crowdsec != nil && len(cfg.Crowdsec.AcquisitionFiles) > 0 {
-		for _, acquisFile := range cfg.Crowdsec.AcquisitionFiles {
+	if csConfig.Crowdsec != nil && len(csConfig.Crowdsec.AcquisitionFiles) > 0 {
+		for _, acquisFile := range csConfig.Crowdsec.AcquisitionFiles {
 			/*if it was the default one, it was already backup'ed*/
-			if cfg.Crowdsec.AcquisitionFilePath == acquisFile {
+			if csConfig.Crowdsec.AcquisitionFilePath == acquisFile {
 				continue
 			}
 
 			targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile)))
 			if err != nil {
-				return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err)
+				return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir)
 			}
 
-			if err = CopyFile(acquisFile, targetFname); err != nil {
-				return fmt.Errorf("failed copy %s to %s: %w", acquisFile, targetFname, err)
+			if err = types.CopyFile(acquisFile, targetFname); err != nil {
+				return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err)
 			}
 
 			log.Infof("Saved acquis %s to %s", acquisFile, targetFname)
@@ -171,49 +88,68 @@ func (cli *cliConfig) backup(dirPath string) error {
 
 	if ConfigFilePath != "" {
 		backupMain := fmt.Sprintf("%s/config.yaml", dirPath)
-		if err = CopyFile(ConfigFilePath, backupMain); err != nil {
-			return fmt.Errorf("failed copy %s to %s: %w", ConfigFilePath, backupMain, err)
+		if err = types.CopyFile(ConfigFilePath, backupMain); err != nil {
+			return fmt.Errorf("failed copy %s to %s : %s", ConfigFilePath, backupMain, err)
 		}
 
 		log.Infof("Saved default yaml to %s", backupMain)
 	}
 
-	if cfg.API != nil && cfg.API.Server != nil && cfg.API.Server.OnlineClient != nil && cfg.API.Server.OnlineClient.CredentialsFilePath != "" {
+	if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" {
 		backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath)
-		if err = CopyFile(cfg.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil {
-			return fmt.Errorf("failed copy %s to %s: %w", cfg.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err)
+		if err = types.CopyFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil {
+			return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err)
 		}
 
 		log.Infof("Saved online API credentials to %s", backupCAPICreds)
 	}
 
-	if cfg.API != nil && cfg.API.Client != nil && cfg.API.Client.CredentialsFilePath != "" {
+	if csConfig.API != nil && csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" {
 		backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath)
-		if err = CopyFile(cfg.API.Client.CredentialsFilePath, backupLAPICreds); err != nil {
-			return fmt.Errorf("failed copy %s to %s: %w", cfg.API.Client.CredentialsFilePath, backupLAPICreds, err)
+		if err = types.CopyFile(csConfig.API.Client.CredentialsFilePath, backupLAPICreds); err != nil {
+			return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Client.CredentialsFilePath, backupLAPICreds, err)
 		}
 
 		log.Infof("Saved local API credentials to %s", backupLAPICreds)
 	}
 
-	if cfg.API != nil && cfg.API.Server != nil && cfg.API.Server.ProfilesPath != "" {
+	if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.ProfilesPath != "" {
 		backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath)
-		if err = CopyFile(cfg.API.Server.ProfilesPath, backupProfiles); err != nil {
-			return fmt.Errorf("failed copy %s to %s: %w", cfg.API.Server.ProfilesPath, backupProfiles, err)
+		if err = types.CopyFile(csConfig.API.Server.ProfilesPath, backupProfiles); err != nil {
+			return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.ProfilesPath, backupProfiles, err)
 		}
 
 		log.Infof("Saved profiles to %s", backupProfiles)
 	}
 
-	if err = cli.backupHub(dirPath); err != nil {
-		return fmt.Errorf("failed to backup hub config: %w", err)
+	if err = BackupHub(dirPath); err != nil {
+		return fmt.Errorf("failed to backup hub config : %s", err)
 	}
 
 	return nil
 }
 
-func (cli *cliConfig) newBackupCmd() *cobra.Command {
-	cmd := &cobra.Command{
+
+func runConfigBackup(cmd *cobra.Command, args []string) error {
+	if err := csConfig.LoadHub(); err != nil {
+		return err
+	}
+
+	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+		log.Info("Run 'sudo cscli hub update' to get the hub index")
+		return fmt.Errorf("failed to get Hub index: %w", err)
+	}
+
+	if err := backupConfigToDirectory(args[0]); err != nil {
+		return fmt.Errorf("failed to backup config: %w", err)
+	}
+
+	return nil
+}
+
+
+func NewConfigBackupCmd() *cobra.Command {
+	cmdConfigBackup := &cobra.Command{
 		Use:   `backup "directory"`,
 		Short: "Backup current config",
 		Long: `Backup the current crowdsec configuration including :
@@ -227,14 +163,8 @@ func (cli *cliConfig) newBackupCmd() *cobra.Command {
 		Example:           `cscli config backup ./my-backup`,
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, args []string) error {
-			if err := cli.backup(args[0]); err != nil {
-				return fmt.Errorf("failed to backup config: %w", err)
-			}
-
-			return nil
-		},
+		RunE:              runConfigBackup,
 	}
 
-	return cmd
+	return cmdConfigBackup
 }

cmd/crowdsec-cli/config_feature_flags.go

@@ -2,16 +2,21 @@ package main
 
 import (
 	"fmt"
-	"path/filepath"
 
 	"github.com/fatih/color"
 	"github.com/spf13/cobra"
 
-	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/fflag"
 )
 
-func (cli *cliConfig) featureFlags(showRetired bool) error {
+func runConfigFeatureFlags(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	showRetired, err := flags.GetBool("retired")
+	if err != nil {
+		return err
+	}
+
 	green := color.New(color.FgGreen).SprintFunc()
 	red := color.New(color.FgRed).SprintFunc()
 	yellow := color.New(color.FgYellow).SprintFunc()
@@ -37,7 +42,6 @@ func (cli *cliConfig) featureFlags(showRetired bool) error {
 		if feat.State == fflag.RetiredState {
 			fmt.Printf("\n  %s %s", magenta("RETIRED"), feat.DeprecationMsg)
 		}
-
 		fmt.Println()
 	}
 
@@ -52,12 +56,10 @@ func (cli *cliConfig) featureFlags(showRetired bool) error {
 			retired = append(retired, feat)
 			continue
 		}
-
 		if feat.IsEnabled() {
 			enabled = append(enabled, feat)
 			continue
 		}
-
 		disabled = append(disabled, feat)
 	}
 
@@ -85,14 +87,7 @@ func (cli *cliConfig) featureFlags(showRetired bool) error {
 
 	fmt.Println("To enable a feature you can: ")
 	fmt.Println("  - set the environment variable CROWDSEC_FEATURE_<uppercase_feature_name> to true")
-
-	featurePath, err := filepath.Abs(csconfig.GetFeatureFilePath(ConfigFilePath))
-	if err != nil {
-		// we already read the file, shouldn't happen
-		return err
-	}
-
-	fmt.Printf("  - add the line '- <feature_name>' to the file %s\n", featurePath)
+	fmt.Printf("  - add the line '- <feature_name>' to the file %s/feature.yaml\n", csConfig.ConfigPaths.ConfigDir)
 	fmt.Println()
 
 	if len(enabled) == 0 && len(disabled) == 0 {
@@ -114,22 +109,18 @@ func (cli *cliConfig) featureFlags(showRetired bool) error {
 	return nil
 }
 
-func (cli *cliConfig) newFeatureFlagsCmd() *cobra.Command {
-	var showRetired bool
-
-	cmd := &cobra.Command{
+func NewConfigFeatureFlagsCmd() *cobra.Command {
+	cmdConfigFeatureFlags := &cobra.Command{
 		Use:               "feature-flags",
 		Short:             "Displays feature flag status",
 		Long:              `Displays the supported feature flags and their current status.`,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			return cli.featureFlags(showRetired)
-		},
+		RunE:              runConfigFeatureFlags,
 	}
 
-	flags := cmd.Flags()
-	flags.BoolVar(&showRetired, "retired", false, "Show retired features")
+	flags := cmdConfigFeatureFlags.Flags()
+	flags.Bool("retired", false, "Show retired features")
 
-	return cmd
+	return cmdConfigFeatureFlags
 }

cmd/crowdsec-cli/config_restore.go

@@ -3,120 +3,26 @@ package main
 import (
 	"encoding/json"
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-func (cli *cliConfig) restoreHub(dirPath string) error {
-	cfg := cli.cfg()
-
-	hub, err := require.Hub(cfg, require.RemoteHub(cfg), nil)
-	if err != nil {
-		return err
-	}
-
-	for _, itype := range cwhub.ItemTypes {
-		itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
-		if _, err = os.Stat(itemDirectory); err != nil {
-			log.Infof("no %s in backup", itype)
-			continue
-		}
-		/*restore the upstream items*/
-		upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
-
-		file, err := os.ReadFile(upstreamListFN)
-		if err != nil {
-			return fmt.Errorf("error while opening %s: %w", upstreamListFN, err)
-		}
-
-		var upstreamList []string
-
-		err = json.Unmarshal(file, &upstreamList)
-		if err != nil {
-			return fmt.Errorf("error unmarshaling %s: %w", upstreamListFN, err)
-		}
-
-		for _, toinstall := range upstreamList {
-			item := hub.GetItem(itype, toinstall)
-			if item == nil {
-				log.Errorf("Item %s/%s not found in hub", itype, toinstall)
-				continue
-			}
-
-			if err = item.Install(false, false); err != nil {
-				log.Errorf("Error while installing %s : %s", toinstall, err)
-			}
-		}
-
-		/*restore the local and tainted items*/
-		files, err := os.ReadDir(itemDirectory)
-		if err != nil {
-			return fmt.Errorf("failed enumerating files of %s: %w", itemDirectory, err)
-		}
-
-		for _, file := range files {
-			// this was the upstream data
-			if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
-				continue
-			}
-
-			if itype == cwhub.PARSERS || itype == cwhub.POSTOVERFLOWS {
-				// we expect a stage here
-				if !file.IsDir() {
-					continue
-				}
-
-				stage := file.Name()
-				stagedir := fmt.Sprintf("%s/%s/%s/", cfg.ConfigPaths.ConfigDir, itype, stage)
-				log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
-
-				if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
-					return fmt.Errorf("error while creating stage directory %s: %w", stagedir, err)
-				}
-
-				// find items
-				ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
-				if err != nil {
-					return fmt.Errorf("failed enumerating files of %s: %w", itemDirectory+"/"+stage, err)
-				}
-
-				// finally copy item
-				for _, tfile := range ifiles {
-					log.Infof("Going to restore local/tainted [%s]", tfile.Name())
-					sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
-
-					destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
-					if err = CopyFile(sourceFile, destinationFile); err != nil {
-						return fmt.Errorf("failed copy %s %s to %s: %w", itype, sourceFile, destinationFile, err)
-					}
-
-					log.Infof("restored %s to %s", sourceFile, destinationFile)
-				}
-			} else {
-				log.Infof("Going to restore local/tainted [%s]", file.Name())
-				sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
-				destinationFile := fmt.Sprintf("%s/%s/%s", cfg.ConfigPaths.ConfigDir, itype, file.Name())
-
-				if err = CopyFile(sourceFile, destinationFile); err != nil {
-					return fmt.Errorf("failed copy %s %s to %s: %w", itype, sourceFile, destinationFile, err)
-				}
-
-				log.Infof("restored %s to %s", sourceFile, destinationFile)
-			}
-		}
-	}
-
-	return nil
+type OldAPICfg struct {
+	MachineID string `json:"machine_id"`
+	Password  string `json:"password"`
 }
 
-/*
-	Restore crowdsec configurations to directory <dirPath>:
+/* Restore crowdsec configurations to directory <dirPath> :
 
 - Main config (config.yaml)
 - Profiles config (profiles.yaml)
@@ -124,66 +30,91 @@ func (cli *cliConfig) restoreHub(dirPath string) error {
 - Backup of API credentials (local API and online API)
 - List of scenarios, parsers, postoverflows and collections that are up-to-date
 - Tainted/local/out-of-date scenarios, parsers, postoverflows and collections
-- Acquisition files (acquis.yaml, acquis.d/*.yaml)
 */
-func (cli *cliConfig) restore(dirPath string) error {
+func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
 	var err error
 
-	cfg := cli.cfg()
-
-	backupMain := fmt.Sprintf("%s/config.yaml", dirPath)
-	if _, err = os.Stat(backupMain); err == nil {
-		if cfg.ConfigPaths != nil && cfg.ConfigPaths.ConfigDir != "" {
-			if err = CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", cfg.ConfigPaths.ConfigDir)); err != nil {
-				return fmt.Errorf("failed copy %s to %s: %w", backupMain, cfg.ConfigPaths.ConfigDir, err)
+	if !oldBackup {
+		backupMain := fmt.Sprintf("%s/config.yaml", dirPath)
+		if _, err = os.Stat(backupMain); err == nil {
+			if csConfig.ConfigPaths != nil && csConfig.ConfigPaths.ConfigDir != "" {
+				if err = types.CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)); err != nil {
+					return fmt.Errorf("failed copy %s to %s : %s", backupMain, csConfig.ConfigPaths.ConfigDir, err)
+				}
 			}
 		}
-	}
 
-	// Now we have config.yaml, we should regenerate config struct to have rights paths etc
-	ConfigFilePath = fmt.Sprintf("%s/config.yaml", cfg.ConfigPaths.ConfigDir)
+		// Now we have config.yaml, we should regenerate config struct to have rights paths etc
+		ConfigFilePath = fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)
 
-	log.Debug("Reloading configuration")
+		initConfig()
 
-	csConfig, _, err = loadConfigFor("config")
-	if err != nil {
-		return fmt.Errorf("failed to reload configuration: %w", err)
-	}
-
-	cfg = cli.cfg()
-
-	backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath)
-	if _, err = os.Stat(backupCAPICreds); err == nil {
-		if err = CopyFile(backupCAPICreds, cfg.API.Server.OnlineClient.CredentialsFilePath); err != nil {
-			return fmt.Errorf("failed copy %s to %s: %w", backupCAPICreds, cfg.API.Server.OnlineClient.CredentialsFilePath, err)
+		backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath)
+		if _, err = os.Stat(backupCAPICreds); err == nil {
+			if err = types.CopyFile(backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath); err != nil {
+				return fmt.Errorf("failed copy %s to %s : %s", backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath, err)
+			}
 		}
-	}
 
-	backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath)
-	if _, err = os.Stat(backupLAPICreds); err == nil {
-		if err = CopyFile(backupLAPICreds, cfg.API.Client.CredentialsFilePath); err != nil {
-			return fmt.Errorf("failed copy %s to %s: %w", backupLAPICreds, cfg.API.Client.CredentialsFilePath, err)
+		backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath)
+		if _, err = os.Stat(backupLAPICreds); err == nil {
+			if err = types.CopyFile(backupLAPICreds, csConfig.API.Client.CredentialsFilePath); err != nil {
+				return fmt.Errorf("failed copy %s to %s : %s", backupLAPICreds, csConfig.API.Client.CredentialsFilePath, err)
+			}
 		}
-	}
 
-	backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath)
-	if _, err = os.Stat(backupProfiles); err == nil {
-		if err = CopyFile(backupProfiles, cfg.API.Server.ProfilesPath); err != nil {
-			return fmt.Errorf("failed copy %s to %s: %w", backupProfiles, cfg.API.Server.ProfilesPath, err)
+		backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath)
+		if _, err = os.Stat(backupProfiles); err == nil {
+			if err = types.CopyFile(backupProfiles, csConfig.API.Server.ProfilesPath); err != nil {
+				return fmt.Errorf("failed copy %s to %s : %s", backupProfiles, csConfig.API.Server.ProfilesPath, err)
+			}
+		}
+	} else {
+		var oldAPICfg OldAPICfg
+		backupOldAPICfg := fmt.Sprintf("%s/api_creds.json", dirPath)
+
+		jsonFile, err := os.Open(backupOldAPICfg)
+		if err != nil {
+			log.Warningf("failed to open %s : %s", backupOldAPICfg, err)
+		} else {
+			byteValue, _ := io.ReadAll(jsonFile)
+			err = json.Unmarshal(byteValue, &oldAPICfg)
+			if err != nil {
+				return fmt.Errorf("failed to load json file %s : %s", backupOldAPICfg, err)
+			}
+
+			apiCfg := csconfig.ApiCredentialsCfg{
+				Login:    oldAPICfg.MachineID,
+				Password: oldAPICfg.Password,
+				URL:      CAPIBaseURL,
+			}
+			apiConfigDump, err := yaml.Marshal(apiCfg)
+			if err != nil {
+				return fmt.Errorf("unable to dump api credentials: %s", err)
+			}
+			apiConfigDumpFile := fmt.Sprintf("%s/online_api_credentials.yaml", csConfig.ConfigPaths.ConfigDir)
+			if csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" {
+				apiConfigDumpFile = csConfig.API.Server.OnlineClient.CredentialsFilePath
+			}
+			err = os.WriteFile(apiConfigDumpFile, apiConfigDump, 0o644)
+			if err != nil {
+				return fmt.Errorf("write api credentials in '%s' failed: %s", apiConfigDumpFile, err)
+			}
+			log.Infof("Saved API credentials to %s", apiConfigDumpFile)
 		}
 	}
 
 	backupSimulation := fmt.Sprintf("%s/simulation.yaml", dirPath)
 	if _, err = os.Stat(backupSimulation); err == nil {
-		if err = CopyFile(backupSimulation, cfg.ConfigPaths.SimulationFilePath); err != nil {
-			return fmt.Errorf("failed copy %s to %s: %w", backupSimulation, cfg.ConfigPaths.SimulationFilePath, err)
+		if err = types.CopyFile(backupSimulation, csConfig.ConfigPaths.SimulationFilePath); err != nil {
+			return fmt.Errorf("failed copy %s to %s : %s", backupSimulation, csConfig.ConfigPaths.SimulationFilePath, err)
 		}
 	}
 
 	/*if there is a acquisition dir, restore its content*/
-	if cfg.Crowdsec.AcquisitionDirPath != "" {
-		if err = os.MkdirAll(cfg.Crowdsec.AcquisitionDirPath, 0o700); err != nil {
-			return fmt.Errorf("error while creating %s: %w", cfg.Crowdsec.AcquisitionDirPath, err)
+	if csConfig.Crowdsec.AcquisitionDirPath != "" {
+		if err = os.Mkdir(csConfig.Crowdsec.AcquisitionDirPath, 0o700); err != nil {
+			return fmt.Errorf("error while creating %s : %s", csConfig.Crowdsec.AcquisitionDirPath, err)
 		}
 	}
 
@@ -192,60 +123,86 @@ func (cli *cliConfig) restore(dirPath string) error {
 	if _, err = os.Stat(backupAcquisition); err == nil {
 		log.Debugf("restoring backup'ed %s", backupAcquisition)
 
-		if err = CopyFile(backupAcquisition, cfg.Crowdsec.AcquisitionFilePath); err != nil {
-			return fmt.Errorf("failed copy %s to %s: %w", backupAcquisition, cfg.Crowdsec.AcquisitionFilePath, err)
+		if err = types.CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil {
+			return fmt.Errorf("failed copy %s to %s : %s", backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath, err)
 		}
 	}
 
-	// if there are files in the acquis backup dir, restore them
+	// if there is files in the acquis backup dir, restore them
 	acquisBackupDir := filepath.Join(dirPath, "acquis", "*.yaml")
 	if acquisFiles, err := filepath.Glob(acquisBackupDir); err == nil {
 		for _, acquisFile := range acquisFiles {
-			targetFname, err := filepath.Abs(cfg.Crowdsec.AcquisitionDirPath + "/" + filepath.Base(acquisFile))
+			targetFname, err := filepath.Abs(csConfig.Crowdsec.AcquisitionDirPath + "/" + filepath.Base(acquisFile))
 			if err != nil {
-				return fmt.Errorf("while saving %s to %s: %w", acquisFile, targetFname, err)
+				return errors.Wrapf(err, "while saving %s to %s", acquisFile, targetFname)
 			}
 
 			log.Debugf("restoring %s to %s", acquisFile, targetFname)
 
-			if err = CopyFile(acquisFile, targetFname); err != nil {
-				return fmt.Errorf("failed copy %s to %s: %w", acquisFile, targetFname, err)
+			if err = types.CopyFile(acquisFile, targetFname); err != nil {
+				return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err)
 			}
 		}
 	}
 
-	if cfg.Crowdsec != nil && len(cfg.Crowdsec.AcquisitionFiles) > 0 {
-		for _, acquisFile := range cfg.Crowdsec.AcquisitionFiles {
+	if csConfig.Crowdsec != nil && len(csConfig.Crowdsec.AcquisitionFiles) > 0 {
+		for _, acquisFile := range csConfig.Crowdsec.AcquisitionFiles {
 			log.Infof("backup filepath from dir -> %s", acquisFile)
 
 			// if it was the default one, it has already been backed up
-			if cfg.Crowdsec.AcquisitionFilePath == acquisFile {
+			if csConfig.Crowdsec.AcquisitionFilePath == acquisFile {
 				log.Infof("skip this one")
 				continue
 			}
 
 			targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile)))
 			if err != nil {
-				return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err)
+				return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir)
 			}
 
-			if err = CopyFile(acquisFile, targetFname); err != nil {
-				return fmt.Errorf("failed copy %s to %s: %w", acquisFile, targetFname, err)
+			if err = types.CopyFile(acquisFile, targetFname); err != nil {
+				return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err)
 			}
 
 			log.Infof("Saved acquis %s to %s", acquisFile, targetFname)
 		}
 	}
 
-	if err = cli.restoreHub(dirPath); err != nil {
-		return fmt.Errorf("failed to restore hub config: %w", err)
+	if err = RestoreHub(dirPath); err != nil {
+		return fmt.Errorf("failed to restore hub config : %s", err)
 	}
 
 	return nil
 }
 
-func (cli *cliConfig) newRestoreCmd() *cobra.Command {
-	cmd := &cobra.Command{
+
+func runConfigRestore(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
+
+	oldBackup, err := flags.GetBool("old-backup")
+	if err != nil {
+		return err
+	}
+
+	if err := csConfig.LoadHub(); err != nil {
+		return err
+	}
+
+	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+		log.Info("Run 'sudo cscli hub update' to get the hub index")
+		return fmt.Errorf("failed to get Hub index: %w", err)
+	}
+
+	if err := restoreConfigFromDirectory(args[0], oldBackup); err != nil {
+		return fmt.Errorf("failed to restore config from %s: %w", args[0], err)
+	}
+
+	return nil
+}
+
+
+func NewConfigRestoreCmd() *cobra.Command {
+	cmdConfigRestore := &cobra.Command{
 		Use:   `restore "directory"`,
 		Short: `Restore config in backup "directory"`,
 		Long: `Restore the crowdsec configuration from specified backup "directory" including:
@@ -258,16 +215,11 @@ func (cli *cliConfig) newRestoreCmd() *cobra.Command {
 - Backup of API credentials (local API and online API)`,
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, args []string) error {
-			dirPath := args[0]
-
-			if err := cli.restore(dirPath); err != nil {
-				return fmt.Errorf("failed to restore config from %s: %w", dirPath, err)
-			}
-
-			return nil
-		},
+		RunE:              runConfigRestore,
 	}
 
-	return cmd
+	flags := cmdConfigRestore.Flags()
+	flags.BoolP("old-backup", "", false, "To use when you are upgrading crowdsec v0.X to v1.X and you need to restore backup from v0.X")
+
+	return cmdConfigRestore
 }

cmd/crowdsec-cli/config_show.go

@@ -7,18 +7,15 @@ import (
 	"text/template"
 
 	"github.com/antonmedv/expr"
-	"github.com/sanity-io/litter"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v3"
+	"gopkg.in/yaml.v2"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 )
 
-func (cli *cliConfig) showKey(key string) error {
-	cfg := cli.cfg()
-
+func showConfigKey(key string) error {
 	type Env struct {
 		Config *csconfig.Config
 	}
@@ -26,26 +23,25 @@ func (cli *cliConfig) showKey(key string) error {
 	opts := []expr.Option{}
 	opts = append(opts, exprhelpers.GetExprOptions(map[string]interface{}{})...)
 	opts = append(opts, expr.Env(Env{}))
-
 	program, err := expr.Compile(key, opts...)
 	if err != nil {
 		return err
 	}
 
-	output, err := expr.Run(program, Env{Config: cfg})
+	output, err := expr.Run(program, Env{Config: csConfig})
 	if err != nil {
 		return err
 	}
 
-	switch cfg.Cscli.Output {
+	switch csConfig.Cscli.Output {
 	case "human", "raw":
-		// Don't use litter for strings, it adds quotes
-		// that would break compatibility with previous versions
 		switch output.(type) {
 		case string:
-			fmt.Println(output)
+			fmt.Printf("%s\n", output)
+		case int:
+			fmt.Printf("%d\n", output)
 		default:
-			litter.Dump(output)
+			fmt.Printf("%v\n", output)
 		}
 	case "json":
 		data, err := json.MarshalIndent(output, "", "  ")
@@ -53,16 +49,15 @@ func (cli *cliConfig) showKey(key string) error {
 			return fmt.Errorf("failed to marshal configuration: %w", err)
 		}
 
-		fmt.Println(string(data))
+		fmt.Printf("%s\n", string(data))
 	}
-
 	return nil
 }
 
-func (cli *cliConfig) template() string {
-	return `Global:
+var configShowTemplate = `Global:
 
 {{- if .ConfigPaths }}
+   - Configuration Folder   : {{.ConfigPaths.ConfigDir}}
    - Configuration Folder   : {{.ConfigPaths.ConfigDir}}
    - Data Folder            : {{.ConfigPaths.DataDir}}
    - Hub Folder             : {{.ConfigPaths.HubDir}}
@@ -76,7 +71,7 @@ func (cli *cliConfig) template() string {
 {{- end }}
 
 {{- if .Crowdsec }}
-Crowdsec{{if and .Crowdsec.Enable (not (ValueBool .Crowdsec.Enable))}} (disabled){{end}}:
+Crowdsec:
   - Acquisition File        : {{.Crowdsec.AcquisitionFilePath}}
   - Parsers routines        : {{.Crowdsec.ParserRoutinesCount}}
 {{- if .Crowdsec.AcquisitionDirPath }}
@@ -88,6 +83,7 @@ Crowdsec{{if and .Crowdsec.Enable (not (ValueBool .Crowdsec.Enable))}} (disabled
 cscli:
   - Output                  : {{.Cscli.Output}}
   - Hub Branch              : {{.Cscli.HubBranch}}
+  - Hub Folder              : {{.Cscli.HubDir}}
 {{- end }}
 
 {{- if .API }}
@@ -101,9 +97,8 @@ API Client:
 {{- end }}
 
 {{- if .API.Server }}
-Local API Server{{if and .API.Server.Enable (not (ValueBool .API.Server.Enable))}} (disabled){{end}}:
+Local API Server:
   - Listen URL              : {{.API.Server.ListenURI}}
-  - Listen Socket           : {{.API.Server.ListenSocket}}
   - Profile File            : {{.API.Server.ProfilesPath}}
 
 {{- if .API.Server.TLS }}
@@ -169,12 +164,6 @@ Central API:
       - User                : {{.DbConfig.User}}
       - DB Name             : {{.DbConfig.DbName}}
 {{- end }}
-{{- if .DbConfig.MaxOpenConns }}
-      - Max Open Conns      : {{.DbConfig.MaxOpenConns}}
-{{- end }}
-{{- if ne .DbConfig.DecisionBulkSize 0 }}
-      - Decision Bulk Size  : {{.DbConfig.DecisionBulkSize}}
-{{- end }}
 {{- if .DbConfig.Flush }}
 {{- if .DbConfig.Flush.MaxAge }}
       - Flush age           : {{.DbConfig.Flush.MaxAge}}
@@ -185,74 +174,64 @@ Central API:
 {{- end }}
 {{- end }}
 `
-}
 
-func (cli *cliConfig) show() error {
-	cfg := cli.cfg()
+func runConfigShow(cmd *cobra.Command, args []string) error {
+	flags := cmd.Flags()
 
-	switch cfg.Cscli.Output {
+	if err := csConfig.LoadAPIClient(); err != nil {
+		log.Errorf("failed to load API client configuration: %s", err)
+		// don't return, we can still show the configuration
+	}
+
+	key, err := flags.GetString("key")
+	if err != nil {
+		return err
+	}
+
+	if key != "" {
+		return showConfigKey(key)
+	}
+
+	switch csConfig.Cscli.Output {
 	case "human":
-		// The tests on .Enable look funny because the option has a true default which has
-		// not been set yet (we don't really load the LAPI) and go templates don't dereference
-		// pointers in boolean tests. Prefix notation is the cherry on top.
-		funcs := template.FuncMap{
-			// can't use generics here
-			"ValueBool": func(b *bool) bool { return b != nil && *b },
-		}
-
-		tmp, err := template.New("config").Funcs(funcs).Parse(cli.template())
+		tmp, err := template.New("config").Parse(configShowTemplate)
 		if err != nil {
 			return err
 		}
-
-		err = tmp.Execute(os.Stdout, cfg)
+		err = tmp.Execute(os.Stdout, csConfig)
 		if err != nil {
 			return err
 		}
 	case "json":
-		data, err := json.MarshalIndent(cfg, "", "  ")
+		data, err := json.MarshalIndent(csConfig, "", "  ")
 		if err != nil {
 			return fmt.Errorf("failed to marshal configuration: %w", err)
 		}
 
-		fmt.Println(string(data))
+		fmt.Printf("%s\n", string(data))
 	case "raw":
-		data, err := yaml.Marshal(cfg)
+		data, err := yaml.Marshal(csConfig)
 		if err != nil {
 			return fmt.Errorf("failed to marshal configuration: %w", err)
 		}
 
-		fmt.Println(string(data))
+		fmt.Printf("%s\n", string(data))
 	}
-
 	return nil
 }
 
-func (cli *cliConfig) newShowCmd() *cobra.Command {
-	var key string
-
-	cmd := &cobra.Command{
+func NewConfigShowCmd() *cobra.Command {
+	cmdConfigShow := &cobra.Command{
 		Use:               "show",
 		Short:             "Displays current config",
 		Long:              `Displays the current cli configuration.`,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			if err := cli.cfg().LoadAPIClient(); err != nil {
-				log.Errorf("failed to load API client configuration: %s", err)
-				// don't return, we can still show the configuration
-			}
-
-			if key != "" {
-				return cli.showKey(key)
-			}
-
-			return cli.show()
-		},
+		RunE:              runConfigShow,
 	}
 
-	flags := cmd.Flags()
-	flags.StringVarP(&key, "key", "", "", "Display only this value (Config.API.Server.ListenURI)")
+	flags := cmdConfigShow.Flags()
+	flags.StringP("key", "", "", "Display only this value (Config.API.Server.ListenURI)")
 
-	return cmd
+	return cmdConfigShow
 }

cmd/crowdsec-cli/config_showyaml.go

@@ -6,21 +6,19 @@ import (
 	"github.com/spf13/cobra"
 )
 
-func (cli *cliConfig) showYAML() error {
+func runConfigShowYAML(cmd *cobra.Command, args []string) error {
 	fmt.Println(mergedConfig)
 	return nil
 }
 
-func (cli *cliConfig) newShowYAMLCmd() *cobra.Command {
-	cmd := &cobra.Command{
+func NewConfigShowYAMLCmd() *cobra.Command {
+	cmdConfigShow := &cobra.Command{
 		Use:               "show-yaml",
 		Short:             "Displays merged config.yaml + config.yaml.local",
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			return cli.showYAML()
-		},
+		RunE:              runConfigShowYAML,
 	}
 
-	return cmd
+	return cmdConfigShow
 }

cmd/crowdsec-cli/console.go

@@ -6,10 +6,9 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io/fs"
 	"net/url"
 	"os"
-	"strconv"
-	"strings"
 
 	"github.com/fatih/color"
 	"github.com/go-openapi/strfmt"
@@ -17,63 +16,51 @@ import (
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v3"
 
-	"github.com/crowdsecurity/go-cs-lib/ptr"
-	"github.com/crowdsecurity/go-cs-lib/version"
+	"github.com/crowdsecurity/go-cs-lib/pkg/ptr"
+	"github.com/crowdsecurity/go-cs-lib/pkg/version"
 
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/crowdsecurity/crowdsec/pkg/fflag"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
-type cliConsole struct {
-	cfg configGetter
-}
-
-func NewCLIConsole(cfg configGetter) *cliConsole {
-	return &cliConsole{
-		cfg: cfg,
-	}
-}
-
-func (cli *cliConsole) NewCommand() *cobra.Command {
-	cmd := &cobra.Command{
+func NewConsoleCmd() *cobra.Command {
+	var cmdConsole = &cobra.Command{
 		Use:               "console [action]",
 		Short:             "Manage interaction with Crowdsec console (https://app.crowdsec.net)",
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
-			cfg := cli.cfg()
-			if err := require.LAPI(cfg); err != nil {
-				return err
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
+				var fdErr *fs.PathError
+				if errors.As(err, &fdErr) {
+					log.Fatalf("Unable to load Local API : %s", fdErr)
+				}
+				if err != nil {
+					log.Fatalf("Unable to load required Local API Configuration : %s", err)
+				}
+				log.Fatal("Local API is disabled, please run this command on the local API machine")
 			}
-			if err := require.CAPI(cfg); err != nil {
-				return err
+			if csConfig.DisableAPI {
+				log.Fatal("Local API is disabled, please run this command on the local API machine")
 			}
-			if err := require.CAPIRegistered(cfg); err != nil {
-				return err
+			if csConfig.API.Server.OnlineClient == nil {
+				log.Fatalf("No configuration for Central API (CAPI) in '%s'", *csConfig.FilePath)
+			}
+			if csConfig.API.Server.OnlineClient.Credentials == nil {
+				log.Fatal("You must configure Central API (CAPI) with `cscli capi register` before accessing console features.")
 			}
-
 			return nil
 		},
 	}
 
-	cmd.AddCommand(cli.newEnrollCmd())
-	cmd.AddCommand(cli.newEnableCmd())
-	cmd.AddCommand(cli.newDisableCmd())
-	cmd.AddCommand(cli.newStatusCmd())
-
-	return cmd
-}
-
-func (cli *cliConsole) newEnrollCmd() *cobra.Command {
 	name := ""
 	overwrite := false
 	tags := []string{}
-	opts := []string{}
 
-	cmd := &cobra.Command{
+	cmdEnroll := &cobra.Command{
 		Use:   "enroll [enroll-key]",
 		Short: "Enroll this instance to https://app.crowdsec.net [requires local API]",
 		Long: `
@@ -81,115 +68,71 @@ Enroll this instance to https://app.crowdsec.net
 		
 You can get your enrollment key by creating an account on https://app.crowdsec.net.
 After running this command your will need to validate the enrollment in the webapp.`,
-		Example: fmt.Sprintf(`cscli console enroll YOUR-ENROLL-KEY
+		Example: `cscli console enroll YOUR-ENROLL-KEY
 		cscli console enroll --name [instance_name] YOUR-ENROLL-KEY
 		cscli console enroll --name [instance_name] --tags [tag_1] --tags [tag_2] YOUR-ENROLL-KEY
-		cscli console enroll --enable context,manual YOUR-ENROLL-KEY
-
-		valid options are : %s,all (see 'cscli console status' for details)`, strings.Join(csconfig.CONSOLE_CONFIGS, ",")),
+`,
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, args []string) error {
-			cfg := cli.cfg()
-			password := strfmt.Password(cfg.API.Server.OnlineClient.Credentials.Password)
-
-			apiURL, err := url.Parse(cfg.API.Server.OnlineClient.Credentials.URL)
+		Run: func(cmd *cobra.Command, args []string) {
+			password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
+			apiURL, err := url.Parse(csConfig.API.Server.OnlineClient.Credentials.URL)
 			if err != nil {
-				return fmt.Errorf("could not parse CAPI URL: %w", err)
+				log.Fatalf("Could not parse CAPI URL : %s", err)
 			}
 
-			hub, err := require.Hub(cfg, nil, nil)
-			if err != nil {
-				return err
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatal(err)
 			}
 
-			scenarios, err := hub.GetInstalledNamesByType(cwhub.SCENARIOS)
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+				log.Fatalf("Failed to load hub index : %s", err)
+				log.Info("Run 'sudo cscli hub update' to get the hub index")
+			}
+
+			scenarios, err := cwhub.GetInstalledScenariosAsString()
 			if err != nil {
-				return fmt.Errorf("failed to get installed scenarios: %w", err)
+				log.Fatalf("failed to get scenarios : %s", err)
 			}
 
 			if len(scenarios) == 0 {
 				scenarios = make([]string, 0)
 			}
 
-			enableOpts := []string{csconfig.SEND_MANUAL_SCENARIOS, csconfig.SEND_TAINTED_SCENARIOS}
-			if len(opts) != 0 {
-				for _, opt := range opts {
-					valid := false
-					if opt == "all" {
-						enableOpts = csconfig.CONSOLE_CONFIGS
-						break
-					}
-					for _, availableOpt := range csconfig.CONSOLE_CONFIGS {
-						if opt == availableOpt {
-							valid = true
-							enable := true
-							for _, enabledOpt := range enableOpts {
-								if opt == enabledOpt {
-									enable = false
-									continue
-								}
-							}
-							if enable {
-								enableOpts = append(enableOpts, opt)
-							}
-
-							break
-						}
-					}
-					if !valid {
-						return fmt.Errorf("option %s doesn't exist", opt)
-					}
-				}
-			}
-
 			c, _ := apiclient.NewClient(&apiclient.Config{
-				MachineID:     cli.cfg().API.Server.OnlineClient.Credentials.Login,
+				MachineID:     csConfig.API.Server.OnlineClient.Credentials.Login,
 				Password:      password,
 				Scenarios:     scenarios,
 				UserAgent:     fmt.Sprintf("crowdsec/%s", version.String()),
 				URL:           apiURL,
 				VersionPrefix: "v3",
 			})
-
 			resp, err := c.Auth.EnrollWatcher(context.Background(), args[0], name, tags, overwrite)
 			if err != nil {
-				return fmt.Errorf("could not enroll instance: %w", err)
+				log.Fatalf("Could not enroll instance: %s", err)
 			}
-
 			if resp.Response.StatusCode == 200 && !overwrite {
 				log.Warning("Instance already enrolled. You can use '--overwrite' to force enroll")
-				return nil
+				return
 			}
 
-			if err := cli.setConsoleOpts(enableOpts, true); err != nil {
-				return err
+			SetConsoleOpts(csconfig.CONSOLE_CONFIGS, true)
+			if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
+				log.Fatalf("failed writing console config : %s", err)
 			}
-
-			for _, opt := range enableOpts {
-				log.Infof("Enabled %s : %s", opt, csconfig.CONSOLE_CONFIGS_HELP[opt])
-			}
-
-			log.Info("Watcher successfully enrolled. Visit https://app.crowdsec.net to accept it.")
-			log.Info("Please restart crowdsec after accepting the enrollment.")
-
-			return nil
+			log.Infof("Enabled tainted&manual alerts sharing, see 'cscli console status'.")
+			log.Infof("Watcher successfully enrolled. Visit https://app.crowdsec.net to accept it.")
+			log.Infof("Please restart crowdsec after accepting the enrollment.")
 		},
 	}
+	cmdEnroll.Flags().StringVarP(&name, "name", "n", "", "Name to display in the console")
+	cmdEnroll.Flags().BoolVarP(&overwrite, "overwrite", "", false, "Force enroll the instance")
+	cmdEnroll.Flags().StringSliceVarP(&tags, "tags", "t", tags, "Tags to display in the console")
+	cmdConsole.AddCommand(cmdEnroll)
 
-	flags := cmd.Flags()
-	flags.StringVarP(&name, "name", "n", "", "Name to display in the console")
-	flags.BoolVarP(&overwrite, "overwrite", "", false, "Force enroll the instance")
-	flags.StringSliceVarP(&tags, "tags", "t", tags, "Tags to display in the console")
-	flags.StringSliceVarP(&opts, "enable", "e", opts, "Enable console options")
+	var enableAll, disableAll bool
 
-	return cmd
-}
-
-func (cli *cliConsole) newEnableCmd() *cobra.Command {
-	var enableAll bool
-
-	cmd := &cobra.Command{
+	cmdEnable := &cobra.Command{
 		Use:     "enable [option]",
 		Short:   "Enable a console option",
 		Example: "sudo cscli console enable tainted",
@@ -197,245 +140,195 @@ func (cli *cliConsole) newEnableCmd() *cobra.Command {
 Enable given information push to the central API. Allows to empower the console`,
 		ValidArgs:         csconfig.CONSOLE_CONFIGS,
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, args []string) error {
+		Run: func(cmd *cobra.Command, args []string) {
 			if enableAll {
-				if err := cli.setConsoleOpts(csconfig.CONSOLE_CONFIGS, true); err != nil {
-					return err
-				}
+				SetConsoleOpts(csconfig.CONSOLE_CONFIGS, true)
 				log.Infof("All features have been enabled successfully")
 			} else {
 				if len(args) == 0 {
-					return errors.New("you must specify at least one feature to enable")
-				}
-				if err := cli.setConsoleOpts(args, true); err != nil {
-					return err
+					log.Fatalf("You must specify at least one feature to enable")
 				}
+				SetConsoleOpts(args, true)
 				log.Infof("%v have been enabled", args)
 			}
-
+			if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
+				log.Fatalf("failed writing console config : %s", err)
+			}
 			log.Infof(ReloadMessage())
-
-			return nil
 		},
 	}
-	cmd.Flags().BoolVarP(&enableAll, "all", "a", false, "Enable all console options")
+	cmdEnable.Flags().BoolVarP(&enableAll, "all", "a", false, "Enable all console options")
+	cmdConsole.AddCommand(cmdEnable)
 
-	return cmd
-}
-
-func (cli *cliConsole) newDisableCmd() *cobra.Command {
-	var disableAll bool
-
-	cmd := &cobra.Command{
+	cmdDisable := &cobra.Command{
 		Use:     "disable [option]",
 		Short:   "Disable a console option",
 		Example: "sudo cscli console disable tainted",
 		Long: `
 Disable given information push to the central API.`,
 		ValidArgs:         csconfig.CONSOLE_CONFIGS,
+		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, args []string) error {
+		Run: func(cmd *cobra.Command, args []string) {
 			if disableAll {
-				if err := cli.setConsoleOpts(csconfig.CONSOLE_CONFIGS, false); err != nil {
-					return err
-				}
-				log.Infof("All features have been disabled")
+				SetConsoleOpts(csconfig.CONSOLE_CONFIGS, false)
 			} else {
-				if err := cli.setConsoleOpts(args, false); err != nil {
-					return err
-				}
-				log.Infof("%v have been disabled", args)
+				SetConsoleOpts(args, false)
 			}
 
+			if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
+				log.Fatalf("failed writing console config : %s", err)
+			}
+			if disableAll {
+				log.Infof("All features have been disabled")
+			} else {
+				log.Infof("%v have been disabled", args)
+			}
 			log.Infof(ReloadMessage())
-
-			return nil
 		},
 	}
-	cmd.Flags().BoolVarP(&disableAll, "all", "a", false, "Disable all console options")
+	cmdDisable.Flags().BoolVarP(&disableAll, "all", "a", false, "Disable all console options")
+	cmdConsole.AddCommand(cmdDisable)
 
-	return cmd
-}
-
-func (cli *cliConsole) newStatusCmd() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:               "status",
-		Short:             "Shows status of the console options",
+	cmdConsoleStatus := &cobra.Command{
+		Use:               "status [option]",
+		Short:             "Shows status of one or all console options",
 		Example:           `sudo cscli console status`,
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			cfg := cli.cfg()
-			consoleCfg := cfg.API.Server.ConsoleConfig
-			switch cfg.Cscli.Output {
+		Run: func(cmd *cobra.Command, args []string) {
+			switch csConfig.Cscli.Output {
 			case "human":
-				cmdConsoleStatusTable(color.Output, *consoleCfg)
+				cmdConsoleStatusTable(color.Output, *csConfig)
 			case "json":
-				out := map[string](*bool){
-					csconfig.SEND_MANUAL_SCENARIOS:  consoleCfg.ShareManualDecisions,
-					csconfig.SEND_CUSTOM_SCENARIOS:  consoleCfg.ShareCustomScenarios,
-					csconfig.SEND_TAINTED_SCENARIOS: consoleCfg.ShareTaintedScenarios,
-					csconfig.SEND_CONTEXT:           consoleCfg.ShareContext,
-					csconfig.CONSOLE_MANAGEMENT:     consoleCfg.ConsoleManagement,
-				}
-				data, err := json.MarshalIndent(out, "", "  ")
+				data, err := json.MarshalIndent(csConfig.API.Server.ConsoleConfig, "", "  ")
 				if err != nil {
-					return fmt.Errorf("failed to marshal configuration: %w", err)
+					log.Fatalf("failed to marshal configuration: %s", err)
 				}
-				fmt.Println(string(data))
+				fmt.Printf("%s\n", string(data))
 			case "raw":
 				csvwriter := csv.NewWriter(os.Stdout)
 				err := csvwriter.Write([]string{"option", "enabled"})
 				if err != nil {
-					return err
+					log.Fatal(err)
 				}
 
 				rows := [][]string{
-					{csconfig.SEND_MANUAL_SCENARIOS, strconv.FormatBool(*consoleCfg.ShareManualDecisions)},
-					{csconfig.SEND_CUSTOM_SCENARIOS, strconv.FormatBool(*consoleCfg.ShareCustomScenarios)},
-					{csconfig.SEND_TAINTED_SCENARIOS, strconv.FormatBool(*consoleCfg.ShareTaintedScenarios)},
-					{csconfig.SEND_CONTEXT, strconv.FormatBool(*consoleCfg.ShareContext)},
-					{csconfig.CONSOLE_MANAGEMENT, strconv.FormatBool(*consoleCfg.ConsoleManagement)},
+					{csconfig.SEND_MANUAL_SCENARIOS, fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareManualDecisions)},
+					{csconfig.SEND_CUSTOM_SCENARIOS, fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareCustomScenarios)},
+					{csconfig.SEND_TAINTED_SCENARIOS, fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios)},
+					{csconfig.SEND_CONTEXT, fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareContext)},
+					{csconfig.CONSOLE_MANAGEMENT, fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ConsoleManagement)},
 				}
 				for _, row := range rows {
 					err = csvwriter.Write(row)
 					if err != nil {
-						return err
+						log.Fatal(err)
 					}
 				}
 				csvwriter.Flush()
 			}
-
-			return nil
 		},
 	}
+	cmdConsole.AddCommand(cmdConsoleStatus)
 
-	return cmd
+	return cmdConsole
 }
 
-func (cli *cliConsole) dumpConfig() error {
-	serverCfg := cli.cfg().API.Server
-
-	out, err := yaml.Marshal(serverCfg.ConsoleConfig)
-	if err != nil {
-		return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", serverCfg.ConsoleConfigPath, err)
-	}
-
-	if serverCfg.ConsoleConfigPath == "" {
-		serverCfg.ConsoleConfigPath = csconfig.DefaultConsoleConfigFilePath
-		log.Debugf("Empty console_path, defaulting to %s", serverCfg.ConsoleConfigPath)
-	}
-
-	if err := os.WriteFile(serverCfg.ConsoleConfigPath, out, 0o600); err != nil {
-		return fmt.Errorf("while dumping console config to %s: %w", serverCfg.ConsoleConfigPath, err)
-	}
-
-	return nil
-}
-
-func (cli *cliConsole) setConsoleOpts(args []string, wanted bool) error {
-	cfg := cli.cfg()
-	consoleCfg := cfg.API.Server.ConsoleConfig
-
+func SetConsoleOpts(args []string, wanted bool) {
 	for _, arg := range args {
 		switch arg {
 		case csconfig.CONSOLE_MANAGEMENT:
+			if !fflag.PapiClient.IsEnabled() {
+				continue
+			}
 			/*for each flag check if it's already set before setting it*/
-			if consoleCfg.ConsoleManagement != nil {
-				if *consoleCfg.ConsoleManagement == wanted {
+			if csConfig.API.Server.ConsoleConfig.ConsoleManagement != nil {
+				if *csConfig.API.Server.ConsoleConfig.ConsoleManagement == wanted {
 					log.Debugf("%s already set to %t", csconfig.CONSOLE_MANAGEMENT, wanted)
 				} else {
 					log.Infof("%s set to %t", csconfig.CONSOLE_MANAGEMENT, wanted)
-					*consoleCfg.ConsoleManagement = wanted
+					*csConfig.API.Server.ConsoleConfig.ConsoleManagement = wanted
 				}
 			} else {
 				log.Infof("%s set to %t", csconfig.CONSOLE_MANAGEMENT, wanted)
-				consoleCfg.ConsoleManagement = ptr.Of(wanted)
+				csConfig.API.Server.ConsoleConfig.ConsoleManagement = ptr.Of(wanted)
 			}
-
-			if cfg.API.Server.OnlineClient.Credentials != nil {
+			if csConfig.API.Server.OnlineClient.Credentials != nil {
 				changed := false
-				if wanted && cfg.API.Server.OnlineClient.Credentials.PapiURL == "" {
+				if wanted && csConfig.API.Server.OnlineClient.Credentials.PapiURL == "" {
 					changed = true
-					cfg.API.Server.OnlineClient.Credentials.PapiURL = types.PAPIBaseURL
-				} else if !wanted && cfg.API.Server.OnlineClient.Credentials.PapiURL != "" {
+					csConfig.API.Server.OnlineClient.Credentials.PapiURL = types.PAPIBaseURL
+				} else if !wanted && csConfig.API.Server.OnlineClient.Credentials.PapiURL != "" {
 					changed = true
-					cfg.API.Server.OnlineClient.Credentials.PapiURL = ""
+					csConfig.API.Server.OnlineClient.Credentials.PapiURL = ""
 				}
-
 				if changed {
-					fileContent, err := yaml.Marshal(cfg.API.Server.OnlineClient.Credentials)
+					fileContent, err := yaml.Marshal(csConfig.API.Server.OnlineClient.Credentials)
 					if err != nil {
-						return fmt.Errorf("cannot marshal credentials: %w", err)
+						log.Fatalf("Cannot marshal credentials: %s", err)
 					}
-
-					log.Infof("Updating credentials file: %s", cfg.API.Server.OnlineClient.CredentialsFilePath)
-
-					err = os.WriteFile(cfg.API.Server.OnlineClient.CredentialsFilePath, fileContent, 0o600)
+					log.Infof("Updating credentials file: %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
+					err = os.WriteFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, fileContent, 0600)
 					if err != nil {
-						return fmt.Errorf("cannot write credentials file: %w", err)
+						log.Fatalf("Cannot write credentials file: %s", err)
 					}
 				}
 			}
 		case csconfig.SEND_CUSTOM_SCENARIOS:
 			/*for each flag check if it's already set before setting it*/
-			if consoleCfg.ShareCustomScenarios != nil {
-				if *consoleCfg.ShareCustomScenarios == wanted {
+			if csConfig.API.Server.ConsoleConfig.ShareCustomScenarios != nil {
+				if *csConfig.API.Server.ConsoleConfig.ShareCustomScenarios == wanted {
 					log.Debugf("%s already set to %t", csconfig.SEND_CUSTOM_SCENARIOS, wanted)
 				} else {
 					log.Infof("%s set to %t", csconfig.SEND_CUSTOM_SCENARIOS, wanted)
-					*consoleCfg.ShareCustomScenarios = wanted
+					*csConfig.API.Server.ConsoleConfig.ShareCustomScenarios = wanted
 				}
 			} else {
 				log.Infof("%s set to %t", csconfig.SEND_CUSTOM_SCENARIOS, wanted)
-				consoleCfg.ShareCustomScenarios = ptr.Of(wanted)
+				csConfig.API.Server.ConsoleConfig.ShareCustomScenarios = ptr.Of(wanted)
 			}
 		case csconfig.SEND_TAINTED_SCENARIOS:
 			/*for each flag check if it's already set before setting it*/
-			if consoleCfg.ShareTaintedScenarios != nil {
-				if *consoleCfg.ShareTaintedScenarios == wanted {
+			if csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios != nil {
+				if *csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios == wanted {
 					log.Debugf("%s already set to %t", csconfig.SEND_TAINTED_SCENARIOS, wanted)
 				} else {
 					log.Infof("%s set to %t", csconfig.SEND_TAINTED_SCENARIOS, wanted)
-					*consoleCfg.ShareTaintedScenarios = wanted
+					*csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios = wanted
 				}
 			} else {
 				log.Infof("%s set to %t", csconfig.SEND_TAINTED_SCENARIOS, wanted)
-				consoleCfg.ShareTaintedScenarios = ptr.Of(wanted)
+				csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios = ptr.Of(wanted)
 			}
 		case csconfig.SEND_MANUAL_SCENARIOS:
 			/*for each flag check if it's already set before setting it*/
-			if consoleCfg.ShareManualDecisions != nil {
-				if *consoleCfg.ShareManualDecisions == wanted {
+			if csConfig.API.Server.ConsoleConfig.ShareManualDecisions != nil {
+				if *csConfig.API.Server.ConsoleConfig.ShareManualDecisions == wanted {
 					log.Debugf("%s already set to %t", csconfig.SEND_MANUAL_SCENARIOS, wanted)
 				} else {
 					log.Infof("%s set to %t", csconfig.SEND_MANUAL_SCENARIOS, wanted)
-					*consoleCfg.ShareManualDecisions = wanted
+					*csConfig.API.Server.ConsoleConfig.ShareManualDecisions = wanted
 				}
 			} else {
 				log.Infof("%s set to %t", csconfig.SEND_MANUAL_SCENARIOS, wanted)
-				consoleCfg.ShareManualDecisions = ptr.Of(wanted)
+				csConfig.API.Server.ConsoleConfig.ShareManualDecisions = ptr.Of(wanted)
 			}
 		case csconfig.SEND_CONTEXT:
 			/*for each flag check if it's already set before setting it*/
-			if consoleCfg.ShareContext != nil {
-				if *consoleCfg.ShareContext == wanted {
+			if csConfig.API.Server.ConsoleConfig.ShareContext != nil {
+				if *csConfig.API.Server.ConsoleConfig.ShareContext == wanted {
 					log.Debugf("%s already set to %t", csconfig.SEND_CONTEXT, wanted)
 				} else {
 					log.Infof("%s set to %t", csconfig.SEND_CONTEXT, wanted)
-					*consoleCfg.ShareContext = wanted
+					*csConfig.API.Server.ConsoleConfig.ShareContext = wanted
 				}
 			} else {
 				log.Infof("%s set to %t", csconfig.SEND_CONTEXT, wanted)
-				consoleCfg.ShareContext = ptr.Of(wanted)
+				csConfig.API.Server.ConsoleConfig.ShareContext = ptr.Of(wanted)
 			}
 		default:
-			return fmt.Errorf("unknown flag %s", arg)
+			log.Fatalf("unknown flag %s", arg)
 		}
 	}
 
-	if err := cli.dumpConfig(); err != nil {
-		return fmt.Errorf("failed writing console config: %w", err)
-	}
-
-	return nil
 }

cmd/crowdsec-cli/console_table.go

@@ -4,12 +4,12 @@ import (
 	"io"
 
 	"github.com/aquasecurity/table"
+	"github.com/enescakir/emoji"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
-	"github.com/crowdsecurity/crowdsec/pkg/emoji"
 )
 
-func cmdConsoleStatusTable(out io.Writer, consoleCfg csconfig.ConsoleConfig) {
+func cmdConsoleStatusTable(out io.Writer, csConfig csconfig.Config) {
 	t := newTable(out)
 	t.SetRowLines(false)
 
@@ -17,32 +17,43 @@ func cmdConsoleStatusTable(out io.Writer, consoleCfg csconfig.ConsoleConfig) {
 	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	for _, option := range csconfig.CONSOLE_CONFIGS {
-		activated := emoji.CrossMark
-
 		switch option {
 		case csconfig.SEND_CUSTOM_SCENARIOS:
-			if *consoleCfg.ShareCustomScenarios {
-				activated = emoji.CheckMarkButton
+			activated := string(emoji.CrossMark)
+			if *csConfig.API.Server.ConsoleConfig.ShareCustomScenarios {
+				activated = string(emoji.CheckMarkButton)
 			}
-		case csconfig.SEND_MANUAL_SCENARIOS:
-			if *consoleCfg.ShareManualDecisions {
-				activated = emoji.CheckMarkButton
-			}
-		case csconfig.SEND_TAINTED_SCENARIOS:
-			if *consoleCfg.ShareTaintedScenarios {
-				activated = emoji.CheckMarkButton
-			}
-		case csconfig.SEND_CONTEXT:
-			if *consoleCfg.ShareContext {
-				activated = emoji.CheckMarkButton
-			}
-		case csconfig.CONSOLE_MANAGEMENT:
-			if *consoleCfg.ConsoleManagement {
-				activated = emoji.CheckMarkButton
-			}
-		}
 
-		t.AddRow(option, activated, csconfig.CONSOLE_CONFIGS_HELP[option])
+			t.AddRow(option, activated, "Send alerts from custom scenarios to the console")
+
+		case csconfig.SEND_MANUAL_SCENARIOS:
+			activated := string(emoji.CrossMark)
+			if *csConfig.API.Server.ConsoleConfig.ShareManualDecisions {
+				activated = string(emoji.CheckMarkButton)
+			}
+
+			t.AddRow(option, activated, "Send manual decisions to the console")
+
+		case csconfig.SEND_TAINTED_SCENARIOS:
+			activated := string(emoji.CrossMark)
+			if *csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios {
+				activated = string(emoji.CheckMarkButton)
+			}
+
+			t.AddRow(option, activated, "Send alerts from tainted scenarios to the console")
+		case csconfig.SEND_CONTEXT:
+			activated := string(emoji.CrossMark)
+			if *csConfig.API.Server.ConsoleConfig.ShareContext {
+				activated = string(emoji.CheckMarkButton)
+			}
+			t.AddRow(option, activated, "Send context with alerts to the console")
+		case csconfig.CONSOLE_MANAGEMENT:
+			activated := string(emoji.CrossMark)
+			if *csConfig.API.Server.ConsoleConfig.ConsoleManagement {
+				activated = string(emoji.CheckMarkButton)
+			}
+			t.AddRow(option, activated, "Receive decisions from console")
+		}
 	}
 
 	t.Render()

cmd/crowdsec-cli/copyfile.go

@@ -1,82 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-
-	log "github.com/sirupsen/logrus"
-)
-
-/*help to copy the file, ioutil doesn't offer the feature*/
-
-func copyFileContents(src, dst string) (err error) {
-	in, err := os.Open(src)
-	if err != nil {
-		return
-	}
-	defer in.Close()
-
-	out, err := os.Create(dst)
-	if err != nil {
-		return
-	}
-
-	defer func() {
-		cerr := out.Close()
-		if err == nil {
-			err = cerr
-		}
-	}()
-
-	if _, err = io.Copy(out, in); err != nil {
-		return
-	}
-
-	err = out.Sync()
-
-	return
-}
-
-/*copy the file, ioutile doesn't offer the feature*/
-func CopyFile(sourceSymLink, destinationFile string) error {
-	sourceFile, err := filepath.EvalSymlinks(sourceSymLink)
-	if err != nil {
-		log.Infof("Not a symlink : %s", err)
-
-		sourceFile = sourceSymLink
-	}
-
-	sourceFileStat, err := os.Stat(sourceFile)
-	if err != nil {
-		return err
-	}
-
-	if !sourceFileStat.Mode().IsRegular() {
-		// cannot copy non-regular files (e.g., directories,
-		// symlinks, devices, etc.)
-		return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String())
-	}
-
-	destinationFileStat, err := os.Stat(destinationFile)
-	if err != nil {
-		if !os.IsNotExist(err) {
-			return err
-		}
-	} else {
-		if !(destinationFileStat.Mode().IsRegular()) {
-			return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String())
-		}
-
-		if os.SameFile(sourceFileStat, destinationFileStat) {
-			return err
-		}
-	}
-
-	if err = os.Link(sourceFile, destinationFile); err != nil {
-		err = copyFileContents(sourceFile, destinationFile)
-	}
-
-	return err
-}

cmd/crowdsec-cli/dashboard.go

@@ -1,8 +1,7 @@
-//go:build linux
-
 package main
 
 import (
+	"errors"
 	"fmt"
 	"math"
 	"os"
@@ -11,7 +10,6 @@ import (
 	"path/filepath"
 	"strconv"
 	"strings"
-	"syscall"
 	"unicode"
 
 	"github.com/AlecAivazis/survey/v2"
@@ -19,18 +17,16 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/metabase"
 )
 
 var (
 	metabaseUser         = "crowdsec@crowdsec.net"
 	metabasePassword     string
-	metabaseDBPath       string
+	metabaseDbPath       string
 	metabaseConfigPath   string
 	metabaseConfigFolder = "metabase/"
 	metabaseConfigFile   = "metabase.yaml"
-	metabaseImage        = "metabase/metabase:v0.46.6.1"
 	/**/
 	metabaseListenAddress = "127.0.0.1"
 	metabaseListenPort    = "3000"
@@ -39,21 +35,12 @@ var (
 
 	forceYes bool
 
-	// information needed to set up a random password on user's behalf
+	/*informations needed to setup a random password on user's behalf*/
 )
 
-type cliDashboard struct {
-	cfg configGetter
-}
-
-func NewCLIDashboard(cfg configGetter) *cliDashboard {
-	return &cliDashboard{
-		cfg: cfg,
-	}
-}
-
-func (cli *cliDashboard) NewCommand() *cobra.Command {
-	cmd := &cobra.Command{
+func NewDashboardCmd() *cobra.Command {
+	/* ---- UPDATE COMMAND */
+	var cmdDashboard = &cobra.Command{
 		Use:   "dashboard [command]",
 		Short: "Manage your metabase dashboard container [requires local API]",
 		Long: `Install/Start/Stop/Remove a metabase container exposing dashboard and metrics.
@@ -67,24 +54,23 @@ cscli dashboard start
 cscli dashboard stop
 cscli dashboard remove
 `,
-		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
-			cfg := cli.cfg()
-			if err := require.LAPI(cfg); err != nil {
-				return err
-			}
-
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
 			if err := metabase.TestAvailability(); err != nil {
-				return err
+				log.Fatalf("%s", err)
 			}
 
-			metabaseConfigFolderPath := filepath.Join(cfg.ConfigPaths.ConfigDir, metabaseConfigFolder)
+			if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
+				log.Fatal("Local API is disabled, please run this command on the local API machine")
+			}
+
+			metabaseConfigFolderPath := filepath.Join(csConfig.ConfigPaths.ConfigDir, metabaseConfigFolder)
 			metabaseConfigPath = filepath.Join(metabaseConfigFolderPath, metabaseConfigFile)
 			if err := os.MkdirAll(metabaseConfigFolderPath, os.ModePerm); err != nil {
-				return err
+				log.Fatal(err)
 			}
-
-			if err := require.DB(cfg); err != nil {
-				return err
+			if err := csConfig.LoadDBConfig(); err != nil {
+				log.Errorf("This command requires direct database access (must be run on the local API machine)")
+				log.Fatal(err)
 			}
 
 			/*
@@ -98,24 +84,23 @@ cscli dashboard remove
 					metabaseContainerID = oldContainerID
 				}
 			}
-
-			return nil
 		},
 	}
 
-	cmd.AddCommand(cli.newSetupCmd())
-	cmd.AddCommand(cli.newStartCmd())
-	cmd.AddCommand(cli.newStopCmd())
-	cmd.AddCommand(cli.newShowPasswordCmd())
-	cmd.AddCommand(cli.newRemoveCmd())
+	cmdDashboard.AddCommand(NewDashboardSetupCmd())
+	cmdDashboard.AddCommand(NewDashboardStartCmd())
+	cmdDashboard.AddCommand(NewDashboardStopCmd())
+	cmdDashboard.AddCommand(NewDashboardShowPasswordCmd())
+	cmdDashboard.AddCommand(NewDashboardRemoveCmd())
 
-	return cmd
+	return cmdDashboard
 }
 
-func (cli *cliDashboard) newSetupCmd() *cobra.Command {
+
+func NewDashboardSetupCmd() *cobra.Command {
 	var force bool
 
-	cmd := &cobra.Command{
+	var cmdDashSetup = &cobra.Command{
 		Use:               "setup",
 		Short:             "Setup a metabase container.",
 		Long:              `Perform a metabase docker setup, download standard dashboards, create a fresh user and start the container`,
@@ -126,9 +111,9 @@ cscli dashboard setup
 cscli dashboard setup --listen 0.0.0.0
 cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
  `,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			if metabaseDBPath == "" {
-				metabaseDBPath = cli.cfg().ConfigPaths.DataDir
+		Run: func(cmd *cobra.Command, args []string) {
+			if metabaseDbPath == "" {
+				metabaseDbPath = csConfig.ConfigPaths.DataDir
 			}
 
 			if metabasePassword == "" {
@@ -138,26 +123,70 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
 					isValid = passwordIsValid(metabasePassword)
 				}
 			}
-			if err := checkSystemMemory(&forceYes); err != nil {
-				return err
+			var answer bool
+			if valid, err := checkSystemMemory(); err == nil && !valid {
+				if !forceYes {
+					prompt := &survey.Confirm{
+						Message: "Metabase requires 1-2GB of RAM, your system is below this requirement continue ?",
+						Default: true,
+					}
+					if err := survey.AskOne(prompt, &answer); err != nil {
+						log.Warnf("unable to ask about RAM check: %s", err)
+					}
+					if !answer {
+						log.Fatal("Unable to continue due to RAM requirement")
+					}
+				} else {
+					log.Warnf("Metabase requires 1-2GB of RAM, your system is below this requirement")
+				}
 			}
-			warnIfNotLoopback(metabaseListenAddress)
-			if err := disclaimer(&forceYes); err != nil {
-				return err
+			groupExist := false
+			dockerGroup, err := user.LookupGroup(crowdsecGroup)
+			if err == nil {
+				groupExist = true
 			}
-			dockerGroup, err := checkGroups(&forceYes)
+			if !forceYes && !groupExist {
+				prompt := &survey.Confirm{
+					Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup),
+					Default: true,
+				}
+				if err := survey.AskOne(prompt, &answer); err != nil {
+					log.Fatalf("unable to ask to force: %s", err)
+				}
+			}
+			if !answer && !forceYes && !groupExist {
+				log.Fatalf("unable to continue without creating '%s' group", crowdsecGroup)
+			}
+			if !groupExist {
+				groupAddCmd, err := exec.LookPath("groupadd")
+				if err != nil {
+					log.Fatalf("unable to find 'groupadd' command, can't continue")
+				}
+
+				groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}}
+				if err := groupAdd.Run(); err != nil {
+					log.Fatalf("unable to add group '%s': %s", dockerGroup, err)
+				}
+				dockerGroup, err = user.LookupGroup(crowdsecGroup)
+				if err != nil {
+					log.Fatalf("unable to lookup '%s' group: %+v", dockerGroup, err)
+				}
+			}
+			intID, err := strconv.Atoi(dockerGroup.Gid)
 			if err != nil {
-				return err
+				log.Fatalf("unable to convert group ID to int: %s", err)
 			}
-			if err = cli.chownDatabase(dockerGroup.Gid); err != nil {
-				return err
+			if err := os.Chown(csConfig.DbConfig.DbPath, 0, intID); err != nil {
+				log.Fatalf("unable to chown sqlite db file '%s': %s", csConfig.DbConfig.DbPath, err)
 			}
-			mb, err := metabase.SetupMetabase(cli.cfg().API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDBPath, dockerGroup.Gid, metabaseContainerID, metabaseImage)
+
+			mb, err := metabase.SetupMetabase(csConfig.API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDbPath, dockerGroup.Gid, metabaseContainerID)
 			if err != nil {
-				return err
+				log.Fatal(err)
 			}
+
 			if err := mb.DumpConfig(metabaseConfigPath); err != nil {
-				return err
+				log.Fatal(err)
 			}
 
 			log.Infof("Metabase is ready")
@@ -165,96 +194,79 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
 			fmt.Printf("\tURL       : '%s'\n", mb.Config.ListenURL)
 			fmt.Printf("\tusername  : '%s'\n", mb.Config.Username)
 			fmt.Printf("\tpassword  : '%s'\n", mb.Config.Password)
-
-			return nil
 		},
 	}
+	cmdDashSetup.Flags().BoolVarP(&force, "force", "f", false, "Force setup : override existing files.")
+	cmdDashSetup.Flags().StringVarP(&metabaseDbPath, "dir", "d", "", "Shared directory with metabase container.")
+	cmdDashSetup.Flags().StringVarP(&metabaseListenAddress, "listen", "l", metabaseListenAddress, "Listen address of container")
+	cmdDashSetup.Flags().StringVarP(&metabaseListenPort, "port", "p", metabaseListenPort, "Listen port of container")
+	cmdDashSetup.Flags().BoolVarP(&forceYes, "yes", "y", false, "force  yes")
+	//cmdDashSetup.Flags().StringVarP(&metabaseUser, "user", "u", "crowdsec@crowdsec.net", "metabase user")
+	cmdDashSetup.Flags().StringVar(&metabasePassword, "password", "", "metabase password")
 
-	flags := cmd.Flags()
-	flags.BoolVarP(&force, "force", "f", false, "Force setup : override existing files")
-	flags.StringVarP(&metabaseDBPath, "dir", "d", "", "Shared directory with metabase container")
-	flags.StringVarP(&metabaseListenAddress, "listen", "l", metabaseListenAddress, "Listen address of container")
-	flags.StringVar(&metabaseImage, "metabase-image", metabaseImage, "Metabase image to use")
-	flags.StringVarP(&metabaseListenPort, "port", "p", metabaseListenPort, "Listen port of container")
-	flags.BoolVarP(&forceYes, "yes", "y", false, "force  yes")
-	// flags.StringVarP(&metabaseUser, "user", "u", "crowdsec@crowdsec.net", "metabase user")
-	flags.StringVar(&metabasePassword, "password", "", "metabase password")
-
-	return cmd
+	return cmdDashSetup
 }
 
-func (cli *cliDashboard) newStartCmd() *cobra.Command {
-	cmd := &cobra.Command{
+func NewDashboardStartCmd() *cobra.Command {
+	var cmdDashStart = &cobra.Command{
 		Use:               "start",
 		Short:             "Start the metabase container.",
 		Long:              `Stats the metabase container using docker.`,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
+		Run: func(cmd *cobra.Command, args []string) {
 			mb, err := metabase.NewMetabase(metabaseConfigPath, metabaseContainerID)
 			if err != nil {
-				return err
-			}
-			warnIfNotLoopback(mb.Config.ListenAddr)
-			if err := disclaimer(&forceYes); err != nil {
-				return err
+				log.Fatal(err)
 			}
 			if err := mb.Container.Start(); err != nil {
-				return fmt.Errorf("failed to start metabase container : %s", err)
+				log.Fatalf("Failed to start metabase container : %s", err)
 			}
 			log.Infof("Started metabase")
-			log.Infof("url : http://%s:%s", mb.Config.ListenAddr, mb.Config.ListenPort)
-
-			return nil
+			log.Infof("url : http://%s:%s", metabaseListenAddress, metabaseListenPort)
 		},
 	}
-
-	cmd.Flags().BoolVarP(&forceYes, "yes", "y", false, "force  yes")
-
-	return cmd
+	return cmdDashStart
 }
 
-func (cli *cliDashboard) newStopCmd() *cobra.Command {
-	cmd := &cobra.Command{
+func NewDashboardStopCmd() *cobra.Command {
+	var cmdDashStop = &cobra.Command{
 		Use:               "stop",
 		Short:             "Stops the metabase container.",
 		Long:              `Stops the metabase container using docker.`,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
+		Run: func(cmd *cobra.Command, args []string) {
 			if err := metabase.StopContainer(metabaseContainerID); err != nil {
-				return fmt.Errorf("unable to stop container '%s': %s", metabaseContainerID, err)
+				log.Fatalf("unable to stop container '%s': %s", metabaseContainerID, err)
 			}
-			return nil
 		},
 	}
-
-	return cmd
+	return cmdDashStop
 }
 
-func (cli *cliDashboard) newShowPasswordCmd() *cobra.Command {
-	cmd := &cobra.Command{Use: "show-password",
+
+func NewDashboardShowPasswordCmd() *cobra.Command {
+	var cmdDashShowPassword = &cobra.Command{Use: "show-password",
 		Short:             "displays password of metabase.",
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
+		Run: func(cmd *cobra.Command, args []string) {
 			m := metabase.Metabase{}
 			if err := m.LoadConfig(metabaseConfigPath); err != nil {
-				return err
+				log.Fatal(err)
 			}
 			log.Printf("'%s'", m.Config.Password)
-
-			return nil
 		},
 	}
-
-	return cmd
+	return cmdDashShowPassword
 }
 
-func (cli *cliDashboard) newRemoveCmd() *cobra.Command {
+
+func NewDashboardRemoveCmd() *cobra.Command {
 	var force bool
 
-	cmd := &cobra.Command{
+	var cmdDashRemove = &cobra.Command{
 		Use:               "remove",
 		Short:             "removes the metabase container.",
 		Long:              `removes the metabase container using docker.`,
@@ -264,77 +276,66 @@ func (cli *cliDashboard) newRemoveCmd() *cobra.Command {
 cscli dashboard remove
 cscli dashboard remove --force
  `,
-		RunE: func(_ *cobra.Command, _ []string) error {
+		Run: func(cmd *cobra.Command, args []string) {
+			answer := true
 			if !forceYes {
-				var answer bool
 				prompt := &survey.Confirm{
 					Message: "Do you really want to remove crowdsec dashboard? (all your changes will be lost)",
 					Default: true,
 				}
 				if err := survey.AskOne(prompt, &answer); err != nil {
-					return fmt.Errorf("unable to ask to force: %s", err)
-				}
-				if !answer {
-					return fmt.Errorf("user stated no to continue")
+					log.Fatalf("unable to ask to force: %s", err)
 				}
 			}
-			if metabase.IsContainerExist(metabaseContainerID) {
-				log.Debugf("Stopping container %s", metabaseContainerID)
-				if err := metabase.StopContainer(metabaseContainerID); err != nil {
-					log.Warningf("unable to stop container '%s': %s", metabaseContainerID, err)
-				}
-				dockerGroup, err := user.LookupGroup(crowdsecGroup)
-				if err == nil { // if group exist, remove it
-					groupDelCmd, err := exec.LookPath("groupdel")
-					if err != nil {
-						return fmt.Errorf("unable to find 'groupdel' command, can't continue")
+			if answer {
+				if metabase.IsContainerExist(metabaseContainerID) {
+					log.Debugf("Stopping container %s", metabaseContainerID)
+					if err := metabase.StopContainer(metabaseContainerID); err != nil {
+						log.Warningf("unable to stop container '%s': %s", metabaseContainerID, err)
 					}
+					dockerGroup, err := user.LookupGroup(crowdsecGroup)
+					if err == nil { // if group exist, remove it
+						groupDelCmd, err := exec.LookPath("groupdel")
+						if err != nil {
+							log.Fatalf("unable to find 'groupdel' command, can't continue")
+						}
 
-					groupDel := &exec.Cmd{Path: groupDelCmd, Args: []string{groupDelCmd, crowdsecGroup}}
-					if err := groupDel.Run(); err != nil {
-						log.Warnf("unable to delete group '%s': %s", dockerGroup, err)
+						groupDel := &exec.Cmd{Path: groupDelCmd, Args: []string{groupDelCmd, crowdsecGroup}}
+						if err := groupDel.Run(); err != nil {
+							log.Errorf("unable to delete group '%s': %s", dockerGroup, err)
+						}
 					}
+					log.Debugf("Removing container %s", metabaseContainerID)
+					if err := metabase.RemoveContainer(metabaseContainerID); err != nil {
+						log.Warningf("unable to remove container '%s': %s", metabaseContainerID, err)
+					}
+					log.Infof("container %s stopped & removed", metabaseContainerID)
 				}
-				log.Debugf("Removing container %s", metabaseContainerID)
-				if err := metabase.RemoveContainer(metabaseContainerID); err != nil {
-					log.Warnf("unable to remove container '%s': %s", metabaseContainerID, err)
+				log.Debugf("Removing metabase db %s", csConfig.ConfigPaths.DataDir)
+				if err := metabase.RemoveDatabase(csConfig.ConfigPaths.DataDir); err != nil {
+					log.Warningf("failed to remove metabase internal db : %s", err)
 				}
-				log.Infof("container %s stopped & removed", metabaseContainerID)
-			}
-			log.Debugf("Removing metabase db %s", cli.cfg().ConfigPaths.DataDir)
-			if err := metabase.RemoveDatabase(cli.cfg().ConfigPaths.DataDir); err != nil {
-				log.Warnf("failed to remove metabase internal db : %s", err)
-			}
-			if force {
-				m := metabase.Metabase{}
-				if err := m.LoadConfig(metabaseConfigPath); err != nil {
-					return err
-				}
-				if err := metabase.RemoveImageContainer(m.Config.Image); err != nil {
-					if !strings.Contains(err.Error(), "No such image") {
-						return fmt.Errorf("removing docker image: %s", err)
+				if force {
+					if err := metabase.RemoveImageContainer(); err != nil {
+						if !strings.Contains(err.Error(), "No such image") {
+							log.Fatalf("removing docker image: %s", err)
+						}
 					}
 				}
 			}
-
-			return nil
 		},
 	}
+	cmdDashRemove.Flags().BoolVarP(&force, "force", "f", false, "Remove also the metabase image")
+	cmdDashRemove.Flags().BoolVarP(&forceYes, "yes", "y", false, "force  yes")
 
-	flags := cmd.Flags()
-	flags.BoolVarP(&force, "force", "f", false, "Remove also the metabase image")
-	flags.BoolVarP(&forceYes, "yes", "y", false, "force  yes")
-
-	return cmd
+	return cmdDashRemove
 }
 
 func passwordIsValid(password string) bool {
 	hasDigit := false
-
 	for _, j := range password {
 		if unicode.IsDigit(j) {
 			hasDigit = true
-
 			break
 		}
 	}
@@ -342,134 +343,17 @@ func passwordIsValid(password string) bool {
 	if !hasDigit || len(password) < 6 {
 		return false
 	}
-
 	return true
+
 }
 
-func checkSystemMemory(forceYes *bool) error {
+func checkSystemMemory() (bool, error) {
 	totMem := memory.TotalMemory()
-	if totMem >= uint64(math.Pow(2, 30)) {
-		return nil
+	if totMem == 0 {
+		return true, errors.New("Unable to get system total memory")
 	}
-
-	if !*forceYes {
-		var answer bool
-
-		prompt := &survey.Confirm{
-			Message: "Metabase requires 1-2GB of RAM, your system is below this requirement continue ?",
-			Default: true,
-		}
-		if err := survey.AskOne(prompt, &answer); err != nil {
-			return fmt.Errorf("unable to ask about RAM check: %s", err)
-		}
-
-		if !answer {
-			return fmt.Errorf("user stated no to continue")
-		}
-
-		return nil
+	if uint64(math.Pow(2, 30)) >= totMem {
+		return false, nil
 	}
-
-	log.Warn("Metabase requires 1-2GB of RAM, your system is below this requirement")
-
-	return nil
-}
-
-func warnIfNotLoopback(addr string) {
-	if addr == "127.0.0.1" || addr == "::1" {
-		return
-	}
-
-	log.Warnf("You are potentially exposing your metabase port to the internet (addr: %s), please consider using a reverse proxy", addr)
-}
-
-func disclaimer(forceYes *bool) error {
-	if !*forceYes {
-		var answer bool
-
-		prompt := &survey.Confirm{
-			Message: "CrowdSec takes no responsibility for the security of your metabase instance. Do you accept these responsibilities ?",
-			Default: true,
-		}
-
-		if err := survey.AskOne(prompt, &answer); err != nil {
-			return fmt.Errorf("unable to ask to question: %s", err)
-		}
-
-		if !answer {
-			return fmt.Errorf("user stated no to responsibilities")
-		}
-
-		return nil
-	}
-
-	log.Warn("CrowdSec takes no responsibility for the security of your metabase instance. You used force yes, so you accept this disclaimer")
-
-	return nil
-}
-
-func checkGroups(forceYes *bool) (*user.Group, error) {
-	dockerGroup, err := user.LookupGroup(crowdsecGroup)
-	if err == nil {
-		return dockerGroup, nil
-	}
-
-	if !*forceYes {
-		var answer bool
-
-		prompt := &survey.Confirm{
-			Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup),
-			Default: true,
-		}
-
-		if err := survey.AskOne(prompt, &answer); err != nil {
-			return dockerGroup, fmt.Errorf("unable to ask to question: %s", err)
-		}
-
-		if !answer {
-			return dockerGroup, fmt.Errorf("unable to continue without creating '%s' group", crowdsecGroup)
-		}
-	}
-
-	groupAddCmd, err := exec.LookPath("groupadd")
-	if err != nil {
-		return dockerGroup, fmt.Errorf("unable to find 'groupadd' command, can't continue")
-	}
-
-	groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}}
-	if err := groupAdd.Run(); err != nil {
-		return dockerGroup, fmt.Errorf("unable to add group '%s': %s", dockerGroup, err)
-	}
-
-	return user.LookupGroup(crowdsecGroup)
-}
-
-func (cli *cliDashboard) chownDatabase(gid string) error {
-	cfg := cli.cfg()
-	intID, err := strconv.Atoi(gid)
-
-	if err != nil {
-		return fmt.Errorf("unable to convert group ID to int: %s", err)
-	}
-
-	if stat, err := os.Stat(cfg.DbConfig.DbPath); !os.IsNotExist(err) {
-		info := stat.Sys()
-		if err := os.Chown(cfg.DbConfig.DbPath, int(info.(*syscall.Stat_t).Uid), intID); err != nil {
-			return fmt.Errorf("unable to chown sqlite db file '%s': %s", cfg.DbConfig.DbPath, err)
-		}
-	}
-
-	if cfg.DbConfig.Type == "sqlite" && cfg.DbConfig.UseWal != nil && *cfg.DbConfig.UseWal {
-		for _, ext := range []string{"-wal", "-shm"} {
-			file := cfg.DbConfig.DbPath + ext
-			if stat, err := os.Stat(file); !os.IsNotExist(err) {
-				info := stat.Sys()
-				if err := os.Chown(file, int(info.(*syscall.Stat_t).Uid), intID); err != nil {
-					return fmt.Errorf("unable to chown sqlite db file '%s': %s", file, err)
-				}
-			}
-		}
-	}
-
-	return nil
+	return true, nil
 }

cmd/crowdsec-cli/dashboard_unsupported.go

@@ -1,32 +0,0 @@
-//go:build !linux
-
-package main
-
-import (
-	"runtime"
-
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-)
-
-type cliDashboard struct{
-	cfg configGetter
-}
-
-func NewCLIDashboard(cfg configGetter) *cliDashboard {
-	return &cliDashboard{
-		cfg: cfg,
-	}
-}
-
-func (cli cliDashboard) NewCommand() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:               "dashboard",
-		DisableAutoGenTag: true,
-		Run: func(_ *cobra.Command, _ []string) {
-			log.Infof("Dashboard command is disabled on %s", runtime.GOOS)
-		},
-	}
-
-	return cmd
-}

cmd/crowdsec-cli/decisions.go

@@ -4,20 +4,23 @@ import (
 	"context"
 	"encoding/csv"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"net/url"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
 
 	"github.com/fatih/color"
 	"github.com/go-openapi/strfmt"
+	"github.com/jszwec/csvutil"
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	"github.com/crowdsecurity/go-cs-lib/version"
+	"github.com/crowdsecurity/go-cs-lib/pkg/ptr"
+	"github.com/crowdsecurity/go-cs-lib/pkg/version"
 
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/models"
@@ -26,7 +29,7 @@ import (
 
 var Client *apiclient.ApiClient
 
-func (cli *cliDecisions) decisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
+func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
 	/*here we cheat a bit : to make it more readable for the user, we dedup some entries*/
 	spamLimit := make(map[string]bool)
 	skipped := 0
@@ -34,36 +37,27 @@ func (cli *cliDecisions) decisionsToTable(alerts *models.GetAlertsResponse, prin
 	for aIdx := 0; aIdx < len(*alerts); aIdx++ {
 		alertItem := (*alerts)[aIdx]
 		newDecisions := make([]*models.Decision, 0)
-
 		for _, decisionItem := range alertItem.Decisions {
 			spamKey := fmt.Sprintf("%t:%s:%s:%s", *decisionItem.Simulated, *decisionItem.Type, *decisionItem.Scope, *decisionItem.Value)
 			if _, ok := spamLimit[spamKey]; ok {
 				skipped++
 				continue
 			}
-
 			spamLimit[spamKey] = true
-
 			newDecisions = append(newDecisions, decisionItem)
 		}
-
 		alertItem.Decisions = newDecisions
 	}
-
-	switch cli.cfg().Cscli.Output {
-	case "raw":
+	if csConfig.Cscli.Output == "raw" {
 		csvwriter := csv.NewWriter(os.Stdout)
 		header := []string{"id", "source", "ip", "reason", "action", "country", "as", "events_count", "expiration", "simulated", "alert_id"}
-
 		if printMachine {
 			header = append(header, "machine")
 		}
-
 		err := csvwriter.Write(header)
 		if err != nil {
 			return err
 		}
-
 		for _, alertItem := range *alerts {
 			for _, decisionItem := range alertItem.Decisions {
 				raw := []string{
@@ -89,46 +83,25 @@ func (cli *cliDecisions) decisionsToTable(alerts *models.GetAlertsResponse, prin
 				}
 			}
 		}
-
 		csvwriter.Flush()
-	case "json":
-		if *alerts == nil {
-			// avoid returning "null" in `json"
-			// could be cleaner if we used slice of alerts directly
-			fmt.Println("[]")
-			return nil
-		}
-
+	} else if csConfig.Cscli.Output == "json" {
 		x, _ := json.MarshalIndent(alerts, "", " ")
 		fmt.Printf("%s", string(x))
-	case "human":
+	} else if csConfig.Cscli.Output == "human" {
 		if len(*alerts) == 0 {
 			fmt.Println("No active decisions")
 			return nil
 		}
-
-		cli.decisionsTable(color.Output, alerts, printMachine)
-
+		decisionsTable(color.Output, alerts, printMachine)
 		if skipped > 0 {
 			fmt.Printf("%d duplicated entries skipped\n", skipped)
 		}
 	}
-
 	return nil
 }
 
-type cliDecisions struct {
-	cfg configGetter
-}
-
-func NewCLIDecisions(cfg configGetter) *cliDecisions {
-	return &cliDecisions{
-		cfg: cfg,
-	}
-}
-
-func (cli *cliDecisions) NewCommand() *cobra.Command {
-	cmd := &cobra.Command{
+func NewDecisionsCmd() *cobra.Command {
+	var cmdDecisions = &cobra.Command{
 		Use:     "decisions [action]",
 		Short:   "Manage decisions",
 		Long:    `Add/List/Delete/Import decisions from LAPI`,
@@ -137,40 +110,38 @@ func (cli *cliDecisions) NewCommand() *cobra.Command {
 		/*TBD example*/
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
-			cfg := cli.cfg()
-			if err := cfg.LoadAPIClient(); err != nil {
-				return fmt.Errorf("loading api client: %w", err)
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := csConfig.LoadAPIClient(); err != nil {
+				return errors.Wrap(err, "loading api client")
 			}
-			password := strfmt.Password(cfg.API.Client.Credentials.Password)
-			apiurl, err := url.Parse(cfg.API.Client.Credentials.URL)
+			password := strfmt.Password(csConfig.API.Client.Credentials.Password)
+			apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL)
 			if err != nil {
-				return fmt.Errorf("parsing api url %s: %w", cfg.API.Client.Credentials.URL, err)
+				return errors.Wrapf(err, "parsing api url %s", csConfig.API.Client.Credentials.URL)
 			}
 			Client, err = apiclient.NewClient(&apiclient.Config{
-				MachineID:     cfg.API.Client.Credentials.Login,
+				MachineID:     csConfig.API.Client.Credentials.Login,
 				Password:      password,
 				UserAgent:     fmt.Sprintf("crowdsec/%s", version.String()),
 				URL:           apiurl,
 				VersionPrefix: "v1",
 			})
 			if err != nil {
-				return fmt.Errorf("creating api client: %w", err)
+				return errors.Wrap(err, "creating api client")
 			}
-
 			return nil
 		},
 	}
 
-	cmd.AddCommand(cli.newListCmd())
-	cmd.AddCommand(cli.newAddCmd())
-	cmd.AddCommand(cli.newDeleteCmd())
-	cmd.AddCommand(cli.newImportCmd())
+	cmdDecisions.AddCommand(NewDecisionsListCmd())
+	cmdDecisions.AddCommand(NewDecisionsAddCmd())
+	cmdDecisions.AddCommand(NewDecisionsDeleteCmd())
+	cmdDecisions.AddCommand(NewDecisionsImportCmd())
 
-	return cmd
+	return cmdDecisions
 }
 
-func (cli *cliDecisions) newListCmd() *cobra.Command {
+func NewDecisionsListCmd() *cobra.Command {
 	var filter = apiclient.AlertsListOpts{
 		ValueEquals:    new(string),
 		ScopeEquals:    new(string),
@@ -184,27 +155,25 @@ func (cli *cliDecisions) newListCmd() *cobra.Command {
 		IncludeCAPI:    new(bool),
 		Limit:          new(int),
 	}
-
 	NoSimu := new(bool)
 	contained := new(bool)
-
 	var printMachine bool
 
-	cmd := &cobra.Command{
+	var cmdDecisionsList = &cobra.Command{
 		Use:   "list [options]",
 		Short: "List decisions from LAPI",
 		Example: `cscli decisions list -i 1.2.3.4
 cscli decisions list -r 1.2.3.0/24
 cscli decisions list -s crowdsecurity/ssh-bf
-cscli decisions list --origin lists --scenario list_name
+cscli decisions list -t ban
 `,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, _ []string) error {
+		Run: func(cmd *cobra.Command, args []string) {
 			var err error
 			/*take care of shorthand options*/
-			if err = manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil {
-				return err
+			if err := manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil {
+				log.Fatalf("%s", err)
 			}
 			filter.ActiveDecisionEquals = new(bool)
 			*filter.ActiveDecisionEquals = true
@@ -220,7 +189,7 @@ cscli decisions list --origin lists --scenario list_name
 				days, err := strconv.Atoi(realDuration)
 				if err != nil {
 					printHelp(cmd)
-					return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until)
+					log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until)
 				}
 				*filter.Until = fmt.Sprintf("%d%s", days*24, "h")
 			}
@@ -233,7 +202,7 @@ cscli decisions list --origin lists --scenario list_name
 				days, err := strconv.Atoi(realDuration)
 				if err != nil {
 					printHelp(cmd)
-					return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Since)
+					log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until)
 				}
 				*filter.Since = fmt.Sprintf("%d%s", days*24, "h")
 			}
@@ -269,37 +238,35 @@ cscli decisions list --origin lists --scenario list_name
 
 			alerts, _, err := Client.Alerts.List(context.Background(), filter)
 			if err != nil {
-				return fmt.Errorf("unable to retrieve decisions: %w", err)
+				log.Fatalf("Unable to list decisions : %v", err)
 			}
 
-			err = cli.decisionsToTable(alerts, printMachine)
+			err = DecisionsToTable(alerts, printMachine)
 			if err != nil {
-				return fmt.Errorf("unable to print decisions: %w", err)
+				log.Fatalf("unable to list decisions : %v", err)
 			}
-
-			return nil
 		},
 	}
-	cmd.Flags().SortFlags = false
-	cmd.Flags().BoolVarP(filter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
-	cmd.Flags().StringVar(filter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
-	cmd.Flags().StringVar(filter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
-	cmd.Flags().StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)")
-	cmd.Flags().StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)")
-	cmd.Flags().StringVar(filter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
-	cmd.Flags().StringVarP(filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)")
-	cmd.Flags().StringVarP(filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)")
-	cmd.Flags().StringVarP(filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
-	cmd.Flags().StringVarP(filter.RangeEquals, "range", "r", "", "restrict to alerts from this source range (shorthand for --scope range --value <RANGE>)")
-	cmd.Flags().IntVarP(filter.Limit, "limit", "l", 100, "number of alerts to get (use 0 to remove the limit)")
-	cmd.Flags().BoolVar(NoSimu, "no-simu", false, "exclude decisions in simulation mode")
-	cmd.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that triggered decisions")
-	cmd.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
+	cmdDecisionsList.Flags().SortFlags = false
+	cmdDecisionsList.Flags().BoolVarP(filter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
+	cmdDecisionsList.Flags().StringVar(filter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
+	cmdDecisionsList.Flags().StringVar(filter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
+	cmdDecisionsList.Flags().StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)")
+	cmdDecisionsList.Flags().StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)")
+	cmdDecisionsList.Flags().StringVar(filter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
+	cmdDecisionsList.Flags().StringVarP(filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)")
+	cmdDecisionsList.Flags().StringVarP(filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)")
+	cmdDecisionsList.Flags().StringVarP(filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
+	cmdDecisionsList.Flags().StringVarP(filter.RangeEquals, "range", "r", "", "restrict to alerts from this source range (shorthand for --scope range --value <RANGE>)")
+	cmdDecisionsList.Flags().IntVarP(filter.Limit, "limit", "l", 100, "number of alerts to get (use 0 to remove the limit)")
+	cmdDecisionsList.Flags().BoolVar(NoSimu, "no-simu", false, "exclude decisions in simulation mode")
+	cmdDecisionsList.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that triggered decisions")
+	cmdDecisionsList.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
 
-	return cmd
+	return cmdDecisionsList
 }
 
-func (cli *cliDecisions) newAddCmd() *cobra.Command {
+func NewDecisionsAddCmd() *cobra.Command {
 	var (
 		addIP       string
 		addRange    string
@@ -310,7 +277,7 @@ func (cli *cliDecisions) newAddCmd() *cobra.Command {
 		addType     string
 	)
 
-	cmd := &cobra.Command{
+	var cmdDecisionsAdd = &cobra.Command{
 		Use:   "add [options]",
 		Short: "Add decision to LAPI",
 		Example: `cscli decisions add --ip 1.2.3.4
@@ -321,7 +288,7 @@ cscli decisions add --scope username --value foobar
 		/*TBD : fix long and example*/
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, _ []string) error {
+		Run: func(cmd *cobra.Command, args []string) {
 			var err error
 			alerts := models.AddAlertsRequest{}
 			origin := types.CscliOrigin
@@ -335,8 +302,8 @@ cscli decisions add --scope username --value foobar
 			createdAt := time.Now().UTC().Format(time.RFC3339)
 
 			/*take care of shorthand options*/
-			if err = manageCliDecisionAlerts(&addIP, &addRange, &addScope, &addValue); err != nil {
-				return err
+			if err := manageCliDecisionAlerts(&addIP, &addRange, &addScope, &addValue); err != nil {
+				log.Fatalf("%s", err)
 			}
 
 			if addIP != "" {
@@ -347,11 +314,11 @@ cscli decisions add --scope username --value foobar
 				addScope = types.Range
 			} else if addValue == "" {
 				printHelp(cmd)
-				return errors.New("missing arguments, a value is required (--ip, --range or --scope and --value)")
+				log.Fatalf("Missing arguments, a value is required (--ip, --range or --scope and --value)")
 			}
 
 			if addReason == "" {
-				addReason = fmt.Sprintf("manual '%s' from '%s'", addType, cli.cfg().API.Client.Credentials.Login)
+				addReason = fmt.Sprintf("manual '%s' from '%s'", addType, csConfig.API.Client.Credentials.Login)
 			}
 			decision := models.Decision{
 				Duration: &addDuration,
@@ -372,7 +339,7 @@ cscli decisions add --scope username --value foobar
 				Scenario:        &addReason,
 				ScenarioVersion: &empty,
 				Simulated:       &simulated,
-				// setting empty scope/value broke plugins, and it didn't seem to be needed anymore w/ latest papi changes
+				//setting empty scope/value broke plugins, and it didn't seem to be needed anymore w/ latest papi changes
 				Source: &models.Source{
 					AsName:   empty,
 					AsNumber: empty,
@@ -390,29 +357,27 @@ cscli decisions add --scope username --value foobar
 
 			_, _, err = Client.Alerts.Add(context.Background(), alerts)
 			if err != nil {
-				return err
+				log.Fatal(err)
 			}
 
 			log.Info("Decision successfully added")
-
-			return nil
 		},
 	}
 
-	cmd.Flags().SortFlags = false
-	cmd.Flags().StringVarP(&addIP, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
-	cmd.Flags().StringVarP(&addRange, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
-	cmd.Flags().StringVarP(&addDuration, "duration", "d", "4h", "Decision duration (ie. 1h,4h,30m)")
-	cmd.Flags().StringVarP(&addValue, "value", "v", "", "The value (ie. --scope username --value foobar)")
-	cmd.Flags().StringVar(&addScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)")
-	cmd.Flags().StringVarP(&addReason, "reason", "R", "", "Decision reason (ie. scenario-name)")
-	cmd.Flags().StringVarP(&addType, "type", "t", "ban", "Decision type (ie. ban,captcha,throttle)")
+	cmdDecisionsAdd.Flags().SortFlags = false
+	cmdDecisionsAdd.Flags().StringVarP(&addIP, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
+	cmdDecisionsAdd.Flags().StringVarP(&addRange, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
+	cmdDecisionsAdd.Flags().StringVarP(&addDuration, "duration", "d", "4h", "Decision duration (ie. 1h,4h,30m)")
+	cmdDecisionsAdd.Flags().StringVarP(&addValue, "value", "v", "", "The value (ie. --scope username --value foobar)")
+	cmdDecisionsAdd.Flags().StringVar(&addScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)")
+	cmdDecisionsAdd.Flags().StringVarP(&addReason, "reason", "R", "", "Decision reason (ie. scenario-name)")
+	cmdDecisionsAdd.Flags().StringVarP(&addType, "type", "t", "ban", "Decision type (ie. ban,captcha,throttle)")
 
-	return cmd
+	return cmdDecisionsAdd
 }
 
-func (cli *cliDecisions) newDeleteCmd() *cobra.Command {
-	delFilter := apiclient.DecisionsDeleteOpts{
+func NewDecisionsDeleteCmd() *cobra.Command {
+	var delFilter = apiclient.DecisionsDeleteOpts{
 		ScopeEquals:    new(string),
 		ValueEquals:    new(string),
 		TypeEquals:     new(string),
@@ -421,14 +386,11 @@ func (cli *cliDecisions) newDeleteCmd() *cobra.Command {
 		ScenarioEquals: new(string),
 		OriginEquals:   new(string),
 	}
-
-	var delDecisionID string
-
+	var delDecisionId string
 	var delDecisionAll bool
-
 	contained := new(bool)
 
-	cmd := &cobra.Command{
+	var cmdDecisionsDelete = &cobra.Command{
 		Use:               "delete [options]",
 		Short:             "Delete decisions",
 		DisableAutoGenTag: true,
@@ -437,30 +399,27 @@ func (cli *cliDecisions) newDeleteCmd() *cobra.Command {
 cscli decisions delete -i 1.2.3.4
 cscli decisions delete --id 42
 cscli decisions delete --type captcha
-cscli decisions delete --origin lists  --scenario list_name
 `,
 		/*TBD : refaire le Long/Example*/
-		PreRunE: func(cmd *cobra.Command, _ []string) error {
+		PreRun: func(cmd *cobra.Command, args []string) {
 			if delDecisionAll {
-				return nil
+				return
 			}
 			if *delFilter.ScopeEquals == "" && *delFilter.ValueEquals == "" &&
 				*delFilter.TypeEquals == "" && *delFilter.IPEquals == "" &&
 				*delFilter.RangeEquals == "" && *delFilter.ScenarioEquals == "" &&
-				*delFilter.OriginEquals == "" && delDecisionID == "" {
+				*delFilter.OriginEquals == "" && delDecisionId == "" {
 				cmd.Usage()
-				return errors.New("at least one filter or --all must be specified")
+				log.Fatalln("At least one filter or --all must be specified")
 			}
-
-			return nil
 		},
-		RunE: func(_ *cobra.Command, _ []string) error {
+		Run: func(cmd *cobra.Command, args []string) {
 			var err error
 			var decisions *models.DeleteDecisionResponse
 
 			/*take care of shorthand options*/
-			if err = manageCliDecisionAlerts(delFilter.IPEquals, delFilter.RangeEquals, delFilter.ScopeEquals, delFilter.ValueEquals); err != nil {
-				return err
+			if err := manageCliDecisionAlerts(delFilter.IPEquals, delFilter.RangeEquals, delFilter.ScopeEquals, delFilter.ValueEquals); err != nil {
+				log.Fatalf("%s", err)
 			}
 			if *delFilter.ScopeEquals == "" {
 				delFilter.ScopeEquals = nil
@@ -487,37 +446,224 @@ cscli decisions delete --origin lists  --scenario list_name
 				delFilter.Contains = new(bool)
 			}
 
-			if delDecisionID == "" {
+			if delDecisionId == "" {
 				decisions, _, err = Client.Decisions.Delete(context.Background(), delFilter)
 				if err != nil {
-					return fmt.Errorf("unable to delete decisions: %v", err)
+					log.Fatalf("Unable to delete decisions : %v", err)
 				}
 			} else {
-				if _, err = strconv.Atoi(delDecisionID); err != nil {
-					return fmt.Errorf("id '%s' is not an integer: %v", delDecisionID, err)
+				if _, err = strconv.Atoi(delDecisionId); err != nil {
+					log.Fatalf("id '%s' is not an integer: %v", delDecisionId, err)
 				}
-				decisions, _, err = Client.Decisions.DeleteOne(context.Background(), delDecisionID)
+				decisions, _, err = Client.Decisions.DeleteOne(context.Background(), delDecisionId)
 				if err != nil {
-					return fmt.Errorf("unable to delete decision: %v", err)
+					log.Fatalf("Unable to delete decision : %v", err)
 				}
 			}
 			log.Infof("%s decision(s) deleted", decisions.NbDeleted)
-
-			return nil
 		},
 	}
 
-	cmd.Flags().SortFlags = false
-	cmd.Flags().StringVarP(delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
-	cmd.Flags().StringVarP(delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
-	cmd.Flags().StringVarP(delFilter.TypeEquals, "type", "t", "", "the decision type (ie. ban,captcha)")
-	cmd.Flags().StringVarP(delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
-	cmd.Flags().StringVarP(delFilter.ScenarioEquals, "scenario", "s", "", "the scenario name (ie. crowdsecurity/ssh-bf)")
-	cmd.Flags().StringVar(delFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
+	cmdDecisionsDelete.Flags().SortFlags = false
+	cmdDecisionsDelete.Flags().StringVarP(delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
+	cmdDecisionsDelete.Flags().StringVarP(delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
+	cmdDecisionsDelete.Flags().StringVarP(delFilter.TypeEquals, "type", "t", "", "the decision type (ie. ban,captcha)")
+	cmdDecisionsDelete.Flags().StringVarP(delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
+	cmdDecisionsDelete.Flags().StringVarP(delFilter.ScenarioEquals, "scenario", "s", "", "the scenario name (ie. crowdsecurity/ssh-bf)")
+	cmdDecisionsDelete.Flags().StringVar(delFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
 
-	cmd.Flags().StringVar(&delDecisionID, "id", "", "decision id")
-	cmd.Flags().BoolVar(&delDecisionAll, "all", false, "delete all decisions")
-	cmd.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
+	cmdDecisionsDelete.Flags().StringVar(&delDecisionId, "id", "", "decision id")
+	cmdDecisionsDelete.Flags().BoolVar(&delDecisionAll, "all", false, "delete all decisions")
+	cmdDecisionsDelete.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
 
-	return cmd
+	return cmdDecisionsDelete
+}
+
+func NewDecisionsImportCmd() *cobra.Command {
+	var (
+		defaultDuration = "4h"
+		defaultScope    = "ip"
+		defaultType     = "ban"
+		defaultReason   = "manual"
+		importDuration  string
+		importScope     string
+		importReason    string
+		importType      string
+		importFile      string
+		batchSize       int
+	)
+
+	var cmdDecisionImport = &cobra.Command{
+		Use:   "import [options]",
+		Short: "Import decisions from json or csv file",
+		Long: "expected format :\n" +
+			"csv  : any of duration,origin,reason,scope,type,value, with a header line\n" +
+			`json : {"duration" : "24h", "origin" : "my-list", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`,
+		DisableAutoGenTag: true,
+		Example: `decisions.csv :
+duration,scope,value
+24h,ip,1.2.3.4
+
+cscsli decisions import -i decisions.csv
+
+decisions.json :
+[{"duration" : "4h", "scope" : "ip", "type" : "ban", "value" : "1.2.3.4"}]
+`,
+		Run: func(cmd *cobra.Command, args []string) {
+			if importFile == "" {
+				log.Fatalf("Please provide a input file containing decisions with -i flag")
+			}
+			csvData, err := os.ReadFile(importFile)
+			if err != nil {
+				log.Fatalf("unable to open '%s': %s", importFile, err)
+			}
+			type decisionRaw struct {
+				Duration string `csv:"duration,omitempty" json:"duration,omitempty"`
+				Origin   string `csv:"origin,omitempty" json:"origin,omitempty"`
+				Scenario string `csv:"reason,omitempty" json:"reason,omitempty"`
+				Scope    string `csv:"scope,omitempty" json:"scope,omitempty"`
+				Type     string `csv:"type,omitempty" json:"type,omitempty"`
+				Value    string `csv:"value" json:"value"`
+			}
+			var decisionsListRaw []decisionRaw
+			switch fileFormat := filepath.Ext(importFile); fileFormat {
+			case ".json":
+				if err := json.Unmarshal(csvData, &decisionsListRaw); err != nil {
+					log.Fatalf("unable to unmarshall json: '%s'", err)
+				}
+			case ".csv":
+				if err := csvutil.Unmarshal(csvData, &decisionsListRaw); err != nil {
+					log.Fatalf("unable to unmarshall csv: '%s'", err)
+				}
+			default:
+				log.Fatalf("file format not supported for '%s'. supported format are 'json' and 'csv'", importFile)
+			}
+
+			decisionsList := make([]*models.Decision, 0)
+			for i, decisionLine := range decisionsListRaw {
+				line := i + 2
+				if decisionLine.Value == "" {
+					log.Fatalf("please provide a 'value' in your csv line %d", line)
+				}
+				/*deal with defaults and cli-override*/
+				if decisionLine.Duration == "" {
+					decisionLine.Duration = defaultDuration
+					log.Debugf("No 'duration' line %d, using default value: '%s'", line, defaultDuration)
+				}
+				if importDuration != "" {
+					decisionLine.Duration = importDuration
+					log.Debugf("'duration' line %d, using supplied value: '%s'", line, importDuration)
+				}
+				decisionLine.Origin = types.CscliImportOrigin
+
+				if decisionLine.Scenario == "" {
+					decisionLine.Scenario = defaultReason
+					log.Debugf("No 'reason' line %d, using value: '%s'", line, decisionLine.Scenario)
+				}
+				if importReason != "" {
+					decisionLine.Scenario = importReason
+					log.Debugf("No 'reason' line %d, using supplied value: '%s'", line, importReason)
+				}
+				if decisionLine.Type == "" {
+					decisionLine.Type = defaultType
+					log.Debugf("No 'type' line %d, using default value: '%s'", line, decisionLine.Type)
+				}
+				if importType != "" {
+					decisionLine.Type = importType
+					log.Debugf("'type' line %d, using supplied value: '%s'", line, importType)
+				}
+				if decisionLine.Scope == "" {
+					decisionLine.Scope = defaultScope
+					log.Debugf("No 'scope' line %d, using default value: '%s'", line, decisionLine.Scope)
+				}
+				if importScope != "" {
+					decisionLine.Scope = importScope
+					log.Debugf("'scope' line %d, using supplied value: '%s'", line, importScope)
+				}
+				decision := models.Decision{
+					Value:     ptr.Of(decisionLine.Value),
+					Duration:  ptr.Of(decisionLine.Duration),
+					Origin:    ptr.Of(decisionLine.Origin),
+					Scenario:  ptr.Of(decisionLine.Scenario),
+					Type:      ptr.Of(decisionLine.Type),
+					Scope:     ptr.Of(decisionLine.Scope),
+					Simulated: new(bool),
+				}
+				decisionsList = append(decisionsList, &decision)
+			}
+			alerts := models.AddAlertsRequest{}
+
+			if batchSize > 0 {
+				for i := 0; i < len(decisionsList); i += batchSize {
+					end := i + batchSize
+					if end > len(decisionsList) {
+						end = len(decisionsList)
+					}
+					decisionBatch := decisionsList[i:end]
+					importAlert := models.Alert{
+						CreatedAt: time.Now().UTC().Format(time.RFC3339),
+						Scenario:  ptr.Of(fmt.Sprintf("import %s : %d IPs", importFile, len(decisionBatch))),
+
+						Message: ptr.Of(""),
+						Events:  []*models.Event{},
+						Source: &models.Source{
+							Scope: ptr.Of(""),
+							Value: ptr.Of(""),
+						},
+						StartAt:         ptr.Of(time.Now().UTC().Format(time.RFC3339)),
+						StopAt:          ptr.Of(time.Now().UTC().Format(time.RFC3339)),
+						Capacity:        ptr.Of(int32(0)),
+						Simulated:       ptr.Of(false),
+						EventsCount:     ptr.Of(int32(len(decisionBatch))),
+						Leakspeed:       ptr.Of(""),
+						ScenarioHash:    ptr.Of(""),
+						ScenarioVersion: ptr.Of(""),
+						Decisions:       decisionBatch,
+					}
+					alerts = append(alerts, &importAlert)
+				}
+			} else {
+				importAlert := models.Alert{
+					CreatedAt: time.Now().UTC().Format(time.RFC3339),
+					Scenario:  ptr.Of(fmt.Sprintf("import %s : %d IPs", importFile, len(decisionsList))),
+					Message:   ptr.Of(""),
+					Events:    []*models.Event{},
+					Source: &models.Source{
+						Scope: ptr.Of(""),
+						Value: ptr.Of(""),
+					},
+					StartAt:         ptr.Of(time.Now().UTC().Format(time.RFC3339)),
+					StopAt:          ptr.Of(time.Now().UTC().Format(time.RFC3339)),
+					Capacity:        ptr.Of(int32(0)),
+					Simulated:       ptr.Of(false),
+					EventsCount:     ptr.Of(int32(len(decisionsList))),
+					Leakspeed:       ptr.Of(""),
+					ScenarioHash:    ptr.Of(""),
+					ScenarioVersion: ptr.Of(""),
+					Decisions:       decisionsList,
+				}
+				alerts = append(alerts, &importAlert)
+			}
+
+			if len(decisionsList) > 1000 {
+				log.Infof("You are about to add %d decisions, this may take a while", len(decisionsList))
+			}
+
+			_, _, err = Client.Alerts.Add(context.Background(), alerts)
+			if err != nil {
+				log.Fatal(err)
+			}
+			log.Infof("%d decisions successfully imported", len(decisionsList))
+		},
+	}
+
+	cmdDecisionImport.Flags().SortFlags = false
+	cmdDecisionImport.Flags().StringVarP(&importFile, "input", "i", "", "Input file")
+	cmdDecisionImport.Flags().StringVarP(&importDuration, "duration", "d", "", "Decision duration (ie. 1h,4h,30m)")
+	cmdDecisionImport.Flags().StringVar(&importScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)")
+	cmdDecisionImport.Flags().StringVarP(&importReason, "reason", "R", "", "Decision reason (ie. scenario-name)")
+	cmdDecisionImport.Flags().StringVarP(&importType, "type", "t", "", "Decision type (ie. ban,captcha,throttle)")
+	cmdDecisionImport.Flags().IntVar(&batchSize, "batch", 0, "Split import in batches of N decisions")
+
+	return cmdDecisionImport
 }

cmd/crowdsec-cli/decisions_import.go

@@ -1,280 +0,0 @@
-package main
-
-import (
-	"bufio"
-	"bytes"
-	"context"
-	"encoding/json"
-	"errors"
-	"fmt"
-	"io"
-	"os"
-	"strings"
-	"time"
-
-	"github.com/jszwec/csvutil"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/go-cs-lib/ptr"
-	"github.com/crowdsecurity/go-cs-lib/slicetools"
-
-	"github.com/crowdsecurity/crowdsec/pkg/models"
-	"github.com/crowdsecurity/crowdsec/pkg/types"
-)
-
-// decisionRaw is only used to unmarshall json/csv decisions
-type decisionRaw struct {
-	Duration string `csv:"duration,omitempty" json:"duration,omitempty"`
-	Scenario string `csv:"reason,omitempty"   json:"reason,omitempty"`
-	Scope    string `csv:"scope,omitempty"    json:"scope,omitempty"`
-	Type     string `csv:"type,omitempty"     json:"type,omitempty"`
-	Value    string `csv:"value"              json:"value"`
-}
-
-func parseDecisionList(content []byte, format string) ([]decisionRaw, error) {
-	ret := []decisionRaw{}
-
-	switch format {
-	case "values":
-		log.Infof("Parsing values")
-
-		scanner := bufio.NewScanner(bytes.NewReader(content))
-		for scanner.Scan() {
-			value := strings.TrimSpace(scanner.Text())
-			ret = append(ret, decisionRaw{Value: value})
-		}
-
-		if err := scanner.Err(); err != nil {
-			return nil, fmt.Errorf("unable to parse values: '%s'", err)
-		}
-	case "json":
-		log.Infof("Parsing json")
-
-		if err := json.Unmarshal(content, &ret); err != nil {
-			return nil, err
-		}
-	case "csv":
-		log.Infof("Parsing csv")
-
-		if err := csvutil.Unmarshal(content, &ret); err != nil {
-			return nil, fmt.Errorf("unable to parse csv: '%s'", err)
-		}
-	default:
-		return nil, fmt.Errorf("invalid format '%s', expected one of 'json', 'csv', 'values'", format)
-	}
-
-	return ret, nil
-}
-
-
-func (cli *cliDecisions) runImport(cmd *cobra.Command, args []string) error  {
-	flags := cmd.Flags()
-
-	input, err := flags.GetString("input")
-	if err != nil {
-		return err
-	}
-
-	defaultDuration, err := flags.GetString("duration")
-	if err != nil {
-		return err
-	}
-
-	if defaultDuration == "" {
-		return errors.New("--duration cannot be empty")
-	}
-
-	defaultScope, err := flags.GetString("scope")
-	if err != nil {
-		return err
-	}
-
-	if defaultScope == "" {
-		return errors.New("--scope cannot be empty")
-	}
-
-	defaultReason, err := flags.GetString("reason")
-	if err != nil {
-		return err
-	}
-
-	if defaultReason == "" {
-		return errors.New("--reason cannot be empty")
-	}
-
-	defaultType, err := flags.GetString("type")
-	if err != nil {
-		return err
-	}
-
-	if defaultType == "" {
-		return errors.New("--type cannot be empty")
-	}
-
-	batchSize, err := flags.GetInt("batch")
-	if err != nil {
-		return err
-	}
-
-	format, err := flags.GetString("format")
-	if err != nil {
-		return err
-	}
-
-	var (
-		content []byte
-		fin	*os.File
-	)
-
-	// set format if the file has a json or csv extension
-	if format == "" {
-		if strings.HasSuffix(input, ".json") {
-			format = "json"
-		} else if strings.HasSuffix(input, ".csv") {
-			format = "csv"
-		}
-	}
-
-	if format == "" {
-		return errors.New("unable to guess format from file extension, please provide a format with --format flag")
-	}
-
-	if input == "-" {
-		fin = os.Stdin
-		input = "stdin"
-	} else {
-		fin, err = os.Open(input)
-		if err != nil {
-			return fmt.Errorf("unable to open %s: %s", input, err)
-		}
-	}
-
-	content, err = io.ReadAll(fin)
-	if err != nil {
-		return fmt.Errorf("unable to read from %s: %s", input, err)
-	}
-
-	decisionsListRaw, err := parseDecisionList(content, format)
-	if err != nil {
-		return err
-	}
-
-	decisions := make([]*models.Decision, len(decisionsListRaw))
-
-	for i, d := range decisionsListRaw {
-		if d.Value == "" {
-			return fmt.Errorf("item %d: missing 'value'", i)
-		}
-
-		if d.Duration == "" {
-			d.Duration = defaultDuration
-			log.Debugf("item %d: missing 'duration', using default '%s'", i, defaultDuration)
-		}
-
-		if d.Scenario == "" {
-			d.Scenario = defaultReason
-			log.Debugf("item %d: missing 'reason', using default '%s'", i, defaultReason)
-		}
-
-		if d.Type == "" {
-			d.Type = defaultType
-			log.Debugf("item %d: missing 'type', using default '%s'", i, defaultType)
-		}
-
-		if d.Scope == "" {
-			d.Scope = defaultScope
-			log.Debugf("item %d: missing 'scope', using default '%s'", i, defaultScope)
-		}
-
-		decisions[i] = &models.Decision{
-			Value:     ptr.Of(d.Value),
-			Duration:  ptr.Of(d.Duration),
-			Origin:    ptr.Of(types.CscliImportOrigin),
-			Scenario:  ptr.Of(d.Scenario),
-			Type:      ptr.Of(d.Type),
-			Scope:     ptr.Of(d.Scope),
-			Simulated: ptr.Of(false),
-		}
-	}
-
-	if len(decisions) > 1000 {
-		log.Infof("You are about to add %d decisions, this may take a while", len(decisions))
-	}
-
-	for _, chunk := range slicetools.Chunks(decisions, batchSize) {
-		log.Debugf("Processing chunk of %d decisions", len(chunk))
-		importAlert := models.Alert{
-			CreatedAt: time.Now().UTC().Format(time.RFC3339),
-			Scenario:  ptr.Of(fmt.Sprintf("import %s: %d IPs", input, len(chunk))),
-
-			Message: ptr.Of(""),
-			Events:  []*models.Event{},
-			Source: &models.Source{
-				Scope: ptr.Of(""),
-				Value: ptr.Of(""),
-			},
-			StartAt:         ptr.Of(time.Now().UTC().Format(time.RFC3339)),
-			StopAt:          ptr.Of(time.Now().UTC().Format(time.RFC3339)),
-			Capacity:        ptr.Of(int32(0)),
-			Simulated:       ptr.Of(false),
-			EventsCount:     ptr.Of(int32(len(chunk))),
-			Leakspeed:       ptr.Of(""),
-			ScenarioHash:    ptr.Of(""),
-			ScenarioVersion: ptr.Of(""),
-			Decisions:       chunk,
-		}
-
-		_, _, err = Client.Alerts.Add(context.Background(), models.AddAlertsRequest{&importAlert})
-		if err != nil {
-			return err
-		}
-	}
-
-	log.Infof("Imported %d decisions", len(decisions))
-
-	return nil
-}
-
-func (cli *cliDecisions) newImportCmd() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:   "import [options]",
-		Short: "Import decisions from a file or pipe",
-		Long: "expected format:\n" +
-			"csv  : any of duration,reason,scope,type,value, with a header line\n" +
-			"json :" + "`{" + `"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"` + "}`",
-		Args:	 cobra.NoArgs,
-		DisableAutoGenTag: true,
-		Example: `decisions.csv:
-duration,scope,value
-24h,ip,1.2.3.4
-
-$ cscli decisions import -i decisions.csv
-
-decisions.json:
-[{"duration" : "4h", "scope" : "ip", "type" : "ban", "value" : "1.2.3.4"}]
-
-The file format is detected from the extension, but can be forced with the --format option
-which is required when reading from standard input.
-
-Raw values, standard input:
-
-$ echo "1.2.3.4" | cscli decisions import -i - --format values
-`,
-		RunE: cli.runImport,
-	}
-
-	flags := cmd.Flags()
-	flags.SortFlags = false
-	flags.StringP("input", "i", "", "Input file")
-	flags.StringP("duration", "d", "4h", "Decision duration: 1h,4h,30m")
-	flags.String("scope", types.Ip, "Decision scope: ip,range,username")
-	flags.StringP("reason", "R", "manual", "Decision reason: <scenario-name>")
-	flags.StringP("type", "t", "ban", "Decision type: ban,captcha,throttle")
-	flags.Int("batch", 0, "Split import in batches of N decisions")
-	flags.String("format", "", "Input format: 'json', 'csv' or 'values' (each line is a value, no headers)")
-
-	cmd.MarkFlagRequired("input")
-
-	return cmd
-}

cmd/crowdsec-cli/decisions_table.go

@@ -8,15 +8,13 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/models"
 )
 
-func (cli *cliDecisions) decisionsTable(out io.Writer, alerts *models.GetAlertsResponse, printMachine bool) {
+func decisionsTable(out io.Writer, alerts *models.GetAlertsResponse, printMachine bool) {
 	t := newTable(out)
 	t.SetRowLines(false)
-
 	header := []string{"ID", "Source", "Scope:Value", "Reason", "Action", "Country", "AS", "Events", "expiration", "Alert ID"}
 	if printMachine {
 		header = append(header, "Machine")
 	}
-
 	t.SetHeaders(header...)
 
 	for _, alertItem := range *alerts {
@@ -24,7 +22,6 @@ func (cli *cliDecisions) decisionsTable(out io.Writer, alerts *models.GetAlertsR
 			if *alertItem.Simulated {
 				*decisionItem.Type = fmt.Sprintf("(simul)%s", *decisionItem.Type)
 			}
-
 			row := []string{
 				strconv.Itoa(int(decisionItem.ID)),
 				*decisionItem.Origin,
@@ -45,6 +42,5 @@ func (cli *cliDecisions) decisionsTable(out io.Writer, alerts *models.GetAlertsR
 			t.AddRow(row...)
 		}
 	}
-
 	t.Render()
 }

cmd/crowdsec-cli/doc.go

@@ -1,51 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"path/filepath"
-	"strings"
-
-	"github.com/spf13/cobra"
-	"github.com/spf13/cobra/doc"
-)
-
-type cliDoc struct{}
-
-func NewCLIDoc() *cliDoc {
-	return &cliDoc{}
-}
-
-func (cli cliDoc) NewCommand(rootCmd *cobra.Command) *cobra.Command {
-	cmd := &cobra.Command{
-		Use:               "doc",
-		Short:             "Generate the documentation in `./doc/`. Directory must exist.",
-		Args:              cobra.ExactArgs(0),
-		Hidden:            true,
-		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			if err := doc.GenMarkdownTreeCustom(rootCmd, "./doc/", cli.filePrepender, cli.linkHandler); err != nil {
-				return fmt.Errorf("failed to generate cobra doc: %s", err)
-			}
-			return nil
-		},
-	}
-
-	return cmd
-}
-
-func (cli cliDoc) filePrepender(filename string) string {
-	const header = `---
-id: %s
-title: %s
----
-`
-
-	name := filepath.Base(filename)
-	base := strings.TrimSuffix(name, filepath.Ext(name))
-
-	return fmt.Sprintf(header, base, strings.ReplaceAll(base, "_", " "))
-}
-
-func (cli cliDoc) linkHandler(name string) string {
-	return fmt.Sprintf("/cscli/%s", name)
-}

cmd/crowdsec-cli/explain.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"bufio"
-	"errors"
 	"fmt"
 	"io"
 	"os"
@@ -12,115 +11,67 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	"github.com/crowdsecurity/crowdsec/pkg/dumps"
 	"github.com/crowdsecurity/crowdsec/pkg/hubtest"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
-func getLineCountForFile(filepath string) (int, error) {
-	f, err := os.Open(filepath)
-	if err != nil {
-		return 0, err
-	}
-	defer f.Close()
-
-	lc := 0
-	fs := bufio.NewReader(f)
-
-	for {
-		input, err := fs.ReadBytes('\n')
-		if len(input) > 1 {
-			lc++
-		}
-
-		if err != nil && err == io.EOF {
-			break
-		}
-	}
-
-	return lc, nil
-}
-
-type cliExplain struct {
-	cfg   configGetter
-	flags struct {
-		logFile               string
-		dsn                   string
-		logLine               string
-		logType               string
-		details               bool
-		skipOk                bool
-		onlySuccessfulParsers bool
-		noClean               bool
-		crowdsec              string
-		labels                string
-	}
-}
-
-func NewCLIExplain(cfg configGetter) *cliExplain {
-	return &cliExplain{
-		cfg: cfg,
-	}
-}
-
-func (cli *cliExplain) NewCommand() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:   "explain",
-		Short: "Explain log pipeline",
-		Long: `
-Explain log pipeline 
-		`,
-		Example: `
-cscli explain --file ./myfile.log --type nginx 
-cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog
-cscli explain --dsn "file://myfile.log" --type nginx
-tail -n 5 myfile.log | cscli explain --type nginx -f -
-		`,
-		Args:              cobra.ExactArgs(0),
-		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			return cli.run()
-		},
-		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
-			fileInfo, _ := os.Stdin.Stat()
-			if cli.flags.logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) {
-				return errors.New("the option -f - is intended to work with pipes")
-			}
-
-			return nil
-		},
-	}
-
+func runExplain(cmd *cobra.Command, args []string) error {
 	flags := cmd.Flags()
 
-	flags.StringVarP(&cli.flags.logFile, "file", "f", "", "Log file to test")
-	flags.StringVarP(&cli.flags.dsn, "dsn", "d", "", "DSN to test")
-	flags.StringVarP(&cli.flags.logLine, "log", "l", "", "Log line to test")
-	flags.StringVarP(&cli.flags.logType, "type", "t", "", "Type of the acquisition to test")
-	flags.StringVar(&cli.flags.labels, "labels", "", "Additional labels to add to the acquisition format (key:value,key2:value2)")
-	flags.BoolVarP(&cli.flags.details, "verbose", "v", false, "Display individual changes")
-	flags.BoolVar(&cli.flags.skipOk, "failures", false, "Only show failed lines")
-	flags.BoolVar(&cli.flags.onlySuccessfulParsers, "only-successful-parsers", false, "Only show successful parsers")
-	flags.StringVar(&cli.flags.crowdsec, "crowdsec", "crowdsec", "Path to crowdsec")
-	flags.BoolVar(&cli.flags.noClean, "no-clean", false, "Don't clean runtime environment after tests")
+	logFile, err := flags.GetString("file")
+	if err != nil {
+		return err
+	}
 
-	cmd.MarkFlagRequired("type")
-	cmd.MarkFlagsOneRequired("log", "file", "dsn")
+	dsn, err := flags.GetString("dsn")
+	if err != nil {
+		return err
+	}
 
-	return cmd
-}
+	logLine, err := flags.GetString("log")
+	if err != nil {
+		return err
+	}
 
-func (cli *cliExplain) run() error {
-	logFile := cli.flags.logFile
-	logLine := cli.flags.logLine
-	logType := cli.flags.logType
-	dsn := cli.flags.dsn
-	labels := cli.flags.labels
-	crowdsec := cli.flags.crowdsec
+	logType, err := flags.GetString("type")
+	if err != nil {
+		return err
+	}
 
-	opts := dumps.DumpOpts{
-		Details:          cli.flags.details,
-		SkipOk:           cli.flags.skipOk,
-		ShowNotOkParsers: !cli.flags.onlySuccessfulParsers,
+	opts := hubtest.DumpOpts{}
+
+	opts.Details, err = flags.GetBool("verbose")
+	if err != nil {
+		return err
+	}
+
+	opts.SkipOk, err = flags.GetBool("failures")
+	if err != nil {
+		return err
+	}
+
+	opts.ShowNotOkParsers, err = flags.GetBool("only-successful-parsers")
+	opts.ShowNotOkParsers = !opts.ShowNotOkParsers
+	if err != nil {
+		return err
+	}
+
+	crowdsec, err := flags.GetString("crowdsec")
+	if err != nil {
+		return err
+	}
+
+	fileInfo, _ := os.Stdin.Stat()
+
+	if logType == "" || (logLine == "" && logFile == "" && dsn == "") {
+		printHelp(cmd)
+		fmt.Println()
+		fmt.Printf("Please provide --type flag\n")
+		os.Exit(1)
+	}
+
+	if logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) {
+		return fmt.Errorf("the option -f - is intended to work with pipes")
 	}
 
 	var f *os.File
@@ -128,25 +79,12 @@ func (cli *cliExplain) run() error {
 	// using empty string fallback to /tmp
 	dir, err := os.MkdirTemp("", "cscli_explain")
 	if err != nil {
-		return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %w", err)
+		return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %s", err)
 	}
-
-	defer func() {
-		if cli.flags.noClean {
-			return
-		}
-
-		if _, err := os.Stat(dir); !os.IsNotExist(err) {
-			if err := os.RemoveAll(dir); err != nil {
-				log.Errorf("unable to delete temporary directory '%s': %s", dir, err)
-			}
-		}
-	}()
-
+	tmpFile := ""
 	// we create a  temporary log file if a log line/stdin has been provided
 	if logLine != "" || logFile == "-" {
-		tmpFile := filepath.Join(dir, "cscli_test_tmp.log")
-
+		tmpFile = filepath.Join(dir, "cscli_test_tmp.log")
 		f, err = os.Create(tmpFile)
 		if err != nil {
 			return err
@@ -160,27 +98,20 @@ func (cli *cliExplain) run() error {
 		} else if logFile == "-" {
 			reader := bufio.NewReader(os.Stdin)
 			errCount := 0
-
 			for {
 				input, err := reader.ReadBytes('\n')
-				if err != nil && errors.Is(err, io.EOF) {
+				if err != nil && err == io.EOF {
 					break
 				}
-
-				if len(input) > 1 {
-					_, err = f.Write(input)
-				}
-
-				if err != nil || len(input) <= 1 {
+				_, err = f.Write(input)
+				if err != nil {
 					errCount++
 				}
 			}
-
 			if errCount > 0 {
-				log.Warnf("Failed to write %d lines to %s", errCount, tmpFile)
+				log.Warnf("Failed to write %d lines to tmp file", errCount)
 			}
 		}
-
 		f.Close()
 		// this is the file that was going to be read by crowdsec anyway
 		logFile = tmpFile
@@ -191,59 +122,81 @@ func (cli *cliExplain) run() error {
 		if err != nil {
 			return fmt.Errorf("unable to get absolute path of '%s', exiting", logFile)
 		}
-
 		dsn = fmt.Sprintf("file://%s", absolutePath)
-
-		lineCount, err := getLineCountForFile(absolutePath)
-		if err != nil {
-			return err
-		}
-
-		log.Debugf("file %s has %d lines", absolutePath, lineCount)
-
-		if lineCount == 0 {
-			return fmt.Errorf("the log file is empty: %s", absolutePath)
-		}
-
+		lineCount := types.GetLineCountForFile(absolutePath)
 		if lineCount > 100 {
-			log.Warnf("%s contains %d lines. This may take a lot of resources.", absolutePath, lineCount)
+			log.Warnf("log file contains %d lines. This may take lot of resources.", lineCount)
 		}
 	}
 
 	if dsn == "" {
-		return errors.New("no acquisition (--file or --dsn) provided, can't run cscli test")
+		return fmt.Errorf("no acquisition (--file or --dsn) provided, can't run cscli test")
 	}
 
 	cmdArgs := []string{"-c", ConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", dir, "-no-api"}
-
-	if labels != "" {
-		log.Debugf("adding labels %s", labels)
-		cmdArgs = append(cmdArgs, "-label", labels)
-	}
-
 	crowdsecCmd := exec.Command(crowdsec, cmdArgs...)
-
 	output, err := crowdsecCmd.CombinedOutput()
 	if err != nil {
 		fmt.Println(string(output))
-
-		return fmt.Errorf("fail to run crowdsec for test: %w", err)
+		return fmt.Errorf("fail to run crowdsec for test: %v", err)
 	}
 
+	// rm the temporary log file if only a log line/stdin was provided
+	if tmpFile != "" {
+		if err := os.Remove(tmpFile); err != nil {
+			return fmt.Errorf("unable to remove tmp log file '%s': %+v", tmpFile, err)
+		}
+	}
 	parserDumpFile := filepath.Join(dir, hubtest.ParserResultFileName)
 	bucketStateDumpFile := filepath.Join(dir, hubtest.BucketPourResultFileName)
 
-	parserDump, err := dumps.LoadParserDump(parserDumpFile)
+	parserDump, err := hubtest.LoadParserDump(parserDumpFile)
 	if err != nil {
-		return fmt.Errorf("unable to load parser dump result: %w", err)
+		return fmt.Errorf("unable to load parser dump result: %s", err)
 	}
 
-	bucketStateDump, err := dumps.LoadBucketPourDump(bucketStateDumpFile)
+	bucketStateDump, err := hubtest.LoadBucketPourDump(bucketStateDumpFile)
 	if err != nil {
-		return fmt.Errorf("unable to load bucket dump result: %w", err)
+		return fmt.Errorf("unable to load bucket dump result: %s", err)
 	}
 
-	dumps.DumpTree(*parserDump, *bucketStateDump, opts)
+	hubtest.DumpTree(*parserDump, *bucketStateDump, opts)
+
+	if err := os.RemoveAll(dir); err != nil {
+		return fmt.Errorf("unable to delete temporary directory '%s': %s", dir, err)
+	}
 
 	return nil
 }
+
+func NewExplainCmd() *cobra.Command {
+	cmdExplain := &cobra.Command{
+		Use:   "explain",
+		Short: "Explain log pipeline",
+		Long: `
+Explain log pipeline 
+		`,
+		Example: `
+cscli explain --file ./myfile.log --type nginx 
+cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog
+cscli explain --dsn "file://myfile.log" --type nginx
+tail -n 5 myfile.log | cscli explain --type nginx -f -
+		`,
+		Args:              cobra.ExactArgs(0),
+		DisableAutoGenTag: true,
+		RunE:              runExplain,
+	}
+
+	flags := cmdExplain.Flags()
+
+	flags.StringP("file", "f", "", "Log file to test")
+	flags.StringP("dsn", "d", "", "DSN to test")
+	flags.StringP("log", "l", "", "Log line to test")
+	flags.StringP("type", "t", "", "Type of the acquisition to test")
+	flags.BoolP("verbose", "v", false, "Display individual changes")
+	flags.Bool("failures", false, "Only show failed lines")
+	flags.Bool("only-successful-parsers", false, "Only show successful parsers")
+	flags.String("crowdsec", "crowdsec", "Path to crowdsec")
+
+	return cmdExplain
+}

cmd/crowdsec-cli/flag.go

@@ -1,29 +0,0 @@
-package main
-
-// Custom types for flag validation and conversion.
-
-import (
-	"errors"
-)
-
-type MachinePassword string
-
-func (p *MachinePassword) String() string {
-    return string(*p)
-}
-
-func (p *MachinePassword) Set(v string) error {
-	// a password can't be more than 72 characters
-	// due to bcrypt limitations
-        if len(v) > 72 {
-                return errors.New("password too long (max 72 characters)")
-        }
-
-        *p = MachinePassword(v)
-
-        return nil
-}
-
-func (p *MachinePassword) Type() string {
-    return "string"
-}

cmd/crowdsec-cli/hub.go

@@ -1,228 +1,164 @@
 package main
 
 import (
-	"encoding/json"
+	"errors"
 	"fmt"
 
 	"github.com/fatih/color"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v3"
 
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-type cliHub struct{
-	cfg configGetter
-}
-
-func NewCLIHub(cfg configGetter) *cliHub {
-	return &cliHub{
-		cfg: cfg,
-	}
-}
-
-func (cli *cliHub) NewCommand() *cobra.Command {
-	cmd := &cobra.Command{
+func NewHubCmd() *cobra.Command {
+	var cmdHub = &cobra.Command{
 		Use:   "hub [action]",
-		Short: "Manage hub index",
-		Long: `Hub management
+		Short: "Manage Hub",
+		Long: `
+Hub management
 
 List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net).
-The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.`,
-		Example: `cscli hub list
-cscli hub update
-cscli hub upgrade`,
+The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.
+		`,
+		Example: `
+cscli hub list   # List all installed configurations
+cscli hub update # Download list of available configurations from the hub
+		`,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-	}
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if csConfig.Cscli == nil {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
 
-	cmd.AddCommand(cli.newListCmd())
-	cmd.AddCommand(cli.newUpdateCmd())
-	cmd.AddCommand(cli.newUpgradeCmd())
-	cmd.AddCommand(cli.newTypesCmd())
-
-	return cmd
-}
-
-func (cli *cliHub) list(all bool) error {
-	hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger())
-	if err != nil {
-		return err
-	}
-
-	for _, v := range hub.Warnings {
-		log.Info(v)
-	}
-
-	for _, line := range hub.ItemStats() {
-		log.Info(line)
-	}
-
-	items := make(map[string][]*cwhub.Item)
-
-	for _, itemType := range cwhub.ItemTypes {
-		items[itemType], err = selectItems(hub, itemType, nil, !all)
-		if err != nil {
-			return err
-		}
-	}
-
-	err = listItems(color.Output, cwhub.ItemTypes, items, true)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (cli *cliHub) newListCmd() *cobra.Command {
-	var all bool
-
-	cmd := &cobra.Command{
-		Use:               "list [-a]",
-		Short:             "List all installed configurations",
-		Args:              cobra.ExactArgs(0),
-		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			return cli.list(all)
+			return nil
 		},
 	}
+	cmdHub.PersistentFlags().StringVarP(&cwhub.HubBranch, "branch", "b", "", "Use given branch from hub")
 
-	flags := cmd.Flags()
-	flags.BoolVarP(&all, "all", "a", false, "List disabled items as well")
+	cmdHub.AddCommand(NewHubListCmd())
+	cmdHub.AddCommand(NewHubUpdateCmd())
+	cmdHub.AddCommand(NewHubUpgradeCmd())
 
-	return cmd
+	return cmdHub
 }
 
-func (cli *cliHub) update() error {
-	local := cli.cfg().Hub
-	remote := require.RemoteHub(cli.cfg())
+func NewHubListCmd() *cobra.Command {
+	var cmdHubList = &cobra.Command{
+		Use:               "list [-a]",
+		Short:             "List installed configs",
+		Args:              cobra.ExactArgs(0),
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
 
-	// don't use require.Hub because if there is no index file, it would fail
-	hub, err := cwhub.NewHub(local, remote, true, log.StandardLogger())
-	if err != nil {
-		return fmt.Errorf("failed to update hub: %w", err)
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatal(err)
+			}
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+				log.Info("Run 'sudo cscli hub update' to get the hub index")
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			//use LocalSync to get warnings about tainted / outdated items
+			_, warn := cwhub.LocalSync(csConfig.Hub)
+			for _, v := range warn {
+				log.Info(v)
+			}
+			cwhub.DisplaySummary()
+			ListItems(color.Output, []string{
+				cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW,
+			}, args, true, false, all)
+		},
 	}
+	cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
 
-	for _, v := range hub.Warnings {
-		log.Info(v)
-	}
-
-	return nil
+	return cmdHubList
 }
 
-func (cli *cliHub) newUpdateCmd() *cobra.Command {
-	cmd := &cobra.Command{
+func NewHubUpdateCmd() *cobra.Command {
+	var cmdHubUpdate = &cobra.Command{
 		Use:   "update",
-		Short: "Download the latest index (catalog of available configurations)",
+		Short: "Fetch available configs from hub",
 		Long: `
-Fetches the .index.json file from the hub, containing the list of available configs.
+Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs.
 `,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			return cli.update()
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if csConfig.Cscli == nil {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+
+			if err := cwhub.SetHubBranch(); err != nil {
+				return fmt.Errorf("error while setting hub branch: %s", err)
+			}
+			return nil
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatal(err)
+			}
+			if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
+				if errors.Is(err, cwhub.ErrIndexNotFound) {
+					log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch)
+					cwhub.HubBranch = "master"
+					if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
+						log.Fatalf("Failed to get Hub index after retry : %v", err)
+					}
+				} else {
+					log.Fatalf("Failed to get Hub index : %v", err)
+				}
+			}
+			//use LocalSync to get warnings about tainted / outdated items
+			_, warn := cwhub.LocalSync(csConfig.Hub)
+			for _, v := range warn {
+				log.Info(v)
+			}
 		},
 	}
 
-	return cmd
+	return cmdHubUpdate
 }
 
-func (cli *cliHub) upgrade(force bool) error {
-	hub, err := require.Hub(cli.cfg(), require.RemoteHub(cli.cfg()), log.StandardLogger())
-	if err != nil {
-		return err
-	}
-
-	for _, itemType := range cwhub.ItemTypes {
-		items, err := hub.GetInstalledItemsByType(itemType)
-		if err != nil {
-			return err
-		}
-
-		updated := 0
-
-		log.Infof("Upgrading %s", itemType)
-
-		for _, item := range items {
-			didUpdate, err := item.Upgrade(force)
-			if err != nil {
-				return err
-			}
-
-			if didUpdate {
-				updated++
-			}
-		}
-
-		log.Infof("Upgraded %d %s", updated, itemType)
-	}
-
-	return nil
-}
-
-func (cli *cliHub) newUpgradeCmd() *cobra.Command {
-	var force bool
-
-	cmd := &cobra.Command{
+func NewHubUpgradeCmd() *cobra.Command {
+	var cmdHubUpgrade = &cobra.Command{
 		Use:   "upgrade",
-		Short: "Upgrade all configurations to their latest version",
+		Short: "Upgrade all configs installed from hub",
 		Long: `
 Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available.
 `,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			return cli.upgrade(force)
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if csConfig.Cscli == nil {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+
+			if err := cwhub.SetHubBranch(); err != nil {
+				return fmt.Errorf("error while setting hub branch: %s", err)
+			}
+			return nil
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatal(err)
+			}
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+				log.Info("Run 'sudo cscli hub update' to get the hub index")
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+
+			log.Infof("Upgrading collections")
+			cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
+			log.Infof("Upgrading parsers")
+			cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
+			log.Infof("Upgrading scenarios")
+			cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
+			log.Infof("Upgrading postoverflows")
+			cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
 		},
 	}
+	cmdHubUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
 
-	flags := cmd.Flags()
-	flags.BoolVar(&force, "force", false, "Force upgrade: overwrite tainted and outdated files")
-
-	return cmd
-}
-
-func (cli *cliHub) types() error {
-	switch cli.cfg().Cscli.Output {
-	case "human":
-		s, err := yaml.Marshal(cwhub.ItemTypes)
-		if err != nil {
-			return err
-		}
-
-		fmt.Print(string(s))
-	case "json":
-		jsonStr, err := json.Marshal(cwhub.ItemTypes)
-		if err != nil {
-			return err
-		}
-
-		fmt.Println(string(jsonStr))
-	case "raw":
-		for _, itemType := range cwhub.ItemTypes {
-			fmt.Println(itemType)
-		}
-	}
-
-	return nil
-}
-
-func (cli *cliHub) newTypesCmd() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:   "types",
-		Short: "List supported item types",
-		Long: `
-List the types of supported hub items.
-`,
-		Args:              cobra.ExactArgs(0),
-		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			return cli.types()
-		},
-	}
-
-	return cmd
+	return cmdHubUpgrade
 }

cmd/crowdsec-cli/hubappsec.go

@@ -1,123 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"os"
-
-	"golang.org/x/text/cases"
-	"golang.org/x/text/language"
-	"gopkg.in/yaml.v3"
-
-	"github.com/crowdsecurity/crowdsec/pkg/appsec"
-	"github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewCLIAppsecConfig(cfg configGetter) *cliItem {
-	return &cliItem{
-		cfg:       cfg,
-		name:      cwhub.APPSEC_CONFIGS,
-		singular:  "appsec-config",
-		oneOrMore: "appsec-config(s)",
-		help: cliHelp{
-			example: `cscli appsec-configs list -a
-cscli appsec-configs install crowdsecurity/vpatch
-cscli appsec-configs inspect crowdsecurity/vpatch
-cscli appsec-configs upgrade crowdsecurity/vpatch
-cscli appsec-configs remove crowdsecurity/vpatch
-`,
-		},
-		installHelp: cliHelp{
-			example: `cscli appsec-configs install crowdsecurity/vpatch`,
-		},
-		removeHelp: cliHelp{
-			example: `cscli appsec-configs remove crowdsecurity/vpatch`,
-		},
-		upgradeHelp: cliHelp{
-			example: `cscli appsec-configs upgrade crowdsecurity/vpatch`,
-		},
-		inspectHelp: cliHelp{
-			example: `cscli appsec-configs inspect crowdsecurity/vpatch`,
-		},
-		listHelp: cliHelp{
-			example: `cscli appsec-configs list
-cscli appsec-configs list -a
-cscli appsec-configs list crowdsecurity/vpatch`,
-		},
-	}
-}
-
-func NewCLIAppsecRule(cfg configGetter) *cliItem {
-	inspectDetail := func(item *cwhub.Item) error {
-		// Only show the converted rules in human mode
-		if csConfig.Cscli.Output != "human" {
-			return nil
-		}
-
-		appsecRule := appsec.AppsecCollectionConfig{}
-
-		yamlContent, err := os.ReadFile(item.State.LocalPath)
-		if err != nil {
-			return fmt.Errorf("unable to read file %s: %w", item.State.LocalPath, err)
-		}
-
-		if err := yaml.Unmarshal(yamlContent, &appsecRule); err != nil {
-			return fmt.Errorf("unable to unmarshal yaml file %s: %w", item.State.LocalPath, err)
-		}
-
-		for _, ruleType := range appsec_rule.SupportedTypes() {
-			fmt.Printf("\n%s format:\n", cases.Title(language.Und, cases.NoLower).String(ruleType))
-
-			for _, rule := range appsecRule.Rules {
-				convertedRule, _, err := rule.Convert(ruleType, appsecRule.Name)
-				if err != nil {
-					return fmt.Errorf("unable to convert rule %s: %w", rule.Name, err)
-				}
-
-				fmt.Println(convertedRule)
-			}
-
-			switch ruleType { //nolint:gocritic
-			case appsec_rule.ModsecurityRuleType:
-				for _, rule := range appsecRule.SecLangRules {
-					fmt.Println(rule)
-				}
-			}
-		}
-
-		return nil
-	}
-
-	return &cliItem{
-		cfg:       cfg,
-		name:      "appsec-rules",
-		singular:  "appsec-rule",
-		oneOrMore: "appsec-rule(s)",
-		help: cliHelp{
-			example: `cscli appsec-rules list -a
-cscli appsec-rules install crowdsecurity/crs
-cscli appsec-rules inspect crowdsecurity/crs
-cscli appsec-rules upgrade crowdsecurity/crs
-cscli appsec-rules remove crowdsecurity/crs
-`,
-		},
-		installHelp: cliHelp{
-			example: `cscli appsec-rules install crowdsecurity/crs`,
-		},
-		removeHelp: cliHelp{
-			example: `cscli appsec-rules remove crowdsecurity/crs`,
-		},
-		upgradeHelp: cliHelp{
-			example: `cscli appsec-rules upgrade crowdsecurity/crs`,
-		},
-		inspectHelp: cliHelp{
-			example: `cscli appsec-rules inspect crowdsecurity/crs`,
-		},
-		inspectDetail: inspectDetail,
-		listHelp: cliHelp{
-			example: `cscli appsec-rules list
-cscli appsec-rules list -a
-cscli appsec-rules list crowdsecurity/crs`,
-		},
-	}
-}

cmd/crowdsec-cli/hubcollection.go

@@ -1,41 +0,0 @@
-package main
-
-import (
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewCLICollection(cfg configGetter) *cliItem {
-	return &cliItem{
-		cfg:       cfg,
-		name:      cwhub.COLLECTIONS,
-		singular:  "collection",
-		oneOrMore: "collection(s)",
-		help: cliHelp{
-			example: `cscli collections list -a
-cscli collections install crowdsecurity/http-cve crowdsecurity/iptables
-cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables
-cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables
-cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
-`,
-		},
-		installHelp: cliHelp{
-			example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`,
-		},
-		removeHelp: cliHelp{
-			example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`,
-		},
-		upgradeHelp: cliHelp{
-			example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`,
-		},
-		inspectHelp: cliHelp{
-			example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`,
-		},
-		listHelp: cliHelp{
-			example: `cscli collections list
-cscli collections list -a
-cscli collections list crowdsecurity/http-cve crowdsecurity/iptables
-
-List only enabled collections unless "-a" or names are specified.`,
-		},
-	}
-}

cmd/crowdsec-cli/hubcontext.go

@@ -1,41 +0,0 @@
-package main
-
-import (
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewCLIContext(cfg configGetter) *cliItem {
-	return &cliItem{
-		cfg:       cfg,
-		name:      cwhub.CONTEXTS,
-		singular:  "context",
-		oneOrMore: "context(s)",
-		help: cliHelp{
-			example: `cscli contexts list -a
-cscli contexts install crowdsecurity/yyy crowdsecurity/zzz
-cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz
-cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz
-cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz
-`,
-		},
-		installHelp: cliHelp{
-			example: `cscli contexts install crowdsecurity/yyy crowdsecurity/zzz`,
-		},
-		removeHelp: cliHelp{
-			example: `cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz`,
-		},
-		upgradeHelp: cliHelp{
-			example: `cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz`,
-		},
-		inspectHelp: cliHelp{
-			example: `cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz`,
-		},
-		listHelp: cliHelp{
-			example: `cscli contexts list
-cscli contexts list -a
-cscli contexts list crowdsecurity/yyy crowdsecurity/zzz
-
-List only enabled contexts unless "-a" or names are specified.`,
-		},
-	}
-}

cmd/crowdsec-cli/hubparser.go

@@ -1,41 +0,0 @@
-package main
-
-import (
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewCLIParser(cfg configGetter) *cliItem {
-	return &cliItem{
-		cfg:       cfg,
-		name:      cwhub.PARSERS,
-		singular:  "parser",
-		oneOrMore: "parser(s)",
-		help: cliHelp{
-			example: `cscli parsers list -a
-cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs
-cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs
-cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs
-cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
-`,
-		},
-		installHelp: cliHelp{
-			example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
-		},
-		removeHelp: cliHelp{
-			example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
-		},
-		upgradeHelp: cliHelp{
-			example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
-		},
-		inspectHelp: cliHelp{
-			example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`,
-		},
-		listHelp: cliHelp{
-			example: `cscli parsers list
-cscli parsers list -a
-cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs
-
-List only enabled parsers unless "-a" or names are specified.`,
-		},
-	}
-}

cmd/crowdsec-cli/hubpostoverflow.go

@@ -1,41 +0,0 @@
-package main
-
-import (
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewCLIPostOverflow(cfg configGetter) *cliItem {
-	return &cliItem{
-		cfg:       cfg,
-		name:      cwhub.POSTOVERFLOWS,
-		singular:  "postoverflow",
-		oneOrMore: "postoverflow(s)",
-		help: cliHelp{
-			example: `cscli postoverflows list -a
-cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns
-cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns
-cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns
-cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
-`,
-		},
-		installHelp: cliHelp{
-			example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
-		},
-		removeHelp: cliHelp{
-			example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
-		},
-		upgradeHelp: cliHelp{
-			example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
-		},
-		inspectHelp: cliHelp{
-			example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
-		},
-		listHelp: cliHelp{
-			example: `cscli postoverflows list
-cscli postoverflows list -a
-cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns
-
-List only enabled postoverflows unless "-a" or names are specified.`,
-		},
-	}
-}

cmd/crowdsec-cli/hubscenario.go

@@ -1,41 +0,0 @@
-package main
-
-import (
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func NewCLIScenario(cfg configGetter) *cliItem {
-	return &cliItem{
-		cfg:       cfg,
-		name:      cwhub.SCENARIOS,
-		singular:  "scenario",
-		oneOrMore: "scenario(s)",
-		help: cliHelp{
-			example: `cscli scenarios list -a
-cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing
-cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing
-cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing
-cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
-`,
-		},
-		installHelp: cliHelp{
-			example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`,
-		},
-		removeHelp: cliHelp{
-			example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`,
-		},
-		upgradeHelp: cliHelp{
-			example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`,
-		},
-		inspectHelp: cliHelp{
-			example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`,
-		},
-		listHelp: cliHelp{
-			example: `cscli scenarios list
-cscli scenarios list -a
-cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing
-
-List only enabled scenarios unless "-a" or names are specified.`,
-		},
-	}
-}

cmd/crowdsec-cli/hubtest.go

@@ -2,106 +2,73 @@ package main
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"math"
 	"os"
 	"path/filepath"
 	"strings"
-	"text/template"
 
 	"github.com/AlecAivazis/survey/v2"
+	"github.com/enescakir/emoji"
 	"github.com/fatih/color"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v3"
+	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/crowdsec/pkg/dumps"
-	"github.com/crowdsecurity/crowdsec/pkg/emoji"
 	"github.com/crowdsecurity/crowdsec/pkg/hubtest"
 )
 
 var (
-	HubTest        hubtest.HubTest
-	HubAppsecTests hubtest.HubTest
-	hubPtr         *hubtest.HubTest
-	isAppsecTest   bool
+	HubTest hubtest.HubTest
 )
 
-type cliHubTest struct {
-	cfg configGetter
-}
+func NewHubTestCmd() *cobra.Command {
+	var hubPath string
+	var crowdsecPath string
+	var cscliPath string
 
-func NewCLIHubTest(cfg configGetter) *cliHubTest {
-	return &cliHubTest{
-		cfg: cfg,
-	}
-}
-
-func (cli *cliHubTest) NewCommand() *cobra.Command {
-	var (
-		hubPath      string
-		crowdsecPath string
-		cscliPath    string
-	)
-
-	cmd := &cobra.Command{
+	var cmdHubTest = &cobra.Command{
 		Use:               "hubtest",
 		Short:             "Run functional tests on hub configurations",
 		Long:              "Run functional tests on hub configurations (parsers, scenarios, collections...)",
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			var err error
-			HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, false)
+			HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath)
 			if err != nil {
 				return fmt.Errorf("unable to load hubtest: %+v", err)
 			}
 
-			HubAppsecTests, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, true)
-			if err != nil {
-				return fmt.Errorf("unable to load appsec specific hubtest: %+v", err)
-			}
-
-			// commands will use the hubPtr, will point to the default hubTest object, or the one dedicated to appsec tests
-			hubPtr = &HubTest
-			if isAppsecTest {
-				hubPtr = &HubAppsecTests
-			}
-
 			return nil
 		},
 	}
+	cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
+	cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
+	cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
 
-	cmd.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
-	cmd.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
-	cmd.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
-	cmd.PersistentFlags().BoolVar(&isAppsecTest, "appsec", false, "Command relates to appsec tests")
+	cmdHubTest.AddCommand(NewHubTestCreateCmd())
+	cmdHubTest.AddCommand(NewHubTestRunCmd())
+	cmdHubTest.AddCommand(NewHubTestCleanCmd())
+	cmdHubTest.AddCommand(NewHubTestInfoCmd())
+	cmdHubTest.AddCommand(NewHubTestListCmd())
+	cmdHubTest.AddCommand(NewHubTestCoverageCmd())
+	cmdHubTest.AddCommand(NewHubTestEvalCmd())
+	cmdHubTest.AddCommand(NewHubTestExplainCmd())
 
-	cmd.AddCommand(cli.NewCreateCmd())
-	cmd.AddCommand(cli.NewRunCmd())
-	cmd.AddCommand(cli.NewCleanCmd())
-	cmd.AddCommand(cli.NewInfoCmd())
-	cmd.AddCommand(cli.NewListCmd())
-	cmd.AddCommand(cli.NewCoverageCmd())
-	cmd.AddCommand(cli.NewEvalCmd())
-	cmd.AddCommand(cli.NewExplainCmd())
-
-	return cmd
+	return cmdHubTest
 }
 
-func (cli *cliHubTest) NewCreateCmd() *cobra.Command {
-	var (
-		ignoreParsers bool
-		labels        map[string]string
-		logType       string
-	)
 
+func NewHubTestCreateCmd() *cobra.Command {
 	parsers := []string{}
 	postoverflows := []string{}
 	scenarios := []string{}
+	var ignoreParsers bool
+	var labels map[string]string
+	var logType string
 
-	cmd := &cobra.Command{
+	var cmdHubTestCreate = &cobra.Command{
 		Use:   "create",
 		Short: "create [test_name]",
 		Example: `cscli hubtest create my-awesome-test --type syslog
@@ -109,170 +76,134 @@ cscli hubtest create my-nginx-custom-test --type nginx
 cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios crowdsecurity/http-probing`,
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			testName := args[0]
-			testPath := filepath.Join(hubPtr.HubTestPath, testName)
+			testPath := filepath.Join(HubTest.HubTestPath, testName)
 			if _, err := os.Stat(testPath); os.IsExist(err) {
 				return fmt.Errorf("test '%s' already exists in '%s', exiting", testName, testPath)
 			}
 
-			if isAppsecTest {
-				logType = "appsec"
-			}
-
 			if logType == "" {
-				return errors.New("please provide a type (--type) for the test")
+				return fmt.Errorf("please provide a type (--type) for the test")
 			}
 
 			if err := os.MkdirAll(testPath, os.ModePerm); err != nil {
 				return fmt.Errorf("unable to create folder '%s': %+v", testPath, err)
 			}
 
-			configFilePath := filepath.Join(testPath, "config.yaml")
+			// create empty log file
+			logFileName := fmt.Sprintf("%s.log", testName)
+			logFilePath := filepath.Join(testPath, logFileName)
+			logFile, err := os.Create(logFilePath)
+			if err != nil {
+				return err
+			}
+			logFile.Close()
 
-			configFileData := &hubtest.HubTestItemConfig{}
-			if logType == "appsec" {
-				// create empty nuclei template file
-				nucleiFileName := fmt.Sprintf("%s.yaml", testName)
-				nucleiFilePath := filepath.Join(testPath, nucleiFileName)
+			// create empty parser assertion file
+			parserAssertFilePath := filepath.Join(testPath, hubtest.ParserAssertFileName)
+			parserAssertFile, err := os.Create(parserAssertFilePath)
+			if err != nil {
+				return err
+			}
+			parserAssertFile.Close()
 
-				nucleiFile, err := os.OpenFile(nucleiFilePath, os.O_RDWR|os.O_CREATE, 0o755)
-				if err != nil {
-					return err
-				}
+			// create empty scenario assertion file
+			scenarioAssertFilePath := filepath.Join(testPath, hubtest.ScenarioAssertFileName)
+			scenarioAssertFile, err := os.Create(scenarioAssertFilePath)
+			if err != nil {
+				return err
+			}
+			scenarioAssertFile.Close()
 
-				ntpl := template.Must(template.New("nuclei").Parse(hubtest.TemplateNucleiFile))
-				if ntpl == nil {
-					return errors.New("unable to parse nuclei template")
-				}
-				ntpl.ExecuteTemplate(nucleiFile, "nuclei", struct{ TestName string }{TestName: testName})
-				nucleiFile.Close()
-				configFileData.AppsecRules = []string{"./appsec-rules/<author>/your_rule_here.yaml"}
-				configFileData.NucleiTemplate = nucleiFileName
-				fmt.Println()
-				fmt.Printf("  Test name                   :  %s\n", testName)
-				fmt.Printf("  Test path                   :  %s\n", testPath)
-				fmt.Printf("  Config File                 :  %s\n", configFilePath)
-				fmt.Printf("  Nuclei Template             :  %s\n", nucleiFilePath)
-			} else {
-				// create empty log file
-				logFileName := fmt.Sprintf("%s.log", testName)
-				logFilePath := filepath.Join(testPath, logFileName)
-				logFile, err := os.Create(logFilePath)
-				if err != nil {
-					return err
-				}
-				logFile.Close()
+			parsers = append(parsers, "crowdsecurity/syslog-logs")
+			parsers = append(parsers, "crowdsecurity/dateparse-enrich")
 
-				// create empty parser assertion file
-				parserAssertFilePath := filepath.Join(testPath, hubtest.ParserAssertFileName)
-				parserAssertFile, err := os.Create(parserAssertFilePath)
-				if err != nil {
-					return err
-				}
-				parserAssertFile.Close()
-				// create empty scenario assertion file
-				scenarioAssertFilePath := filepath.Join(testPath, hubtest.ScenarioAssertFileName)
-				scenarioAssertFile, err := os.Create(scenarioAssertFilePath)
-				if err != nil {
-					return err
-				}
-				scenarioAssertFile.Close()
-
-				parsers = append(parsers, "crowdsecurity/syslog-logs")
-				parsers = append(parsers, "crowdsecurity/dateparse-enrich")
-
-				if len(scenarios) == 0 {
-					scenarios = append(scenarios, "")
-				}
-
-				if len(postoverflows) == 0 {
-					postoverflows = append(postoverflows, "")
-				}
-				configFileData.Parsers = parsers
-				configFileData.Scenarios = scenarios
-				configFileData.PostOverflows = postoverflows
-				configFileData.LogFile = logFileName
-				configFileData.LogType = logType
-				configFileData.IgnoreParsers = ignoreParsers
-				configFileData.Labels = labels
-				fmt.Println()
-				fmt.Printf("  Test name                   :  %s\n", testName)
-				fmt.Printf("  Test path                   :  %s\n", testPath)
-				fmt.Printf("  Log file                    :  %s (please fill it with logs)\n", logFilePath)
-				fmt.Printf("  Parser assertion file       :  %s (please fill it with assertion)\n", parserAssertFilePath)
-				fmt.Printf("  Scenario assertion file     :  %s (please fill it with assertion)\n", scenarioAssertFilePath)
-				fmt.Printf("  Configuration File          :  %s (please fill it with parsers, scenarios...)\n", configFilePath)
+			if len(scenarios) == 0 {
+				scenarios = append(scenarios, "")
 			}
 
-			fd, err := os.Create(configFilePath)
+			if len(postoverflows) == 0 {
+				postoverflows = append(postoverflows, "")
+			}
+
+			configFileData := &hubtest.HubTestItemConfig{
+				Parsers:       parsers,
+				Scenarios:     scenarios,
+				PostOVerflows: postoverflows,
+				LogFile:       logFileName,
+				LogType:       logType,
+				IgnoreParsers: ignoreParsers,
+				Labels:        labels,
+			}
+
+			configFilePath := filepath.Join(testPath, "config.yaml")
+			fd, err := os.OpenFile(configFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
 			if err != nil {
-				return fmt.Errorf("open: %w", err)
+				return fmt.Errorf("open: %s", err)
 			}
 			data, err := yaml.Marshal(configFileData)
 			if err != nil {
-				return fmt.Errorf("marshal: %w", err)
+				return fmt.Errorf("marshal: %s", err)
 			}
 			_, err = fd.Write(data)
 			if err != nil {
-				return fmt.Errorf("write: %w", err)
+				return fmt.Errorf("write: %s", err)
 			}
 			if err := fd.Close(); err != nil {
-				return fmt.Errorf("close: %w", err)
+				return fmt.Errorf("close: %s", err)
 			}
+			fmt.Println()
+			fmt.Printf("  Test name                   :  %s\n", testName)
+			fmt.Printf("  Test path                   :  %s\n", testPath)
+			fmt.Printf("  Log file                    :  %s (please fill it with logs)\n", logFilePath)
+			fmt.Printf("  Parser assertion file       :  %s (please fill it with assertion)\n", parserAssertFilePath)
+			fmt.Printf("  Scenario assertion file     :  %s (please fill it with assertion)\n", scenarioAssertFilePath)
+			fmt.Printf("  Configuration File          :  %s (please fill it with parsers, scenarios...)\n", configFilePath)
 
 			return nil
 		},
 	}
+	cmdHubTestCreate.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test")
+	cmdHubTestCreate.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test")
+	cmdHubTestCreate.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test")
+	cmdHubTestCreate.Flags().StringSliceVarP(&scenarios, "scenarios", "s", scenarios, "Scenarios to add to test")
+	cmdHubTestCreate.PersistentFlags().BoolVar(&ignoreParsers, "ignore-parsers", false, "Don't run test on parsers")
 
-	cmd.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test")
-	cmd.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test")
-	cmd.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test")
-	cmd.Flags().StringSliceVarP(&scenarios, "scenarios", "s", scenarios, "Scenarios to add to test")
-	cmd.PersistentFlags().BoolVar(&ignoreParsers, "ignore-parsers", false, "Don't run test on parsers")
-
-	return cmd
+	return cmdHubTestCreate
 }
 
-func (cli *cliHubTest) NewRunCmd() *cobra.Command {
-	var (
-		noClean          bool
-		runAll           bool
-		forceClean       bool
-		NucleiTargetHost string
-		AppSecHost       string
-	)
 
-	cmd := &cobra.Command{
+func NewHubTestRunCmd() *cobra.Command {
+	var noClean bool
+	var runAll bool
+	var forceClean bool
+
+	var cmdHubTestRun = &cobra.Command{
 		Use:               "run",
 		Short:             "run [test_name]",
 		DisableAutoGenTag: true,
 		RunE: func(cmd *cobra.Command, args []string) error {
-			cfg := cli.cfg()
-
 			if !runAll && len(args) == 0 {
 				printHelp(cmd)
-				return errors.New("please provide test to run or --all flag")
+				return fmt.Errorf("Please provide test to run or --all flag")
 			}
-			hubPtr.NucleiTargetHost = NucleiTargetHost
-			hubPtr.AppSecHost = AppSecHost
+
 			if runAll {
-				if err := hubPtr.LoadAllTests(); err != nil {
+				if err := HubTest.LoadAllTests(); err != nil {
 					return fmt.Errorf("unable to load all tests: %+v", err)
 				}
 			} else {
 				for _, testName := range args {
-					_, err := hubPtr.LoadTestItem(testName)
+					_, err := HubTest.LoadTestItem(testName)
 					if err != nil {
-						return fmt.Errorf("unable to load test '%s': %w", testName, err)
+						return fmt.Errorf("unable to load test '%s': %s", testName, err)
 					}
 				}
 			}
 
-			// set timezone to avoid DST issues
-			os.Setenv("TZ", "UTC")
-			for _, test := range hubPtr.Tests {
-				if cfg.Cscli.Output == "human" {
+			for _, test := range HubTest.Tests {
+				if csConfig.Cscli.Output == "human" {
 					log.Infof("Running test '%s'", test.Name)
 				}
 				err := test.Run()
@@ -283,13 +214,11 @@ func (cli *cliHubTest) NewRunCmd() *cobra.Command {
 
 			return nil
 		},
-		PersistentPostRunE: func(_ *cobra.Command, _ []string) error {
-			cfg := cli.cfg()
-
+		PersistentPostRunE: func(cmd *cobra.Command, args []string) error {
 			success := true
 			testResult := make(map[string]bool)
-			for _, test := range hubPtr.Tests {
-				if test.AutoGen && !isAppsecTest {
+			for _, test := range HubTest.Tests {
+				if test.AutoGen {
 					if test.ParserAssert.AutoGenAssert {
 						log.Warningf("Assert file '%s' is empty, generating assertion:", test.ParserAssert.File)
 						fmt.Println()
@@ -302,7 +231,7 @@ func (cli *cliHubTest) NewRunCmd() *cobra.Command {
 					}
 					if !noClean {
 						if err := test.Clean(); err != nil {
-							return fmt.Errorf("unable to clean test '%s' env: %w", test.Name, err)
+							return fmt.Errorf("unable to clean test '%s' env: %s", test.Name, err)
 						}
 					}
 					fmt.Printf("\nPlease fill your assert file(s) for test '%s', exiting\n", test.Name)
@@ -310,18 +239,18 @@ func (cli *cliHubTest) NewRunCmd() *cobra.Command {
 				}
 				testResult[test.Name] = test.Success
 				if test.Success {
-					if cfg.Cscli.Output == "human" {
+					if csConfig.Cscli.Output == "human" {
 						log.Infof("Test '%s' passed successfully (%d assertions)\n", test.Name, test.ParserAssert.NbAssert+test.ScenarioAssert.NbAssert)
 					}
 					if !noClean {
 						if err := test.Clean(); err != nil {
-							return fmt.Errorf("unable to clean test '%s' env: %w", test.Name, err)
+							return fmt.Errorf("unable to clean test '%s' env: %s", test.Name, err)
 						}
 					}
 				} else {
 					success = false
 					cleanTestEnv := false
-					if cfg.Cscli.Output == "human" {
+					if csConfig.Cscli.Output == "human" {
 						if len(test.ParserAssert.Fails) > 0 {
 							fmt.Println()
 							log.Errorf("Parser test '%s' failed (%d errors)\n", test.Name, len(test.ParserAssert.Fails))
@@ -352,23 +281,21 @@ func (cli *cliHubTest) NewRunCmd() *cobra.Command {
 								Default: true,
 							}
 							if err := survey.AskOne(prompt, &cleanTestEnv); err != nil {
-								return fmt.Errorf("unable to ask to remove runtime folder: %w", err)
+								return fmt.Errorf("unable to ask to remove runtime folder: %s", err)
 							}
 						}
 					}
 
 					if cleanTestEnv || forceClean {
 						if err := test.Clean(); err != nil {
-							return fmt.Errorf("unable to clean test '%s' env: %w", test.Name, err)
+							return fmt.Errorf("unable to clean test '%s' env: %s", test.Name, err)
 						}
 					}
 				}
 			}
-
-			switch cfg.Cscli.Output {
-			case "human":
+			if csConfig.Cscli.Output == "human" {
 				hubTestResultTable(color.Output, testResult)
-			case "json":
+			} else if csConfig.Cscli.Output == "json" {
 				jsonResult := make(map[string][]string, 0)
 				jsonResult["success"] = make([]string, 0)
 				jsonResult["fail"] = make([]string, 0)
@@ -381,11 +308,9 @@ func (cli *cliHubTest) NewRunCmd() *cobra.Command {
 				}
 				jsonStr, err := json.Marshal(jsonResult)
 				if err != nil {
-					return fmt.Errorf("unable to json test result: %w", err)
+					return fmt.Errorf("unable to json test result: %s", err)
 				}
 				fmt.Println(string(jsonStr))
-			default:
-				return errors.New("only human/json output modes are supported")
 			}
 
 			if !success {
@@ -395,30 +320,28 @@ func (cli *cliHubTest) NewRunCmd() *cobra.Command {
 			return nil
 		},
 	}
+	cmdHubTestRun.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
+	cmdHubTestRun.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
+	cmdHubTestRun.Flags().BoolVar(&runAll, "all", false, "Run all tests")
 
-	cmd.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
-	cmd.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
-	cmd.Flags().StringVar(&NucleiTargetHost, "target", hubtest.DefaultNucleiTarget, "Target for AppSec Test")
-	cmd.Flags().StringVar(&AppSecHost, "host", hubtest.DefaultAppsecHost, "Address to expose AppSec for hubtest")
-	cmd.Flags().BoolVar(&runAll, "all", false, "Run all tests")
-
-	return cmd
+	return cmdHubTestRun
 }
 
-func (cli *cliHubTest) NewCleanCmd() *cobra.Command {
-	cmd := &cobra.Command{
+
+func NewHubTestCleanCmd() *cobra.Command {
+	var cmdHubTestClean = &cobra.Command{
 		Use:               "clean",
 		Short:             "clean [test_name]",
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			for _, testName := range args {
-				test, err := hubPtr.LoadTestItem(testName)
+				test, err := HubTest.LoadTestItem(testName)
 				if err != nil {
-					return fmt.Errorf("unable to load test '%s': %w", testName, err)
+					return fmt.Errorf("unable to load test '%s': %s", testName, err)
 				}
 				if err := test.Clean(); err != nil {
-					return fmt.Errorf("unable to clean test '%s' env: %w", test.Name, err)
+					return fmt.Errorf("unable to clean test '%s' env: %s", test.Name, err)
 				}
 			}
 
@@ -426,32 +349,28 @@ func (cli *cliHubTest) NewCleanCmd() *cobra.Command {
 		},
 	}
 
-	return cmd
+	return cmdHubTestClean
 }
 
-func (cli *cliHubTest) NewInfoCmd() *cobra.Command {
-	cmd := &cobra.Command{
+
+func NewHubTestInfoCmd() *cobra.Command {
+	var cmdHubTestInfo = &cobra.Command{
 		Use:               "info",
 		Short:             "info [test_name]",
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			for _, testName := range args {
-				test, err := hubPtr.LoadTestItem(testName)
+				test, err := HubTest.LoadTestItem(testName)
 				if err != nil {
-					return fmt.Errorf("unable to load test '%s': %w", testName, err)
+					return fmt.Errorf("unable to load test '%s': %s", testName, err)
 				}
 				fmt.Println()
 				fmt.Printf("  Test name                   :  %s\n", test.Name)
 				fmt.Printf("  Test path                   :  %s\n", test.Path)
-				if isAppsecTest {
-					fmt.Printf("  Nuclei Template             :  %s\n", test.Config.NucleiTemplate)
-					fmt.Printf("  Appsec Rules                  :  %s\n", strings.Join(test.Config.AppsecRules, ", "))
-				} else {
-					fmt.Printf("  Log file                    :  %s\n", filepath.Join(test.Path, test.Config.LogFile))
-					fmt.Printf("  Parser assertion file       :  %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName))
-					fmt.Printf("  Scenario assertion file     :  %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName))
-				}
+				fmt.Printf("  Log file                    :  %s\n", filepath.Join(test.Path, test.Config.LogFile))
+				fmt.Printf("  Parser assertion file       :  %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName))
+				fmt.Printf("  Scenario assertion file     :  %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName))
 				fmt.Printf("  Configuration File          :  %s\n", filepath.Join(test.Path, "config.yaml"))
 			}
 
@@ -459,80 +378,72 @@ func (cli *cliHubTest) NewInfoCmd() *cobra.Command {
 		},
 	}
 
-	return cmd
+	return cmdHubTestInfo
 }
 
-func (cli *cliHubTest) NewListCmd() *cobra.Command {
-	cmd := &cobra.Command{
+
+func NewHubTestListCmd() *cobra.Command {
+	var cmdHubTestList = &cobra.Command{
 		Use:               "list",
 		Short:             "list",
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			cfg := cli.cfg()
-
-			if err := hubPtr.LoadAllTests(); err != nil {
-				return fmt.Errorf("unable to load all tests: %w", err)
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if err := HubTest.LoadAllTests(); err != nil {
+				return fmt.Errorf("unable to load all tests: %s", err)
 			}
 
-			switch cfg.Cscli.Output {
+			switch csConfig.Cscli.Output {
 			case "human":
-				hubTestListTable(color.Output, hubPtr.Tests)
+				hubTestListTable(color.Output, HubTest.Tests)
 			case "json":
-				j, err := json.MarshalIndent(hubPtr.Tests, " ", "  ")
+				j, err := json.MarshalIndent(HubTest.Tests, " ", "  ")
 				if err != nil {
 					return err
 				}
 				fmt.Println(string(j))
 			default:
-				return errors.New("only human/json output modes are supported")
+				return fmt.Errorf("only human/json output modes are supported")
 			}
 
 			return nil
 		},
 	}
 
-	return cmd
+	return cmdHubTestList
 }
 
-func (cli *cliHubTest) NewCoverageCmd() *cobra.Command {
-	var (
-		showParserCov   bool
-		showScenarioCov bool
-		showOnlyPercent bool
-		showAppsecCov   bool
-	)
 
-	cmd := &cobra.Command{
+func NewHubTestCoverageCmd() *cobra.Command {
+	var showParserCov bool
+	var showScenarioCov bool
+	var showOnlyPercent bool
+
+	var cmdHubTestCoverage = &cobra.Command{
 		Use:               "coverage",
 		Short:             "coverage",
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			cfg := cli.cfg()
-
-			// for this one we explicitly don't do for appsec
+		RunE: func(cmd *cobra.Command, args []string) error {
 			if err := HubTest.LoadAllTests(); err != nil {
 				return fmt.Errorf("unable to load all tests: %+v", err)
 			}
 			var err error
-			scenarioCoverage := []hubtest.Coverage{}
-			parserCoverage := []hubtest.Coverage{}
-			appsecRuleCoverage := []hubtest.Coverage{}
+			scenarioCoverage := []hubtest.ScenarioCoverage{}
+			parserCoverage := []hubtest.ParserCoverage{}
 			scenarioCoveragePercent := 0
 			parserCoveragePercent := 0
-			appsecRuleCoveragePercent := 0
 
 			// if both are false (flag by default), show both
-			showAll := !showScenarioCov && !showParserCov && !showAppsecCov
+			showAll := !showScenarioCov && !showParserCov
 
 			if showParserCov || showAll {
 				parserCoverage, err = HubTest.GetParsersCoverage()
 				if err != nil {
-					return fmt.Errorf("while getting parser coverage: %w", err)
+					return fmt.Errorf("while getting parser coverage: %s", err)
 				}
 				parserTested := 0
 				for _, test := range parserCoverage {
 					if test.TestsCount > 0 {
-						parserTested++
+						parserTested += 1
 					}
 				}
 				parserCoveragePercent = int(math.Round((float64(parserTested) / float64(len(parserCoverage)) * 100)))
@@ -541,50 +452,29 @@ func (cli *cliHubTest) NewCoverageCmd() *cobra.Command {
 			if showScenarioCov || showAll {
 				scenarioCoverage, err = HubTest.GetScenariosCoverage()
 				if err != nil {
-					return fmt.Errorf("while getting scenario coverage: %w", err)
+					return fmt.Errorf("while getting scenario coverage: %s", err)
 				}
-
 				scenarioTested := 0
 				for _, test := range scenarioCoverage {
 					if test.TestsCount > 0 {
-						scenarioTested++
+						scenarioTested += 1
 					}
 				}
-
 				scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100)))
 			}
 
-			if showAppsecCov || showAll {
-				appsecRuleCoverage, err = HubTest.GetAppsecCoverage()
-				if err != nil {
-					return fmt.Errorf("while getting scenario coverage: %w", err)
-				}
-
-				appsecRuleTested := 0
-				for _, test := range appsecRuleCoverage {
-					if test.TestsCount > 0 {
-						appsecRuleTested++
-					}
-				}
-				appsecRuleCoveragePercent = int(math.Round((float64(appsecRuleTested) / float64(len(appsecRuleCoverage)) * 100)))
-			}
-
 			if showOnlyPercent {
-				switch {
-				case showAll:
-					fmt.Printf("parsers=%d%%\nscenarios=%d%%\nappsec_rules=%d%%", parserCoveragePercent, scenarioCoveragePercent, appsecRuleCoveragePercent)
-				case showParserCov:
+				if showAll {
+					fmt.Printf("parsers=%d%%\nscenarios=%d%%", parserCoveragePercent, scenarioCoveragePercent)
+				} else if showParserCov {
 					fmt.Printf("parsers=%d%%", parserCoveragePercent)
-				case showScenarioCov:
+				} else if showScenarioCov {
 					fmt.Printf("scenarios=%d%%", scenarioCoveragePercent)
-				case showAppsecCov:
-					fmt.Printf("appsec_rules=%d%%", appsecRuleCoveragePercent)
 				}
 				os.Exit(0)
 			}
 
-			switch cfg.Cscli.Output {
-			case "human":
+			if csConfig.Cscli.Output == "human" {
 				if showParserCov || showAll {
 					hubTestParserCoverageTable(color.Output, parserCoverage)
 				}
@@ -592,11 +482,6 @@ func (cli *cliHubTest) NewCoverageCmd() *cobra.Command {
 				if showScenarioCov || showAll {
 					hubTestScenarioCoverageTable(color.Output, scenarioCoverage)
 				}
-
-				if showAppsecCov || showAll {
-					hubTestAppsecRuleCoverageTable(color.Output, appsecRuleCoverage)
-				}
-
 				fmt.Println()
 				if showParserCov || showAll {
 					fmt.Printf("PARSERS    : %d%% of coverage\n", parserCoveragePercent)
@@ -604,10 +489,7 @@ func (cli *cliHubTest) NewCoverageCmd() *cobra.Command {
 				if showScenarioCov || showAll {
 					fmt.Printf("SCENARIOS  : %d%% of coverage\n", scenarioCoveragePercent)
 				}
-				if showAppsecCov || showAll {
-					fmt.Printf("APPSEC RULES  : %d%% of coverage\n", appsecRuleCoveragePercent)
-				}
-			case "json":
+			} else if csConfig.Cscli.Output == "json" {
 				dump, err := json.MarshalIndent(parserCoverage, "", " ")
 				if err != nil {
 					return err
@@ -618,71 +500,29 @@ func (cli *cliHubTest) NewCoverageCmd() *cobra.Command {
 					return err
 				}
 				fmt.Printf("%s", dump)
-				dump, err = json.MarshalIndent(appsecRuleCoverage, "", " ")
-				if err != nil {
-					return err
-				}
-				fmt.Printf("%s", dump)
-			default:
-				return errors.New("only human/json output modes are supported")
+			} else {
+				return fmt.Errorf("only human/json output modes are supported")
 			}
 
 			return nil
 		},
 	}
+	cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
+	cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
+	cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
 
-	cmd.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
-	cmd.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
-	cmd.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
-	cmd.PersistentFlags().BoolVar(&showAppsecCov, "appsec", false, "Show only appsec coverage")
-
-	return cmd
+	return cmdHubTestCoverage
 }
 
-func (cli *cliHubTest) NewEvalCmd() *cobra.Command {
-	var evalExpression string
 
-	cmd := &cobra.Command{
+func NewHubTestEvalCmd() *cobra.Command {
+	var evalExpression string
+	var cmdHubTestEval = &cobra.Command{
 		Use:               "eval",
 		Short:             "eval [test_name]",
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, args []string) error {
-			for _, testName := range args {
-				test, err := hubPtr.LoadTestItem(testName)
-				if err != nil {
-					return fmt.Errorf("can't load test: %+v", err)
-				}
-
-				err = test.ParserAssert.LoadTest(test.ParserResultFile)
-				if err != nil {
-					return fmt.Errorf("can't load test results from '%s': %+v", test.ParserResultFile, err)
-				}
-
-				output, err := test.ParserAssert.EvalExpression(evalExpression)
-				if err != nil {
-					return err
-				}
-
-				fmt.Print(output)
-			}
-
-			return nil
-		},
-	}
-
-	cmd.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval")
-
-	return cmd
-}
-
-func (cli *cliHubTest) NewExplainCmd() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:               "explain",
-		Short:             "explain [test_name]",
-		Args:              cobra.ExactArgs(1),
-		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, args []string) error {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			for _, testName := range args {
 				test, err := HubTest.LoadTestItem(testName)
 				if err != nil {
@@ -690,32 +530,66 @@ func (cli *cliHubTest) NewExplainCmd() *cobra.Command {
 				}
 				err = test.ParserAssert.LoadTest(test.ParserResultFile)
 				if err != nil {
-					if err = test.Run(); err != nil {
+					return fmt.Errorf("can't load test results from '%s': %+v", test.ParserResultFile, err)
+				}
+				output, err := test.ParserAssert.EvalExpression(evalExpression)
+				if err != nil {
+					return err
+				}
+				fmt.Print(output)
+			}
+
+			return nil
+		},
+	}
+	cmdHubTestEval.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval")
+
+	return cmdHubTestEval
+}
+
+
+func NewHubTestExplainCmd() *cobra.Command {
+	var cmdHubTestExplain = &cobra.Command{
+		Use:               "explain",
+		Short:             "explain [test_name]",
+		Args:              cobra.ExactArgs(1),
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			for _, testName := range args {
+				test, err := HubTest.LoadTestItem(testName)
+				if err != nil {
+					return fmt.Errorf("can't load test: %+v", err)
+				}
+				err = test.ParserAssert.LoadTest(test.ParserResultFile)
+				if err != nil {
+					err := test.Run()
+					if err != nil {
 						return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
 					}
-
-					if err = test.ParserAssert.LoadTest(test.ParserResultFile); err != nil {
-						return fmt.Errorf("unable to load parser result after run: %w", err)
+					err = test.ParserAssert.LoadTest(test.ParserResultFile)
+					if err != nil {
+						return fmt.Errorf("unable to load parser result after run: %s", err)
 					}
 				}
 
 				err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
 				if err != nil {
-					if err = test.Run(); err != nil {
+					err := test.Run()
+					if err != nil {
 						return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
 					}
-
-					if err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile); err != nil {
-						return fmt.Errorf("unable to load scenario result after run: %w", err)
+					err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
+					if err != nil {
+						return fmt.Errorf("unable to load scenario result after run: %s", err)
 					}
 				}
-				opts := dumps.DumpOpts{}
-				dumps.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData, opts)
+				opts := hubtest.DumpOpts{}
+				hubtest.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData, opts)
 			}
 
 			return nil
 		},
 	}
 
-	return cmd
+	return cmdHubTestExplain
 }

cmd/crowdsec-cli/hubtest_table.go

@@ -5,8 +5,8 @@ import (
 	"io"
 
 	"github.com/aquasecurity/table"
+	"github.com/enescakir/emoji"
 
-	"github.com/crowdsecurity/crowdsec/pkg/emoji"
 	"github.com/crowdsecurity/crowdsec/pkg/hubtest"
 )
 
@@ -17,9 +17,9 @@ func hubTestResultTable(out io.Writer, testResult map[string]bool) {
 	t.SetAlignment(table.AlignLeft)
 
 	for testName, success := range testResult {
-		status := emoji.CheckMarkButton
+		status := emoji.CheckMarkButton.String()
 		if !success {
-			status = emoji.CrossMark
+			status = emoji.CrossMark.String()
 		}
 
 		t.AddRow(testName, status)
@@ -41,64 +41,39 @@ func hubTestListTable(out io.Writer, tests []*hubtest.HubTestItem) {
 	t.Render()
 }
 
-func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
+func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.ParserCoverage) {
 	t := newLightTable(out)
 	t.SetHeaders("Parser", "Status", "Number of tests")
 	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	parserTested := 0
-
 	for _, test := range coverage {
-		status := emoji.RedCircle
+		status := emoji.RedCircle.String()
 		if test.TestsCount > 0 {
-			status = emoji.GreenCircle
+			status = emoji.GreenCircle.String()
 			parserTested++
 		}
-
-		t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
+		t.AddRow(test.Parser, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
 	}
 
 	t.Render()
 }
 
-func hubTestAppsecRuleCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
-	t := newLightTable(out)
-	t.SetHeaders("Appsec Rule", "Status", "Number of tests")
-	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
-	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
-
-	parserTested := 0
-
-	for _, test := range coverage {
-		status := emoji.RedCircle
-		if test.TestsCount > 0 {
-			status = emoji.GreenCircle
-			parserTested++
-		}
-
-		t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
-	}
-
-	t.Render()
-}
-
-func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
+func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.ScenarioCoverage) {
 	t := newLightTable(out)
 	t.SetHeaders("Scenario", "Status", "Number of tests")
 	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	parserTested := 0
-
 	for _, test := range coverage {
-		status := emoji.RedCircle
+		status := emoji.RedCircle.String()
 		if test.TestsCount > 0 {
-			status = emoji.GreenCircle
+			status = emoji.GreenCircle.String()
 			parserTested++
 		}
-
-		t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
+		t.AddRow(test.Scenario, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
 	}
 
 	t.Render()

cmd/crowdsec-cli/item_metrics.go

@@ -1,327 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"math"
-	"net/http"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/fatih/color"
-	dto "github.com/prometheus/client_model/go"
-	"github.com/prometheus/prom2json"
-	log "github.com/sirupsen/logrus"
-
-	"github.com/crowdsecurity/go-cs-lib/trace"
-
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func ShowMetrics(hubItem *cwhub.Item) error {
-	switch hubItem.Type {
-	case cwhub.PARSERS:
-		metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
-		parserMetricsTable(color.Output, hubItem.Name, metrics)
-	case cwhub.SCENARIOS:
-		metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
-		scenarioMetricsTable(color.Output, hubItem.Name, metrics)
-	case cwhub.COLLECTIONS:
-		for _, sub := range hubItem.SubItems() {
-			if err := ShowMetrics(sub); err != nil {
-				return err
-			}
-		}
-	case cwhub.APPSEC_RULES:
-		metrics := GetAppsecRuleMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
-		appsecMetricsTable(color.Output, hubItem.Name, metrics)
-	default: // no metrics for this item type
-	}
-
-	return nil
-}
-
-// GetParserMetric is a complete rip from prom2json
-func GetParserMetric(url string, itemName string) map[string]map[string]int {
-	stats := make(map[string]map[string]int)
-
-	result := GetPrometheusMetric(url)
-	for idx, fam := range result {
-		if !strings.HasPrefix(fam.Name, "cs_") {
-			continue
-		}
-
-		log.Tracef("round %d", idx)
-
-		for _, m := range fam.Metrics {
-			metric, ok := m.(prom2json.Metric)
-			if !ok {
-				log.Debugf("failed to convert metric to prom2json.Metric")
-				continue
-			}
-
-			name, ok := metric.Labels["name"]
-			if !ok {
-				log.Debugf("no name in Metric %v", metric.Labels)
-			}
-
-			if name != itemName {
-				continue
-			}
-
-			source, ok := metric.Labels["source"]
-
-			if !ok {
-				log.Debugf("no source in Metric %v", metric.Labels)
-			} else {
-				if srctype, ok := metric.Labels["type"]; ok {
-					source = srctype + ":" + source
-				}
-			}
-
-			value := m.(prom2json.Metric).Value
-
-			fval, err := strconv.ParseFloat(value, 32)
-			if err != nil {
-				log.Errorf("Unexpected int value %s : %s", value, err)
-				continue
-			}
-
-			ival := int(fval)
-
-			switch fam.Name {
-			case "cs_reader_hits_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-					stats[source]["parsed"] = 0
-					stats[source]["reads"] = 0
-					stats[source]["unparsed"] = 0
-					stats[source]["hits"] = 0
-				}
-				stats[source]["reads"] += ival
-			case "cs_parser_hits_ok_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["parsed"] += ival
-			case "cs_parser_hits_ko_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["unparsed"] += ival
-			case "cs_node_hits_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["hits"] += ival
-			case "cs_node_hits_ok_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["parsed"] += ival
-			case "cs_node_hits_ko_total":
-				if _, ok := stats[source]; !ok {
-					stats[source] = make(map[string]int)
-				}
-				stats[source]["unparsed"] += ival
-			default:
-				continue
-			}
-		}
-	}
-
-	return stats
-}
-
-func GetScenarioMetric(url string, itemName string) map[string]int {
-	stats := make(map[string]int)
-
-	stats["instantiation"] = 0
-	stats["curr_count"] = 0
-	stats["overflow"] = 0
-	stats["pour"] = 0
-	stats["underflow"] = 0
-
-	result := GetPrometheusMetric(url)
-	for idx, fam := range result {
-		if !strings.HasPrefix(fam.Name, "cs_") {
-			continue
-		}
-
-		log.Tracef("round %d", idx)
-
-		for _, m := range fam.Metrics {
-			metric, ok := m.(prom2json.Metric)
-			if !ok {
-				log.Debugf("failed to convert metric to prom2json.Metric")
-				continue
-			}
-
-			name, ok := metric.Labels["name"]
-
-			if !ok {
-				log.Debugf("no name in Metric %v", metric.Labels)
-			}
-
-			if name != itemName {
-				continue
-			}
-
-			value := m.(prom2json.Metric).Value
-
-			fval, err := strconv.ParseFloat(value, 32)
-			if err != nil {
-				log.Errorf("Unexpected int value %s : %s", value, err)
-				continue
-			}
-
-			ival := int(fval)
-
-			switch fam.Name {
-			case "cs_bucket_created_total":
-				stats["instantiation"] += ival
-			case "cs_buckets":
-				stats["curr_count"] += ival
-			case "cs_bucket_overflowed_total":
-				stats["overflow"] += ival
-			case "cs_bucket_poured_total":
-				stats["pour"] += ival
-			case "cs_bucket_underflowed_total":
-				stats["underflow"] += ival
-			default:
-				continue
-			}
-		}
-	}
-
-	return stats
-}
-
-func GetAppsecRuleMetric(url string, itemName string) map[string]int {
-	stats := make(map[string]int)
-
-	stats["inband_hits"] = 0
-	stats["outband_hits"] = 0
-
-	results := GetPrometheusMetric(url)
-	for idx, fam := range results {
-		if !strings.HasPrefix(fam.Name, "cs_") {
-			continue
-		}
-
-		log.Tracef("round %d", idx)
-
-		for _, m := range fam.Metrics {
-			metric, ok := m.(prom2json.Metric)
-			if !ok {
-				log.Debugf("failed to convert metric to prom2json.Metric")
-				continue
-			}
-
-			name, ok := metric.Labels["rule_name"]
-
-			if !ok {
-				log.Debugf("no rule_name in Metric %v", metric.Labels)
-			}
-
-			if name != itemName {
-				continue
-			}
-
-			band, ok := metric.Labels["type"]
-			if !ok {
-				log.Debugf("no type in Metric %v", metric.Labels)
-			}
-
-			value := m.(prom2json.Metric).Value
-
-			fval, err := strconv.ParseFloat(value, 32)
-			if err != nil {
-				log.Errorf("Unexpected int value %s : %s", value, err)
-				continue
-			}
-
-			ival := int(fval)
-
-			switch fam.Name {
-			case "cs_appsec_rule_hits":
-				switch band {
-				case "inband":
-					stats["inband_hits"] += ival
-				case "outband":
-					stats["outband_hits"] += ival
-				default:
-					continue
-				}
-			default:
-				continue
-			}
-		}
-	}
-
-	return stats
-}
-
-func GetPrometheusMetric(url string) []*prom2json.Family {
-	mfChan := make(chan *dto.MetricFamily, 1024)
-
-	// Start with the DefaultTransport for sane defaults.
-	transport := http.DefaultTransport.(*http.Transport).Clone()
-	// Conservatively disable HTTP keep-alives as this program will only
-	// ever need a single HTTP request.
-	transport.DisableKeepAlives = true
-	// Timeout early if the server doesn't even return the headers.
-	transport.ResponseHeaderTimeout = time.Minute
-
-	go func() {
-		defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
-
-		err := prom2json.FetchMetricFamilies(url, mfChan, transport)
-		if err != nil {
-			log.Fatalf("failed to fetch prometheus metrics : %v", err)
-		}
-	}()
-
-	result := []*prom2json.Family{}
-	for mf := range mfChan {
-		result = append(result, prom2json.NewFamily(mf))
-	}
-
-	log.Debugf("Finished reading prometheus output, %d entries", len(result))
-
-	return result
-}
-
-type unit struct {
-	value  int64
-	symbol string
-}
-
-var ranges = []unit{
-	{value: 1e18, symbol: "E"},
-	{value: 1e15, symbol: "P"},
-	{value: 1e12, symbol: "T"},
-	{value: 1e9, symbol: "G"},
-	{value: 1e6, symbol: "M"},
-	{value: 1e3, symbol: "k"},
-	{value: 1, symbol: ""},
-}
-
-func formatNumber(num int) string {
-	goodUnit := unit{}
-
-	for _, u := range ranges {
-		if int64(num) >= u.value {
-			goodUnit = u
-			break
-		}
-	}
-
-	if goodUnit.value == 1 {
-		return fmt.Sprintf("%d%s", num, goodUnit.symbol)
-	}
-
-	res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
-
-	return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
-}

cmd/crowdsec-cli/item_suggest.go

@@ -1,85 +0,0 @@
-package main
-
-import (
-	"fmt"
-	"slices"
-	"strings"
-
-	"github.com/agext/levenshtein"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-// suggestNearestMessage returns a message with the most similar item name, if one is found
-func suggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string {
-	const maxDistance = 7
-
-	score := 100
-	nearest := ""
-
-	for _, item := range hub.GetItemMap(itemType) {
-		d := levenshtein.Distance(itemName, item.Name, nil)
-		if d < score {
-			score = d
-			nearest = item.Name
-		}
-	}
-
-	msg := fmt.Sprintf("can't find '%s' in %s", itemName, itemType)
-
-	if score < maxDistance {
-		msg += fmt.Sprintf(", did you mean '%s'?", nearest)
-	}
-
-	return msg
-}
-
-func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	hub, err := require.Hub(csConfig, nil, nil)
-	if err != nil {
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
-	comp := make([]string, 0)
-
-	for _, item := range hub.GetItemMap(itemType) {
-		if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
-			comp = append(comp, item.Name)
-		}
-	}
-
-	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
-
-	return comp, cobra.ShellCompDirectiveNoFileComp
-}
-
-func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	hub, err := require.Hub(csConfig, nil, nil)
-	if err != nil {
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
-	items, err := hub.GetInstalledNamesByType(itemType)
-	if err != nil {
-		cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
-		return nil, cobra.ShellCompDirectiveDefault
-	}
-
-	comp := make([]string, 0)
-
-	if toComplete != "" {
-		for _, item := range items {
-			if strings.Contains(item, toComplete) {
-				comp = append(comp, item)
-			}
-		}
-	} else {
-		comp = items
-	}
-
-	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
-
-	return comp, cobra.ShellCompDirectiveNoFileComp
-}

cmd/crowdsec-cli/itemcli.go

@@ -1,546 +0,0 @@
-package main
-
-import (
-	"errors"
-	"fmt"
-	"os"
-	"strings"
-
-	"github.com/fatih/color"
-	"github.com/hexops/gotextdiff"
-	"github.com/hexops/gotextdiff/myers"
-	"github.com/hexops/gotextdiff/span"
-	log "github.com/sirupsen/logrus"
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/go-cs-lib/coalesce"
-
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-type cliHelp struct {
-	// Example is required, the others have a default value
-	// generated from the item type
-	use     string
-	short   string
-	long    string
-	example string
-}
-
-type cliItem struct {
-	cfg           configGetter
-	name          string // plural, as used in the hub index
-	singular      string
-	oneOrMore     string // parenthetical pluralizaion: "parser(s)"
-	help          cliHelp
-	installHelp   cliHelp
-	removeHelp    cliHelp
-	upgradeHelp   cliHelp
-	inspectHelp   cliHelp
-	inspectDetail func(item *cwhub.Item) error
-	listHelp      cliHelp
-}
-
-func (cli cliItem) NewCommand() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:               coalesce.String(cli.help.use, fmt.Sprintf("%s <action> [item]...", cli.name)),
-		Short:             coalesce.String(cli.help.short, fmt.Sprintf("Manage hub %s", cli.name)),
-		Long:              cli.help.long,
-		Example:           cli.help.example,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{cli.singular},
-		DisableAutoGenTag: true,
-	}
-
-	cmd.AddCommand(cli.newInstallCmd())
-	cmd.AddCommand(cli.newRemoveCmd())
-	cmd.AddCommand(cli.newUpgradeCmd())
-	cmd.AddCommand(cli.newInspectCmd())
-	cmd.AddCommand(cli.newListCmd())
-
-	return cmd
-}
-
-func (cli cliItem) install(args []string, downloadOnly bool, force bool, ignoreError bool) error {
-	cfg := cli.cfg()
-
-	hub, err := require.Hub(cfg, require.RemoteHub(cfg), log.StandardLogger())
-	if err != nil {
-		return err
-	}
-
-	for _, name := range args {
-		item := hub.GetItem(cli.name, name)
-		if item == nil {
-			msg := suggestNearestMessage(hub, cli.name, name)
-			if !ignoreError {
-				return errors.New(msg)
-			}
-
-			log.Errorf(msg)
-
-			continue
-		}
-
-		if err := item.Install(force, downloadOnly); err != nil {
-			if !ignoreError {
-				return fmt.Errorf("error while installing '%s': %w", item.Name, err)
-			}
-
-			log.Errorf("Error while installing '%s': %s", item.Name, err)
-		}
-	}
-
-	log.Infof(ReloadMessage())
-
-	return nil
-}
-
-func (cli cliItem) newInstallCmd() *cobra.Command {
-	var (
-		downloadOnly bool
-		force        bool
-		ignoreError  bool
-	)
-
-	cmd := &cobra.Command{
-		Use:               coalesce.String(cli.installHelp.use, "install [item]..."),
-		Short:             coalesce.String(cli.installHelp.short, fmt.Sprintf("Install given %s", cli.oneOrMore)),
-		Long:              coalesce.String(cli.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", cli.name)),
-		Example:           cli.installHelp.example,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compAllItems(cli.name, args, toComplete)
-		},
-		RunE: func(_ *cobra.Command, args []string) error {
-			return cli.install(args, downloadOnly, force, ignoreError)
-		},
-	}
-
-	flags := cmd.Flags()
-	flags.BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
-	flags.BoolVar(&force, "force", false, "Force install: overwrite tainted and outdated files")
-	flags.BoolVar(&ignoreError, "ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", cli.name))
-
-	return cmd
-}
-
-// return the names of the installed parents of an item, used to check if we can remove it
-func istalledParentNames(item *cwhub.Item) []string {
-	ret := make([]string, 0)
-
-	for _, parent := range item.Ancestors() {
-		if parent.State.Installed {
-			ret = append(ret, parent.Name)
-		}
-	}
-
-	return ret
-}
-
-func (cli cliItem) remove(args []string, purge bool, force bool, all bool) error {
-	hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger())
-	if err != nil {
-		return err
-	}
-
-	if all {
-		getter := hub.GetInstalledItemsByType
-		if purge {
-			getter = hub.GetItemsByType
-		}
-
-		items, err := getter(cli.name)
-		if err != nil {
-			return err
-		}
-
-		removed := 0
-
-		for _, item := range items {
-			didRemove, err := item.Remove(purge, force)
-			if err != nil {
-				return err
-			}
-
-			if didRemove {
-				log.Infof("Removed %s", item.Name)
-
-				removed++
-			}
-		}
-
-		log.Infof("Removed %d %s", removed, cli.name)
-
-		if removed > 0 {
-			log.Infof(ReloadMessage())
-		}
-
-		return nil
-	}
-
-	if len(args) == 0 {
-		return fmt.Errorf("specify at least one %s to remove or '--all'", cli.singular)
-	}
-
-	removed := 0
-
-	for _, itemName := range args {
-		item := hub.GetItem(cli.name, itemName)
-		if item == nil {
-			return fmt.Errorf("can't find '%s' in %s", itemName, cli.name)
-		}
-
-		parents := istalledParentNames(item)
-
-		if !force && len(parents) > 0 {
-			log.Warningf("%s belongs to collections: %s", item.Name, parents)
-			log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, cli.singular)
-
-			continue
-		}
-
-		didRemove, err := item.Remove(purge, force)
-		if err != nil {
-			return err
-		}
-
-		if didRemove {
-			log.Infof("Removed %s", item.Name)
-
-			removed++
-		}
-	}
-
-	log.Infof("Removed %d %s", removed, cli.name)
-
-	if removed > 0 {
-		log.Infof(ReloadMessage())
-	}
-
-	return nil
-}
-
-func (cli cliItem) newRemoveCmd() *cobra.Command {
-	var (
-		purge bool
-		force bool
-		all   bool
-	)
-
-	cmd := &cobra.Command{
-		Use:               coalesce.String(cli.removeHelp.use, "remove [item]..."),
-		Short:             coalesce.String(cli.removeHelp.short, fmt.Sprintf("Remove given %s", cli.oneOrMore)),
-		Long:              coalesce.String(cli.removeHelp.long, fmt.Sprintf("Remove one or more %s", cli.name)),
-		Example:           cli.removeHelp.example,
-		Aliases:           []string{"delete"},
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cli.name, args, toComplete)
-		},
-		RunE: func(_ *cobra.Command, args []string) error {
-			return cli.remove(args, purge, force, all)
-		},
-	}
-
-	flags := cmd.Flags()
-	flags.BoolVar(&purge, "purge", false, "Delete source file too")
-	flags.BoolVar(&force, "force", false, "Force remove: remove tainted and outdated files")
-	flags.BoolVar(&all, "all", false, fmt.Sprintf("Remove all the %s", cli.name))
-
-	return cmd
-}
-
-func (cli cliItem) upgrade(args []string, force bool, all bool) error {
-	cfg := cli.cfg()
-
-	hub, err := require.Hub(cfg, require.RemoteHub(cfg), log.StandardLogger())
-	if err != nil {
-		return err
-	}
-
-	if all {
-		items, err := hub.GetInstalledItemsByType(cli.name)
-		if err != nil {
-			return err
-		}
-
-		updated := 0
-
-		for _, item := range items {
-			didUpdate, err := item.Upgrade(force)
-			if err != nil {
-				return err
-			}
-
-			if didUpdate {
-				updated++
-			}
-		}
-
-		log.Infof("Updated %d %s", updated, cli.name)
-
-		if updated > 0 {
-			log.Infof(ReloadMessage())
-		}
-
-		return nil
-	}
-
-	if len(args) == 0 {
-		return fmt.Errorf("specify at least one %s to upgrade or '--all'", cli.singular)
-	}
-
-	updated := 0
-
-	for _, itemName := range args {
-		item := hub.GetItem(cli.name, itemName)
-		if item == nil {
-			return fmt.Errorf("can't find '%s' in %s", itemName, cli.name)
-		}
-
-		didUpdate, err := item.Upgrade(force)
-		if err != nil {
-			return err
-		}
-
-		if didUpdate {
-			log.Infof("Updated %s", item.Name)
-
-			updated++
-		}
-	}
-
-	if updated > 0 {
-		log.Infof(ReloadMessage())
-	}
-
-	return nil
-}
-
-func (cli cliItem) newUpgradeCmd() *cobra.Command {
-	var (
-		all   bool
-		force bool
-	)
-
-	cmd := &cobra.Command{
-		Use:               coalesce.String(cli.upgradeHelp.use, "upgrade [item]..."),
-		Short:             coalesce.String(cli.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", cli.oneOrMore)),
-		Long:              coalesce.String(cli.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", cli.name)),
-		Example:           cli.upgradeHelp.example,
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cli.name, args, toComplete)
-		},
-		RunE: func(_ *cobra.Command, args []string) error {
-			return cli.upgrade(args, force, all)
-		},
-	}
-
-	flags := cmd.Flags()
-	flags.BoolVarP(&all, "all", "a", false, fmt.Sprintf("Upgrade all the %s", cli.name))
-	flags.BoolVar(&force, "force", false, "Force upgrade: overwrite tainted and outdated files")
-
-	return cmd
-}
-
-func (cli cliItem) inspect(args []string, url string, diff bool, rev bool, noMetrics bool) error {
-	cfg := cli.cfg()
-
-	if rev && !diff {
-		return errors.New("--rev can only be used with --diff")
-	}
-
-	if url != "" {
-		cfg.Cscli.PrometheusUrl = url
-	}
-
-	remote := (*cwhub.RemoteHubCfg)(nil)
-
-	if diff {
-		remote = require.RemoteHub(cfg)
-	}
-
-	hub, err := require.Hub(cfg, remote, log.StandardLogger())
-	if err != nil {
-		return err
-	}
-
-	for _, name := range args {
-		item := hub.GetItem(cli.name, name)
-		if item == nil {
-			return fmt.Errorf("can't find '%s' in %s", name, cli.name)
-		}
-
-		if diff {
-			fmt.Println(cli.whyTainted(hub, item, rev))
-
-			continue
-		}
-
-		if err = inspectItem(item, !noMetrics); err != nil {
-			return err
-		}
-
-		if cli.inspectDetail != nil {
-			if err = cli.inspectDetail(item); err != nil {
-				return err
-			}
-		}
-	}
-
-	return nil
-}
-
-func (cli cliItem) newInspectCmd() *cobra.Command {
-	var (
-		url       string
-		diff      bool
-		rev       bool
-		noMetrics bool
-	)
-
-	cmd := &cobra.Command{
-		Use:               coalesce.String(cli.inspectHelp.use, "inspect [item]..."),
-		Short:             coalesce.String(cli.inspectHelp.short, fmt.Sprintf("Inspect given %s", cli.oneOrMore)),
-		Long:              coalesce.String(cli.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", cli.name)),
-		Example:           cli.inspectHelp.example,
-		Args:              cobra.MinimumNArgs(1),
-		DisableAutoGenTag: true,
-		ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-			return compInstalledItems(cli.name, args, toComplete)
-		},
-		RunE: func(_ *cobra.Command, args []string) error {
-			return cli.inspect(args, url, diff, rev, noMetrics)
-		},
-	}
-
-	flags := cmd.Flags()
-	flags.StringVarP(&url, "url", "u", "", "Prometheus url")
-	flags.BoolVar(&diff, "diff", false, "Show diff with latest version (for tainted items)")
-	flags.BoolVar(&rev, "rev", false, "Reverse diff output")
-	flags.BoolVar(&noMetrics, "no-metrics", false, "Don't show metrics (when cscli.output=human)")
-
-	return cmd
-}
-
-func (cli cliItem) list(args []string, all bool) error {
-	hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger())
-	if err != nil {
-		return err
-	}
-
-	items := make(map[string][]*cwhub.Item)
-
-	items[cli.name], err = selectItems(hub, cli.name, args, !all)
-	if err != nil {
-		return err
-	}
-
-	if err = listItems(color.Output, []string{cli.name}, items, false); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (cli cliItem) newListCmd() *cobra.Command {
-	var all bool
-
-	cmd := &cobra.Command{
-		Use:               coalesce.String(cli.listHelp.use, "list [item... | -a]"),
-		Short:             coalesce.String(cli.listHelp.short, fmt.Sprintf("List %s", cli.oneOrMore)),
-		Long:              coalesce.String(cli.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", cli.name)),
-		Example:           cli.listHelp.example,
-		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, args []string) error {
-			return cli.list(args, all)
-		},
-	}
-
-	flags := cmd.Flags()
-	flags.BoolVarP(&all, "all", "a", false, "List disabled items as well")
-
-	return cmd
-}
-
-// return the diff between the installed version and the latest version
-func (cli cliItem) itemDiff(item *cwhub.Item, reverse bool) (string, error) {
-	if !item.State.Installed {
-		return "", fmt.Errorf("'%s' is not installed", item.FQName())
-	}
-
-	latestContent, remoteURL, err := item.FetchLatest()
-	if err != nil {
-		return "", err
-	}
-
-	localContent, err := os.ReadFile(item.State.LocalPath)
-	if err != nil {
-		return "", fmt.Errorf("while reading %s: %w", item.State.LocalPath, err)
-	}
-
-	file1 := item.State.LocalPath
-	file2 := remoteURL
-	content1 := string(localContent)
-	content2 := string(latestContent)
-
-	if reverse {
-		file1, file2 = file2, file1
-		content1, content2 = content2, content1
-	}
-
-	edits := myers.ComputeEdits(span.URIFromPath(file1), content1, content2)
-	diff := gotextdiff.ToUnified(file1, file2, content1, edits)
-
-	return fmt.Sprintf("%s", diff), nil
-}
-
-func (cli cliItem) whyTainted(hub *cwhub.Hub, item *cwhub.Item, reverse bool) string {
-	if !item.State.Installed {
-		return fmt.Sprintf("# %s is not installed", item.FQName())
-	}
-
-	if !item.State.Tainted {
-		return fmt.Sprintf("# %s is not tainted", item.FQName())
-	}
-
-	if len(item.State.TaintedBy) == 0 {
-		return fmt.Sprintf("# %s is tainted but we don't know why. please report this as a bug", item.FQName())
-	}
-
-	ret := []string{
-		fmt.Sprintf("# Let's see why %s is tainted.", item.FQName()),
-	}
-
-	for _, fqsub := range item.State.TaintedBy {
-		ret = append(ret, fmt.Sprintf("\n-> %s\n", fqsub))
-
-		sub, err := hub.GetItemFQ(fqsub)
-		if err != nil {
-			ret = append(ret, err.Error())
-		}
-
-		diff, err := cli.itemDiff(sub, reverse)
-		if err != nil {
-			ret = append(ret, err.Error())
-		}
-
-		if diff != "" {
-			ret = append(ret, diff)
-		} else if len(sub.State.TaintedBy) > 0 {
-			taintList := strings.Join(sub.State.TaintedBy, ", ")
-			if sub.FQName() == taintList {
-				// hack: avoid message "item is tainted by itself"
-				continue
-			}
-
-			ret = append(ret, fmt.Sprintf("# %s is tainted by %s", sub.FQName(), taintList))
-		}
-	}
-
-	return strings.Join(ret, "\n")
-}

cmd/crowdsec-cli/items.go

@@ -1,183 +0,0 @@
-package main
-
-import (
-	"encoding/csv"
-	"encoding/json"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
-	"slices"
-	"strings"
-
-	"gopkg.in/yaml.v3"
-
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-// selectItems returns a slice of items of a given type, selected by name and sorted by case-insensitive name
-func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]*cwhub.Item, error) {
-	itemNames := hub.GetNamesByType(itemType)
-
-	notExist := []string{}
-
-	if len(args) > 0 {
-		for _, arg := range args {
-			if !slices.Contains(itemNames, arg) {
-				notExist = append(notExist, arg)
-			}
-		}
-	}
-
-	if len(notExist) > 0 {
-		return nil, fmt.Errorf("item(s) '%s' not found in %s", strings.Join(notExist, ", "), itemType)
-	}
-
-	if len(args) > 0 {
-		itemNames = args
-		installedOnly = false
-	}
-
-	items := make([]*cwhub.Item, 0, len(itemNames))
-
-	for _, itemName := range itemNames {
-		item := hub.GetItem(itemType, itemName)
-		if installedOnly && !item.State.Installed {
-			continue
-		}
-
-		items = append(items, item)
-	}
-
-	cwhub.SortItemSlice(items)
-
-	return items, nil
-}
-
-func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item, omitIfEmpty bool) error {
-	switch csConfig.Cscli.Output {
-	case "human":
-		nothingToDisplay := true
-
-		for _, itemType := range itemTypes {
-			if omitIfEmpty && len(items[itemType]) == 0 {
-				continue
-			}
-
-			listHubItemTable(out, "\n"+strings.ToUpper(itemType), items[itemType])
-
-			nothingToDisplay = false
-		}
-
-		if nothingToDisplay {
-			fmt.Println("No items to display")
-		}
-	case "json":
-		type itemHubStatus struct {
-			Name         string `json:"name"`
-			LocalVersion string `json:"local_version"`
-			LocalPath    string `json:"local_path"`
-			Description  string `json:"description"`
-			UTF8Status   string `json:"utf8_status"`
-			Status       string `json:"status"`
-		}
-
-		hubStatus := make(map[string][]itemHubStatus)
-		for _, itemType := range itemTypes {
-			// empty slice in case there are no items of this type
-			hubStatus[itemType] = make([]itemHubStatus, len(items[itemType]))
-
-			for i, item := range items[itemType] {
-				status := item.State.Text()
-				statusEmo := item.State.Emoji()
-				hubStatus[itemType][i] = itemHubStatus{
-					Name:         item.Name,
-					LocalVersion: item.State.LocalVersion,
-					LocalPath:    item.State.LocalPath,
-					Description:  item.Description,
-					Status:       status,
-					UTF8Status:   fmt.Sprintf("%v  %s", statusEmo, status),
-				}
-			}
-		}
-
-		x, err := json.MarshalIndent(hubStatus, "", " ")
-		if err != nil {
-			return fmt.Errorf("failed to unmarshal: %w", err)
-		}
-
-		out.Write(x)
-	case "raw":
-		csvwriter := csv.NewWriter(out)
-
-		header := []string{"name", "status", "version", "description"}
-		if len(itemTypes) > 1 {
-			header = append(header, "type")
-		}
-
-		if err := csvwriter.Write(header); err != nil {
-			return fmt.Errorf("failed to write header: %w", err)
-		}
-
-		for _, itemType := range itemTypes {
-			for _, item := range items[itemType] {
-				row := []string{
-					item.Name,
-					item.State.Text(),
-					item.State.LocalVersion,
-					item.Description,
-				}
-				if len(itemTypes) > 1 {
-					row = append(row, itemType)
-				}
-
-				if err := csvwriter.Write(row); err != nil {
-					return fmt.Errorf("failed to write raw output: %w", err)
-				}
-			}
-		}
-
-		csvwriter.Flush()
-	}
-
-	return nil
-}
-
-func inspectItem(item *cwhub.Item, showMetrics bool) error {
-	switch csConfig.Cscli.Output {
-	case "human", "raw":
-		enc := yaml.NewEncoder(os.Stdout)
-		enc.SetIndent(2)
-
-		if err := enc.Encode(item); err != nil {
-			return fmt.Errorf("unable to encode item: %w", err)
-		}
-	case "json":
-		b, err := json.MarshalIndent(*item, "", "  ")
-		if err != nil {
-			return fmt.Errorf("unable to marshal item: %w", err)
-		}
-
-		fmt.Print(string(b))
-	}
-
-	if csConfig.Cscli.Output != "human" {
-		return nil
-	}
-
-	if item.State.Tainted {
-		fmt.Println()
-		fmt.Printf(`This item is tainted. Use "%s %s inspect --diff %s" to see why.`, filepath.Base(os.Args[0]), item.Type, item.Name)
-		fmt.Println()
-	}
-
-	if showMetrics {
-		fmt.Printf("\nCurrent metrics: \n")
-
-		if err := ShowMetrics(item); err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

cmd/crowdsec-cli/lapi.go

@@ -2,22 +2,21 @@ package main
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"net/url"
 	"os"
-	"slices"
 	"sort"
 	"strings"
 
 	"github.com/go-openapi/strfmt"
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v3"
+	"golang.org/x/exp/slices"
+	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/version"
+	"github.com/crowdsecurity/go-cs-lib/pkg/version"
 
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
@@ -25,90 +24,103 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
 	"github.com/crowdsecurity/crowdsec/pkg/models"
 	"github.com/crowdsecurity/crowdsec/pkg/parser"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
-const LAPIURLPrefix = "v1"
+var LAPIURLPrefix string = "v1"
 
-type cliLapi struct {
-	cfg configGetter
-}
+func runLapiStatus(cmd *cobra.Command, args []string) error {
+	var err error
 
-func NewCLILapi(cfg configGetter) *cliLapi {
-	return &cliLapi{
-		cfg: cfg,
-	}
-}
-
-func (cli *cliLapi) status() error {
-	cfg := cli.cfg()
-	password := strfmt.Password(cfg.API.Client.Credentials.Password)
-	login := cfg.API.Client.Credentials.Login
-
-	origURL := cfg.API.Client.Credentials.URL
-
-	apiURL, err := url.Parse(origURL)
+	password := strfmt.Password(csConfig.API.Client.Credentials.Password)
+	apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL)
+	login := csConfig.API.Client.Credentials.Login
 	if err != nil {
-		return fmt.Errorf("parsing api url: %w", err)
+		log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
+	}
+	if err := csConfig.LoadHub(); err != nil {
+		log.Fatal(err)
 	}
 
-	hub, err := require.Hub(cfg, nil, nil)
+	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+		log.Info("Run 'sudo cscli hub update' to get the hub index")
+		log.Fatalf("Failed to load hub index : %s", err)
+	}
+	scenarios, err := cwhub.GetInstalledScenariosAsString()
 	if err != nil {
-		return err
+		log.Fatalf("failed to get scenarios : %s", err)
 	}
 
-	scenarios, err := hub.GetInstalledNamesByType(cwhub.SCENARIOS)
-	if err != nil {
-		return fmt.Errorf("failed to get scenarios: %w", err)
-	}
-
-	Client, err = apiclient.NewDefaultClient(apiURL,
+	Client, err = apiclient.NewDefaultClient(apiurl,
 		LAPIURLPrefix,
 		fmt.Sprintf("crowdsec/%s", version.String()),
 		nil)
 	if err != nil {
-		return fmt.Errorf("init default client: %w", err)
+		log.Fatalf("init default client: %s", err)
 	}
-
 	t := models.WatcherAuthRequest{
 		MachineID: &login,
 		Password:  &password,
 		Scenarios: scenarios,
 	}
-
-	log.Infof("Loaded credentials from %s", cfg.API.Client.CredentialsFilePath)
-	// use the original string because apiURL would print 'http://unix/'
-	log.Infof("Trying to authenticate with username %s on %s", login, origURL)
-
+	log.Infof("Loaded credentials from %s", csConfig.API.Client.CredentialsFilePath)
+	log.Infof("Trying to authenticate with username %s on %s", login, apiurl)
 	_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
 	if err != nil {
-		return fmt.Errorf("failed to authenticate to Local API (LAPI): %w", err)
+		log.Fatalf("Failed to authenticate to Local API (LAPI) : %s", err)
+	} else {
+		log.Infof("You can successfully interact with Local API (LAPI)")
 	}
 
-	log.Infof("You can successfully interact with Local API (LAPI)")
-
 	return nil
 }
 
-func (cli *cliLapi) register(apiURL string, outputFile string, machine string) error {
+func runLapiRegister(cmd *cobra.Command, args []string) error {
 	var err error
 
-	lapiUser := machine
-	cfg := cli.cfg()
+	flags := cmd.Flags()
+
+	apiURL, err := flags.GetString("url")
+	if err != nil {
+		return err
+	}
+
+	outputFile, err := flags.GetString("file")
+	if err != nil {
+		return err
+	}
+
+	lapiUser, err := flags.GetString("machine")
+	if err != nil {
+		return err
+	}
 
 	if lapiUser == "" {
 		lapiUser, err = generateID("")
 		if err != nil {
-			return fmt.Errorf("unable to generate machine id: %w", err)
+			log.Fatalf("unable to generate machine id: %s", err)
 		}
 	}
-
 	password := strfmt.Password(generatePassword(passwordLength))
-
-	apiurl, err := prepareAPIURL(cfg.API.Client, apiURL)
-	if err != nil {
-		return fmt.Errorf("parsing api url: %w", err)
+	if apiURL == "" {
+		if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" {
+			apiURL = csConfig.API.Client.Credentials.URL
+		} else {
+			log.Fatalf("No Local API URL. Please provide it in your configuration or with the -u parameter")
+		}
+	}
+	/*URL needs to end with /, but user doesn't care*/
+	if !strings.HasSuffix(apiURL, "/") {
+		apiURL += "/"
+	}
+	/*URL needs to start with http://, but user doesn't care*/
+	if !strings.HasPrefix(apiURL, "http://") && !strings.HasPrefix(apiURL, "https://") {
+		apiURL = "http://" + apiURL
+	}
+	apiurl, err := url.Parse(apiURL)
+	if err != nil {
+		log.Fatalf("parsing api url: %s", err)
 	}
-
 	_, err = apiclient.RegisterClient(&apiclient.Config{
 		MachineID:     lapiUser,
 		Password:      password,
@@ -116,284 +128,208 @@ func (cli *cliLapi) register(apiURL string, outputFile string, machine string) e
 		URL:           apiurl,
 		VersionPrefix: LAPIURLPrefix,
 	}, nil)
+
 	if err != nil {
-		return fmt.Errorf("api client register: %w", err)
+		log.Fatalf("api client register: %s", err)
 	}
 
 	log.Printf("Successfully registered to Local API (LAPI)")
 
 	var dumpFile string
-
 	if outputFile != "" {
 		dumpFile = outputFile
-	} else if cfg.API.Client.CredentialsFilePath != "" {
-		dumpFile = cfg.API.Client.CredentialsFilePath
+	} else if csConfig.API.Client.CredentialsFilePath != "" {
+		dumpFile = csConfig.API.Client.CredentialsFilePath
 	} else {
 		dumpFile = ""
 	}
-
 	apiCfg := csconfig.ApiCredentialsCfg{
 		Login:    lapiUser,
 		Password: password.String(),
 		URL:      apiURL,
 	}
-
 	apiConfigDump, err := yaml.Marshal(apiCfg)
 	if err != nil {
-		return fmt.Errorf("unable to marshal api credentials: %w", err)
+		log.Fatalf("unable to marshal api credentials: %s", err)
 	}
-
 	if dumpFile != "" {
-		err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
+		err = os.WriteFile(dumpFile, apiConfigDump, 0644)
 		if err != nil {
-			return fmt.Errorf("write api credentials to '%s' failed: %w", dumpFile, err)
+			log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
 		}
-
-		log.Printf("Local API credentials written to '%s'", dumpFile)
+		log.Printf("Local API credentials dumped to '%s'", dumpFile)
 	} else {
 		fmt.Printf("%s\n", string(apiConfigDump))
 	}
-
 	log.Warning(ReloadMessage())
 
 	return nil
 }
 
-// prepareAPIURL checks/fixes a LAPI connection url (http, https or socket) and returns an URL struct
-func prepareAPIURL(clientCfg *csconfig.LocalApiClientCfg, apiURL string) (*url.URL, error) {
-	if apiURL == "" {
-		if clientCfg == nil || clientCfg.Credentials == nil || clientCfg.Credentials.URL == "" {
-			return nil, errors.New("no Local API URL. Please provide it in your configuration or with the -u parameter")
-		}
-
-		apiURL = clientCfg.Credentials.URL
-	}
-
-	// URL needs to end with /, but user doesn't care
-	if !strings.HasSuffix(apiURL, "/") {
-		apiURL += "/"
-	}
-
-	// URL needs to start with http://, but user doesn't care
-	if !strings.HasPrefix(apiURL, "http://") && !strings.HasPrefix(apiURL, "https://") && !strings.HasPrefix(apiURL, "/") {
-		apiURL = "http://" + apiURL
-	}
-
-	return url.Parse(apiURL)
-}
-
-func (cli *cliLapi) newStatusCmd() *cobra.Command {
+func NewLapiStatusCmd() *cobra.Command {
 	cmdLapiStatus := &cobra.Command{
 		Use:               "status",
 		Short:             "Check authentication to Local API (LAPI)",
 		Args:              cobra.MinimumNArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			return cli.status()
-		},
+		RunE:              runLapiStatus,
 	}
 
 	return cmdLapiStatus
 }
 
-func (cli *cliLapi) newRegisterCmd() *cobra.Command {
-	var (
-		apiURL     string
-		outputFile string
-		machine    string
-	)
-
-	cmd := &cobra.Command{
+func NewLapiRegisterCmd() *cobra.Command {
+	cmdLapiRegister := &cobra.Command{
 		Use:   "register",
 		Short: "Register a machine to Local API (LAPI)",
-		Long: `Register your machine to the Local API (LAPI).
+		Long: `Register you machine to the Local API (LAPI).
 Keep in mind the machine needs to be validated by an administrator on LAPI side to be effective.`,
 		Args:              cobra.MinimumNArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			return cli.register(apiURL, outputFile, machine)
-		},
+		RunE:              runLapiRegister,
 	}
 
-	flags := cmd.Flags()
-	flags.StringVarP(&apiURL, "url", "u", "", "URL of the API (ie. http://127.0.0.1)")
-	flags.StringVarP(&outputFile, "file", "f", "", "output file destination")
-	flags.StringVar(&machine, "machine", "", "Name of the machine to register with")
+	flags := cmdLapiRegister.Flags()
+	flags.StringP("url", "u", "", "URL of the API (ie. http://127.0.0.1)")
+	flags.StringP("file", "f", "", "output file destination")
+	flags.String("machine", "", "Name of the machine to register with")
 
-	return cmd
+	return cmdLapiRegister
 }
 
-func (cli *cliLapi) NewCommand() *cobra.Command {
-	cmd := &cobra.Command{
+func NewLapiCmd() *cobra.Command {
+	var cmdLapi = &cobra.Command{
 		Use:               "lapi [action]",
 		Short:             "Manage interaction with Local API (LAPI)",
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
-			if err := cli.cfg().LoadAPIClient(); err != nil {
-				return fmt.Errorf("loading api client: %w", err)
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := csConfig.LoadAPIClient(); err != nil {
+				return errors.Wrap(err, "loading api client")
 			}
 			return nil
 		},
 	}
 
-	cmd.AddCommand(cli.newRegisterCmd())
-	cmd.AddCommand(cli.newStatusCmd())
-	cmd.AddCommand(cli.newContextCmd())
+	cmdLapi.AddCommand(NewLapiRegisterCmd())
+	cmdLapi.AddCommand(NewLapiStatusCmd())
+	cmdLapi.AddCommand(NewLapiContextCmd())
 
-	return cmd
+	return cmdLapi
 }
 
-func (cli *cliLapi) addContext(key string, values []string) error {
-	cfg := cli.cfg()
+func NewLapiContextCmd() *cobra.Command {
+	cmdContext := &cobra.Command{
+		Use:               "context [command]",
+		Short:             "Manage context to send with alerts",
+		DisableAutoGenTag: true,
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := csConfig.LoadCrowdsec(); err != nil {
+				fileNotFoundMessage := fmt.Sprintf("failed to open context file: open %s: no such file or directory", csConfig.Crowdsec.ConsoleContextPath)
+				if err.Error() != fileNotFoundMessage {
+					log.Fatalf("Unable to load CrowdSec Agent: %s", err)
+				}
+			}
+			if csConfig.DisableAgent {
+				log.Fatalf("Agent is disabled and lapi context can only be used on the agent")
+			}
 
-	if err := alertcontext.ValidateContextExpr(key, values); err != nil {
-		return fmt.Errorf("invalid context configuration: %w", err)
+			return nil
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			printHelp(cmd)
+		},
 	}
 
-	if _, ok := cfg.Crowdsec.ContextToSend[key]; !ok {
-		cfg.Crowdsec.ContextToSend[key] = make([]string, 0)
-
-		log.Infof("key '%s' added", key)
-	}
-
-	data := cfg.Crowdsec.ContextToSend[key]
-
-	for _, val := range values {
-		if !slices.Contains(data, val) {
-			log.Infof("value '%s' added to key '%s'", val, key)
-			data = append(data, val)
-		}
-
-		cfg.Crowdsec.ContextToSend[key] = data
-	}
-
-	if err := cfg.Crowdsec.DumpContextConfigFile(); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (cli *cliLapi) newContextAddCmd() *cobra.Command {
-	var (
-		keyToAdd    string
-		valuesToAdd []string
-	)
-
-	cmd := &cobra.Command{
+	var keyToAdd string
+	var valuesToAdd []string
+	cmdContextAdd := &cobra.Command{
 		Use:   "add",
 		Short: "Add context to send with alerts. You must specify the output key with the expr value you want",
 		Example: `cscli lapi context add --key source_ip --value evt.Meta.source_ip
 cscli lapi context add --key file_source --value evt.Line.Src
-cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user 
 		`,
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			hub, err := require.Hub(cli.cfg(), nil, nil)
-			if err != nil {
-				return err
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := alertcontext.ValidateContextExpr(keyToAdd, valuesToAdd); err != nil {
+				log.Fatalf("invalid context configuration :%s", err)
 			}
-
-			if err = alertcontext.LoadConsoleContext(cli.cfg(), hub); err != nil {
-				return fmt.Errorf("while loading context: %w", err)
+			if _, ok := csConfig.Crowdsec.ContextToSend[keyToAdd]; !ok {
+				csConfig.Crowdsec.ContextToSend[keyToAdd] = make([]string, 0)
+				log.Infof("key '%s' added", keyToAdd)
 			}
-
-			if keyToAdd != "" {
-				if err := cli.addContext(keyToAdd, valuesToAdd); err != nil {
-					return err
+			data := csConfig.Crowdsec.ContextToSend[keyToAdd]
+			for _, val := range valuesToAdd {
+				if !slices.Contains(data, val) {
+					log.Infof("value '%s' added to key '%s'", val, keyToAdd)
+					data = append(data, val)
 				}
-				return nil
+				csConfig.Crowdsec.ContextToSend[keyToAdd] = data
 			}
-
-			for _, v := range valuesToAdd {
-				keySlice := strings.Split(v, ".")
-				key := keySlice[len(keySlice)-1]
-				value := []string{v}
-				if err := cli.addContext(key, value); err != nil {
-					return err
-				}
+			if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
+				log.Fatalf(err.Error())
 			}
-
-			return nil
 		},
 	}
+	cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
+	cmdContextAdd.Flags().StringSliceVar(&valuesToAdd, "value", []string{}, "The expr fields to associate with the key")
+	cmdContextAdd.MarkFlagRequired("key")
+	cmdContextAdd.MarkFlagRequired("value")
+	cmdContext.AddCommand(cmdContextAdd)
 
-	flags := cmd.Flags()
-	flags.StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
-	flags.StringSliceVar(&valuesToAdd, "value", []string{}, "The expr fields to associate with the key")
-	cmd.MarkFlagRequired("value")
-
-	return cmd
-}
-
-func (cli *cliLapi) newContextStatusCmd() *cobra.Command {
-	cmd := &cobra.Command{
+	cmdContextStatus := &cobra.Command{
 		Use:               "status",
 		Short:             "List context to send with alerts",
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			cfg := cli.cfg()
-			hub, err := require.Hub(cfg, nil, nil)
-			if err != nil {
-				return err
-			}
-
-			if err = alertcontext.LoadConsoleContext(cfg, hub); err != nil {
-				return fmt.Errorf("while loading context: %w", err)
-			}
-
-			if len(cfg.Crowdsec.ContextToSend) == 0 {
+		Run: func(cmd *cobra.Command, args []string) {
+			if len(csConfig.Crowdsec.ContextToSend) == 0 {
 				fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.")
-				return nil
+				return
 			}
 
-			dump, err := yaml.Marshal(cfg.Crowdsec.ContextToSend)
+			dump, err := yaml.Marshal(csConfig.Crowdsec.ContextToSend)
 			if err != nil {
-				return fmt.Errorf("unable to show context status: %w", err)
+				log.Fatalf("unable to show context status: %s", err)
 			}
 
-			fmt.Print(string(dump))
+			fmt.Println(string(dump))
 
-			return nil
 		},
 	}
+	cmdContext.AddCommand(cmdContextStatus)
 
-	return cmd
-}
-
-func (cli *cliLapi) newContextDetectCmd() *cobra.Command {
 	var detectAll bool
-
-	cmd := &cobra.Command{
+	cmdContextDetect := &cobra.Command{
 		Use:   "detect",
 		Short: "Detect available fields from the installed parsers",
 		Example: `cscli lapi context detect --all
 cscli lapi context detect crowdsecurity/sshd-logs
 		`,
 		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			cfg := cli.cfg()
+		Run: func(cmd *cobra.Command, args []string) {
+			var err error
+
 			if !detectAll && len(args) == 0 {
 				log.Infof("Please provide parsers to detect or --all flag.")
 				printHelp(cmd)
 			}
 
 			// to avoid all the log.Info from the loaders functions
-			log.SetLevel(log.WarnLevel)
+			log.SetLevel(log.ErrorLevel)
 
-			if err := exprhelpers.Init(nil); err != nil {
-				return fmt.Errorf("failed to init expr helpers: %w", err)
-			}
-
-			hub, err := require.Hub(cfg, nil, nil)
+			err = exprhelpers.Init(nil)
 			if err != nil {
-				return err
+				log.Fatalf("Failed to init expr helpers : %s", err)
 			}
 
-			csParsers := parser.NewParsers(hub)
-			if csParsers, err = parser.LoadParsers(cfg, csParsers); err != nil {
-				return fmt.Errorf("unable to load parsers: %w", err)
+			// Populate cwhub package tools
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+				log.Fatalf("Failed to load hub index : %s", err)
+			}
+
+			csParsers := parser.NewParsers()
+			if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil {
+				log.Fatalf("unable to load parsers: %s", err)
 			}
 
 			fieldByParsers := make(map[string][]string)
@@ -413,6 +349,7 @@ cscli lapi context detect crowdsecurity/sshd-logs
 						fieldByParsers[node.Name] = append(fieldByParsers[node.Name], field)
 					}
 				}
+
 			}
 
 			fmt.Printf("Acquisition :\n\n")
@@ -445,89 +382,84 @@ cscli lapi context detect crowdsecurity/sshd-logs
 					log.Errorf("parser '%s' not found, can't detect fields", parserNotFound)
 				}
 			}
-
-			return nil
 		},
 	}
-	cmd.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser")
+	cmdContextDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser")
+	cmdContext.AddCommand(cmdContextDetect)
 
-	return cmd
-}
-
-func (cli *cliLapi) newContextDeleteCmd() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:               "delete",
+	var keysToDelete []string
+	var valuesToDelete []string
+	cmdContextDelete := &cobra.Command{
+		Use:   "delete",
+		Short: "Delete context to send with alerts",
+		Example: `cscli lapi context delete --key source_ip
+cscli lapi context delete --value evt.Line.Src
+		`,
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			filePath := cli.cfg().Crowdsec.ConsoleContextPath
-			if filePath == "" {
-				filePath = "the context file"
+		Run: func(cmd *cobra.Command, args []string) {
+			if len(keysToDelete) == 0 && len(valuesToDelete) == 0 {
+				log.Fatalf("please provide at least a key or a value to delete")
 			}
-			fmt.Printf("Command 'delete' is deprecated, please manually edit %s.", filePath)
 
-			return nil
-		},
-	}
-
-	return cmd
-}
-
-func (cli *cliLapi) newContextCmd() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:               "context [command]",
-		Short:             "Manage context to send with alerts",
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
-			cfg := cli.cfg()
-			if err := cfg.LoadCrowdsec(); err != nil {
-				fileNotFoundMessage := fmt.Sprintf("failed to open context file: open %s: no such file or directory", cfg.Crowdsec.ConsoleContextPath)
-				if err.Error() != fileNotFoundMessage {
-					return fmt.Errorf("unable to load CrowdSec agent configuration: %w", err)
+			for _, key := range keysToDelete {
+				if _, ok := csConfig.Crowdsec.ContextToSend[key]; ok {
+					delete(csConfig.Crowdsec.ContextToSend, key)
+					log.Infof("key '%s' has been removed", key)
+				} else {
+					log.Warningf("key '%s' doesn't exist", key)
 				}
 			}
-			if cfg.DisableAgent {
-				return errors.New("agent is disabled and lapi context can only be used on the agent")
+
+			for _, value := range valuesToDelete {
+				valueFound := false
+				for key, context := range csConfig.Crowdsec.ContextToSend {
+					if slices.Contains(context, value) {
+						valueFound = true
+						csConfig.Crowdsec.ContextToSend[key] = removeFromSlice(value, context)
+						log.Infof("value '%s' has been removed from key '%s'", value, key)
+					}
+					if len(csConfig.Crowdsec.ContextToSend[key]) == 0 {
+						delete(csConfig.Crowdsec.ContextToSend, key)
+					}
+				}
+				if !valueFound {
+					log.Warningf("value '%s' not found", value)
+				}
+			}
+
+			if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
+				log.Fatalf(err.Error())
 			}
 
-			return nil
-		},
-		Run: func(cmd *cobra.Command, _ []string) {
-			printHelp(cmd)
 		},
 	}
+	cmdContextDelete.Flags().StringSliceVarP(&keysToDelete, "key", "k", []string{}, "The keys to delete")
+	cmdContextDelete.Flags().StringSliceVar(&valuesToDelete, "value", []string{}, "The expr fields to delete")
+	cmdContext.AddCommand(cmdContextDelete)
 
-	cmd.AddCommand(cli.newContextAddCmd())
-	cmd.AddCommand(cli.newContextStatusCmd())
-	cmd.AddCommand(cli.newContextDetectCmd())
-	cmd.AddCommand(cli.newContextDeleteCmd())
-
-	return cmd
+	return cmdContext
 }
 
-func detectStaticField(grokStatics []parser.ExtraField) []string {
+func detectStaticField(GrokStatics []types.ExtraField) []string {
 	ret := make([]string, 0)
-
-	for _, static := range grokStatics {
+	for _, static := range GrokStatics {
 		if static.Parsed != "" {
 			fieldName := fmt.Sprintf("evt.Parsed.%s", static.Parsed)
 			if !slices.Contains(ret, fieldName) {
 				ret = append(ret, fieldName)
 			}
 		}
-
 		if static.Meta != "" {
 			fieldName := fmt.Sprintf("evt.Meta.%s", static.Meta)
 			if !slices.Contains(ret, fieldName) {
 				ret = append(ret, fieldName)
 			}
 		}
-
 		if static.TargetByName != "" {
 			fieldName := static.TargetByName
 			if !strings.HasPrefix(fieldName, "evt.") {
 				fieldName = "evt." + fieldName
 			}
-
 			if !slices.Contains(ret, fieldName) {
 				ret = append(ret, fieldName)
 			}
@@ -538,8 +470,7 @@ func detectStaticField(grokStatics []parser.ExtraField) []string {
 }
 
 func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
-	ret := make([]string, 0)
-
+	var ret = make([]string, 0)
 	if node.Grok.RunTimeRegexp != nil {
 		for _, capturedField := range node.Grok.RunTimeRegexp.Names() {
 			fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
@@ -551,13 +482,13 @@ func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
 
 	if node.Grok.RegexpName != "" {
 		grokCompiled, err := parserCTX.Grok.Get(node.Grok.RegexpName)
-		// ignore error (parser does not exist?)
-		if err == nil {
-			for _, capturedField := range grokCompiled.Names() {
-				fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
-				if !slices.Contains(ret, fieldName) {
-					ret = append(ret, fieldName)
-				}
+		if err != nil {
+			log.Warningf("Can't get subgrok: %s", err)
+		}
+		for _, capturedField := range grokCompiled.Names() {
+			fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
+			if !slices.Contains(ret, fieldName) {
+				ret = append(ret, fieldName)
 			}
 		}
 	}
@@ -584,7 +515,7 @@ func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
 }
 
 func detectSubNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
-	ret := make([]string, 0)
+	var ret = make([]string, 0)
 
 	for _, subnode := range node.LeavesNodes {
 		if subnode.Grok.RunTimeRegexp != nil {
@@ -595,16 +526,15 @@ func detectSubNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
 				}
 			}
 		}
-
 		if subnode.Grok.RegexpName != "" {
 			grokCompiled, err := parserCTX.Grok.Get(subnode.Grok.RegexpName)
-			if err == nil {
-				// ignore error (parser does not exist?)
-				for _, capturedField := range grokCompiled.Names() {
-					fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
-					if !slices.Contains(ret, fieldName) {
-						ret = append(ret, fieldName)
-					}
+			if err != nil {
+				log.Warningf("Can't get subgrok: %s", err)
+			}
+			for _, capturedField := range grokCompiled.Names() {
+				fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
+				if !slices.Contains(ret, fieldName) {
+					ret = append(ret, fieldName)
 				}
 			}
 		}

cmd/crowdsec-cli/lapi_test.go

@@ -1,49 +0,0 @@
-package main
-
-import (
-	"testing"
-
-	"github.com/stretchr/testify/assert"
-	"github.com/stretchr/testify/require"
-
-	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
-)
-
-func TestPrepareAPIURL_NoProtocol(t *testing.T) {
-	url, err := prepareAPIURL(nil, "localhost:81")
-	require.NoError(t, err)
-	assert.Equal(t, "http://localhost:81/", url.String())
-}
-
-func TestPrepareAPIURL_Http(t *testing.T) {
-	url, err := prepareAPIURL(nil, "http://localhost:81")
-	require.NoError(t, err)
-	assert.Equal(t, "http://localhost:81/", url.String())
-}
-
-func TestPrepareAPIURL_Https(t *testing.T) {
-	url, err := prepareAPIURL(nil, "https://localhost:81")
-	require.NoError(t, err)
-	assert.Equal(t, "https://localhost:81/", url.String())
-}
-
-func TestPrepareAPIURL_UnixSocket(t *testing.T) {
-	url, err := prepareAPIURL(nil, "/path/socket")
-	require.NoError(t, err)
-	assert.Equal(t, "/path/socket/", url.String())
-}
-
-func TestPrepareAPIURL_Empty(t *testing.T) {
-	_, err := prepareAPIURL(nil, "")
-	require.Error(t, err)
-}
-
-func TestPrepareAPIURL_Empty_ConfigOverride(t *testing.T) {
-	url, err := prepareAPIURL(&csconfig.LocalApiClientCfg{
-		Credentials: &csconfig.ApiCredentialsCfg{
-			URL: "localhost:80",
-		},
-	}, "")
-	require.NoError(t, err)
-	assert.Equal(t, "http://localhost:80/", url.String())
-}

cmd/crowdsec-cli/machines.go

@@ -4,32 +4,34 @@ import (
 	saferand "crypto/rand"
 	"encoding/csv"
 	"encoding/json"
-	"errors"
 	"fmt"
+	"io"
 	"math/big"
 	"os"
-	"slices"
 	"strings"
 	"time"
 
 	"github.com/AlecAivazis/survey/v2"
+	"github.com/enescakir/emoji"
 	"github.com/fatih/color"
 	"github.com/go-openapi/strfmt"
 	"github.com/google/uuid"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v3"
+	"golang.org/x/exp/slices"
+	"gopkg.in/yaml.v2"
 
 	"github.com/crowdsecurity/machineid"
 
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/database/ent"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
-const passwordLength = 64
+var (
+	passwordLength = 64
+)
 
 func generatePassword(length int) string {
 	upper := "ABCDEFGHIJKLMNOPQRSTUVWXY"
@@ -40,13 +42,11 @@ func generatePassword(length int) string {
 	charsetLength := len(charset)
 
 	buf := make([]byte, length)
-
 	for i := 0; i < length; i++ {
 		rInt, err := saferand.Int(saferand.Reader, big.NewInt(int64(charsetLength)))
 		if err != nil {
 			log.Fatalf("failed getting data from prng for password generation : %s", err)
 		}
-
 		buf[i] = charset[rInt.Int64()]
 	}
 
@@ -61,14 +61,12 @@ func generateIDPrefix() (string, error) {
 	if err == nil {
 		return prefix, nil
 	}
-
 	log.Debugf("failed to get machine-id with usual files: %s", err)
 
-	bID, err := uuid.NewRandom()
+	bId, err := uuid.NewRandom()
 	if err == nil {
-		return bID.String(), nil
+		return bId.String(), nil
 	}
-
 	return "", fmt.Errorf("generating machine id: %w", err)
 }
 
@@ -79,427 +77,356 @@ func generateID(prefix string) (string, error) {
 	if prefix == "" {
 		prefix, err = generateIDPrefix()
 	}
-
 	if err != nil {
 		return "", err
 	}
-
 	prefix = strings.ReplaceAll(prefix, "-", "")[:32]
 	suffix := generatePassword(16)
-
 	return prefix + suffix, nil
 }
 
-// getLastHeartbeat returns the last heartbeat timestamp of a machine
-// and a boolean indicating if the machine is considered active or not.
-func getLastHeartbeat(m *ent.Machine) (string, bool) {
-	if m.LastHeartbeat == nil {
-		return "-", false
+func displayLastHeartBeat(m *ent.Machine, fancy bool) string {
+	var hbDisplay string
+
+	if m.LastHeartbeat != nil {
+		lastHeartBeat := time.Now().UTC().Sub(*m.LastHeartbeat)
+		hbDisplay = lastHeartBeat.Truncate(time.Second).String()
+		if fancy && lastHeartBeat > 2*time.Minute {
+			hbDisplay = fmt.Sprintf("%s %s", emoji.Warning.String(), lastHeartBeat.Truncate(time.Second).String())
+		}
+	} else {
+		hbDisplay = "-"
+		if fancy {
+			hbDisplay = emoji.Warning.String() + " -"
+		}
 	}
-
-	elapsed := time.Now().UTC().Sub(*m.LastHeartbeat)
-
-	hb := elapsed.Truncate(time.Second).String()
-	if elapsed > 2*time.Minute {
-		return hb, false
-	}
-
-	return hb, true
+	return hbDisplay
 }
 
-type cliMachines struct {
-	db  *database.Client
-	cfg configGetter
-}
-
-func NewCLIMachines(cfg configGetter) *cliMachines {
-	return &cliMachines{
-		cfg: cfg,
+func getAgents(out io.Writer, dbClient *database.Client) error {
+	machines, err := dbClient.ListMachines()
+	if err != nil {
+		return fmt.Errorf("unable to list machines: %s", err)
 	}
-}
-
-func (cli *cliMachines) NewCommand() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:   "machines [action]",
-		Short: "Manage local API machines [requires local API]",
-		Long: `To list/add/delete/validate/prune machines.
-Note: This command requires database direct access, so is intended to be run on the local API machine.
-`,
-		Example:           `cscli machines [action]`,
-		DisableAutoGenTag: true,
-		Aliases:           []string{"machine"},
-		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
-			var err error
-			if err = require.LAPI(cli.cfg()); err != nil {
-				return err
+	if csConfig.Cscli.Output == "human" {
+		getAgentsTable(out, machines)
+	} else if csConfig.Cscli.Output == "json" {
+		enc := json.NewEncoder(out)
+		enc.SetIndent("", "  ")
+		if err := enc.Encode(machines); err != nil {
+			return fmt.Errorf("failed to marshal")
+		}
+		return nil
+	} else if csConfig.Cscli.Output == "raw" {
+		csvwriter := csv.NewWriter(out)
+		err := csvwriter.Write([]string{"machine_id", "ip_address", "updated_at", "validated", "version", "auth_type", "last_heartbeat"})
+		if err != nil {
+			return fmt.Errorf("failed to write header: %s", err)
+		}
+		for _, m := range machines {
+			var validated string
+			if m.IsValidated {
+				validated = "true"
+			} else {
+				validated = "false"
 			}
-			cli.db, err = database.NewClient(cli.cfg().DbConfig)
+			err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, false)})
 			if err != nil {
-				return fmt.Errorf("unable to create new database client: %w", err)
+				return fmt.Errorf("failed to write raw output : %s", err)
+			}
+		}
+		csvwriter.Flush()
+	} else {
+		log.Errorf("unknown output '%s'", csConfig.Cscli.Output)
+	}
+	return nil
+}
+
+func NewMachinesListCmd() *cobra.Command {
+	cmdMachinesList := &cobra.Command{
+		Use:               "list",
+		Short:             "List machines",
+		Long:              `List `,
+		Example:           `cscli machines list`,
+		Args:              cobra.MaximumNArgs(1),
+		DisableAutoGenTag: true,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			var err error
+			dbClient, err = database.NewClient(csConfig.DbConfig)
+			if err != nil {
+				return fmt.Errorf("unable to create new database client: %s", err)
+			}
+
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			err := getAgents(color.Output, dbClient)
+			if err != nil {
+				return fmt.Errorf("unable to list machines: %s", err)
 			}
 
 			return nil
 		},
 	}
 
-	cmd.AddCommand(cli.newListCmd())
-	cmd.AddCommand(cli.newAddCmd())
-	cmd.AddCommand(cli.newDeleteCmd())
-	cmd.AddCommand(cli.newValidateCmd())
-	cmd.AddCommand(cli.newPruneCmd())
-
-	return cmd
+	return cmdMachinesList
 }
 
-func (cli *cliMachines) list() error {
-	out := color.Output
-
-	machines, err := cli.db.ListMachines()
-	if err != nil {
-		return fmt.Errorf("unable to list machines: %w", err)
-	}
-
-	switch cli.cfg().Cscli.Output {
-	case "human":
-		getAgentsTable(out, machines)
-	case "json":
-		enc := json.NewEncoder(out)
-		enc.SetIndent("", "  ")
-
-		if err := enc.Encode(machines); err != nil {
-			return errors.New("failed to marshal")
-		}
-
-		return nil
-	case "raw":
-		csvwriter := csv.NewWriter(out)
-
-		err := csvwriter.Write([]string{"machine_id", "ip_address", "updated_at", "validated", "version", "auth_type", "last_heartbeat"})
-		if err != nil {
-			return fmt.Errorf("failed to write header: %w", err)
-		}
-
-		for _, m := range machines {
-			validated := "false"
-			if m.IsValidated {
-				validated = "true"
-			}
-
-			hb, _ := getLastHeartbeat(m)
-
-			if err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb}); err != nil {
-				return fmt.Errorf("failed to write raw output: %w", err)
-			}
-		}
-
-		csvwriter.Flush()
-	}
-
-	return nil
-}
-
-func (cli *cliMachines) newListCmd() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:               "list",
-		Short:             "list all machines in the database",
-		Long:              `list all machines in the database with their status and last heartbeat`,
-		Example:           `cscli machines list`,
-		Args:              cobra.NoArgs,
-		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			return cli.list()
-		},
-	}
-
-	return cmd
-}
-
-func (cli *cliMachines) newAddCmd() *cobra.Command {
-	var (
-		password    MachinePassword
-		dumpFile    string
-		apiURL      string
-		interactive bool
-		autoAdd     bool
-		force       bool
-	)
-
-	cmd := &cobra.Command{
+func NewMachinesAddCmd() *cobra.Command {
+	cmdMachinesAdd := &cobra.Command{
 		Use:               "add",
-		Short:             "add a single machine to the database",
+		Short:             "add machine to the database.",
 		DisableAutoGenTag: true,
 		Long:              `Register a new machine in the database. cscli should be on the same machine as LAPI.`,
-		Example: `cscli machines add --auto
+		Example: `
+cscli machines add --auto
 cscli machines add MyTestMachine --auto
 cscli machines add MyTestMachine --password MyPassword
-cscli machines add -f- --auto > /tmp/mycreds.yaml`,
-		RunE: func(_ *cobra.Command, args []string) error {
-			return cli.add(args, string(password), dumpFile, apiURL, interactive, autoAdd, force)
+`,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			var err error
+			dbClient, err = database.NewClient(csConfig.DbConfig)
+			if err != nil {
+				return fmt.Errorf("unable to create new database client: %s", err)
+			}
+
+			return nil
 		},
+		RunE: runMachinesAdd,
 	}
 
-	flags := cmd.Flags()
-	flags.VarP(&password, "password", "p", "machine password to login to the API")
-	flags.StringVarP(&dumpFile, "file", "f", "", "output file destination (defaults to "+csconfig.DefaultConfigPath("local_api_credentials.yaml")+")")
-	flags.StringVarP(&apiURL, "url", "u", "", "URL of the local API")
-	flags.BoolVarP(&interactive, "interactive", "i", false, "interfactive mode to enter the password")
-	flags.BoolVarP(&autoAdd, "auto", "a", false, "automatically generate password (and username if not provided)")
-	flags.BoolVar(&force, "force", false, "will force add the machine if it already exist")
+	flags := cmdMachinesAdd.Flags()
+	flags.StringP("password", "p", "", "machine password to login to the API")
+	flags.StringP("file", "f", "", "output file destination (defaults to "+csconfig.DefaultConfigPath("local_api_credentials.yaml")+")")
+	flags.StringP("url", "u", "", "URL of the local API")
+	flags.BoolP("interactive", "i", false, "interfactive mode to enter the password")
+	flags.BoolP("auto", "a", false, "automatically generate password (and username if not provided)")
+	flags.Bool("force", false, "will force add the machine if it already exist")
 
-	return cmd
+	return cmdMachinesAdd
 }
 
-func (cli *cliMachines) add(args []string, machinePassword string, dumpFile string, apiURL string, interactive bool, autoAdd bool, force bool) error {
-	var (
-		err       error
-		machineID string
-	)
+func runMachinesAdd(cmd *cobra.Command, args []string) error {
+	var dumpFile string
+	var err error
+
+	flags := cmd.Flags()
+
+	machinePassword, err := flags.GetString("password")
+	if err != nil {
+		return err
+	}
+
+	outputFile, err := flags.GetString("file")
+	if err != nil {
+		return err
+	}
+
+	apiURL, err := flags.GetString("url")
+	if err != nil {
+		return err
+	}
+
+	interactive, err := flags.GetBool("interactive")
+	if err != nil {
+		return err
+	}
+
+	autoAdd, err := flags.GetBool("auto")
+	if err != nil {
+		return err
+	}
+
+	forceAdd, err := flags.GetBool("force")
+	if err != nil {
+		return err
+	}
+
+	var machineID string
 
 	// create machineID if not specified by user
 	if len(args) == 0 {
 		if !autoAdd {
-			return errors.New("please specify a machine name to add, or use --auto")
+			printHelp(cmd)
+			return nil
 		}
-
 		machineID, err = generateID("")
 		if err != nil {
-			return fmt.Errorf("unable to generate machine id: %w", err)
+			return fmt.Errorf("unable to generate machine id: %s", err)
 		}
 	} else {
 		machineID = args[0]
 	}
 
-	clientCfg := cli.cfg().API.Client
-	serverCfg := cli.cfg().API.Server
-
 	/*check if file already exists*/
-	if dumpFile == "" && clientCfg != nil && clientCfg.CredentialsFilePath != "" {
-		credFile := clientCfg.CredentialsFilePath
-		// use the default only if the file does not exist
-		_, err = os.Stat(credFile)
-
-		switch {
-		case os.IsNotExist(err) || force:
-			dumpFile = credFile
-		case err != nil:
-			return fmt.Errorf("unable to stat '%s': %w", credFile, err)
-		default:
-			return fmt.Errorf(`credentials file '%s' already exists: please remove it, use "--force" or specify a different file with "-f" ("-f -" for standard output)`, credFile)
-		}
-	}
-
-	if dumpFile == "" {
-		return errors.New(`please specify a file to dump credentials to, with -f ("-f -" for standard output)`)
+	if outputFile != "" {
+		dumpFile = outputFile
+	} else if csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" {
+		dumpFile = csConfig.API.Client.CredentialsFilePath
 	}
 
 	// create a password if it's not specified by user
 	if machinePassword == "" && !interactive {
 		if !autoAdd {
-			return errors.New("please specify a password with --password or use --auto")
+			printHelp(cmd)
+			return nil
 		}
-
 		machinePassword = generatePassword(passwordLength)
 	} else if machinePassword == "" && interactive {
 		qs := &survey.Password{
-			Message: "Please provide a password for the machine:",
+			Message: "Please provide a password for the machine",
 		}
 		survey.AskOne(qs, &machinePassword)
 	}
-
 	password := strfmt.Password(machinePassword)
-
-	_, err = cli.db.CreateMachine(&machineID, &password, "", true, force, types.PasswordAuthType)
+	_, err = dbClient.CreateMachine(&machineID, &password, "", true, forceAdd, types.PasswordAuthType)
 	if err != nil {
-		return fmt.Errorf("unable to create machine: %w", err)
+		return fmt.Errorf("unable to create machine: %s", err)
 	}
-
-	fmt.Fprintf(os.Stderr, "Machine '%s' successfully added to the local API.\n", machineID)
+	log.Infof("Machine '%s' successfully added to the local API", machineID)
 
 	if apiURL == "" {
-		if clientCfg != nil && clientCfg.Credentials != nil && clientCfg.Credentials.URL != "" {
-			apiURL = clientCfg.Credentials.URL
-		} else if serverCfg.ClientURL() != "" {
-			apiURL = serverCfg.ClientURL()
+		if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" {
+			apiURL = csConfig.API.Client.Credentials.URL
+		} else if csConfig.API.Server != nil && csConfig.API.Server.ListenURI != "" {
+			apiURL = "http://" + csConfig.API.Server.ListenURI
 		} else {
-			return errors.New("unable to dump an api URL. Please provide it in your configuration or with the -u parameter")
+			return fmt.Errorf("unable to dump an api URL. Please provide it in your configuration or with the -u parameter")
 		}
 	}
-
 	apiCfg := csconfig.ApiCredentialsCfg{
 		Login:    machineID,
 		Password: password.String(),
 		URL:      apiURL,
 	}
-
 	apiConfigDump, err := yaml.Marshal(apiCfg)
 	if err != nil {
-		return fmt.Errorf("unable to marshal api credentials: %w", err)
+		return fmt.Errorf("unable to marshal api credentials: %s", err)
 	}
-
 	if dumpFile != "" && dumpFile != "-" {
-		if err = os.WriteFile(dumpFile, apiConfigDump, 0o600); err != nil {
-			return fmt.Errorf("write api credentials in '%s' failed: %w", dumpFile, err)
+		err = os.WriteFile(dumpFile, apiConfigDump, 0644)
+		if err != nil {
+			return fmt.Errorf("write api credentials in '%s' failed: %s", dumpFile, err)
 		}
-
-		fmt.Fprintf(os.Stderr, "API credentials written to '%s'.\n", dumpFile)
+		log.Printf("API credentials dumped to '%s'", dumpFile)
 	} else {
-		fmt.Print(string(apiConfigDump))
+		fmt.Printf("%s\n", string(apiConfigDump))
 	}
 
 	return nil
 }
 
-func (cli *cliMachines) deleteValid(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
-	machines, err := cli.db.ListMachines()
-	if err != nil {
-		cobra.CompError("unable to list machines " + err.Error())
+func NewMachinesDeleteCmd() *cobra.Command {
+	cmdMachinesDelete := &cobra.Command{
+		Use:               "delete [machine_name]...",
+		Short:             "delete machines",
+		Example:           `cscli machines delete "machine1" "machine2"`,
+		Args:              cobra.MinimumNArgs(1),
+		Aliases:           []string{"remove"},
+		DisableAutoGenTag: true,
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			var err error
+			dbClient, err = database.NewClient(csConfig.DbConfig)
+			if err != nil {
+				return fmt.Errorf("unable to create new database client: %s", err)
+			}
+			return nil
+		},
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			var err error
+			dbClient, err = getDBClient()
+			if err != nil {
+				cobra.CompError("unable to create new database client: " + err.Error())
+				return nil, cobra.ShellCompDirectiveNoFileComp
+			}
+			machines, err := dbClient.ListMachines()
+			if err != nil {
+				cobra.CompError("unable to list machines " + err.Error())
+			}
+			ret := make([]string, 0)
+			for _, machine := range machines {
+				if strings.Contains(machine.MachineId, toComplete) && !slices.Contains(args, machine.MachineId) {
+					ret = append(ret, machine.MachineId)
+				}
+			}
+			return ret, cobra.ShellCompDirectiveNoFileComp
+		},
+		RunE: runMachinesDelete,
 	}
 
-	ret := []string{}
-
-	for _, machine := range machines {
-		if strings.Contains(machine.MachineId, toComplete) && !slices.Contains(args, machine.MachineId) {
-			ret = append(ret, machine.MachineId)
-		}
-	}
-
-	return ret, cobra.ShellCompDirectiveNoFileComp
+	return cmdMachinesDelete
 }
 
-func (cli *cliMachines) delete(machines []string) error {
-	for _, machineID := range machines {
-		if err := cli.db.DeleteWatcher(machineID); err != nil {
+func runMachinesDelete(cmd *cobra.Command, args []string) error {
+	for _, machineID := range args {
+		err := dbClient.DeleteWatcher(machineID)
+		if err != nil {
 			log.Errorf("unable to delete machine '%s': %s", machineID, err)
 			return nil
 		}
-
 		log.Infof("machine '%s' deleted successfully", machineID)
 	}
 
 	return nil
 }
 
-func (cli *cliMachines) newDeleteCmd() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:               "delete [machine_name]...",
-		Short:             "delete machine(s) by name",
-		Example:           `cscli machines delete "machine1" "machine2"`,
-		Args:              cobra.MinimumNArgs(1),
-		Aliases:           []string{"remove"},
-		DisableAutoGenTag: true,
-		ValidArgsFunction: cli.deleteValid,
-		RunE: func(_ *cobra.Command, args []string) error {
-			return cli.delete(args)
-		},
-	}
-
-	return cmd
-}
-
-func (cli *cliMachines) prune(duration time.Duration, notValidOnly bool, force bool) error {
-	if duration < 2*time.Minute && !notValidOnly {
-		if yes, err := askYesNo(
-				"The duration you provided is less than 2 minutes. " +
-				"This can break installations if the machines are only temporarily disconnected. Continue?", false); err != nil {
-			return err
-		} else if !yes {
-			fmt.Println("User aborted prune. No changes were made.")
-			return nil
-		}
-	}
-
-	machines := []*ent.Machine{}
-	if pending, err := cli.db.QueryPendingMachine(); err == nil {
-		machines = append(machines, pending...)
-	}
-
-	if !notValidOnly {
-		if pending, err := cli.db.QueryLastValidatedHeartbeatLT(time.Now().UTC().Add(-duration)); err == nil {
-			machines = append(machines, pending...)
-		}
-	}
-
-	if len(machines) == 0 {
-		fmt.Println("No machines to prune.")
-		return nil
-	}
-
-	getAgentsTable(color.Output, machines)
-
-	if !force {
-		if yes, err := askYesNo(
-				"You are about to PERMANENTLY remove the above machines from the database. " +
-				"These will NOT be recoverable. Continue?", false); err != nil {
-			return err
-		} else if !yes {
-			fmt.Println("User aborted prune. No changes were made.")
-			return nil
-		}
-	}
-
-	deleted, err := cli.db.BulkDeleteWatchers(machines)
-	if err != nil {
-		return fmt.Errorf("unable to prune machines: %w", err)
-	}
-
-	fmt.Fprintf(os.Stderr, "successfully delete %d machines\n", deleted)
-
-	return nil
-}
-
-func (cli *cliMachines) newPruneCmd() *cobra.Command {
-	var (
-		duration     time.Duration
-		notValidOnly bool
-		force        bool
-	)
-
-	const defaultDuration = 10 * time.Minute
-
-	cmd := &cobra.Command{
-		Use:   "prune",
-		Short: "prune multiple machines from the database",
-		Long:  `prune multiple machines that are not validated or have not connected to the local API in a given duration.`,
-		Example: `cscli machines prune
-cscli machines prune --duration 1h
-cscli machines prune --not-validated-only --force`,
-		Args:              cobra.NoArgs,
-		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			return cli.prune(duration, notValidOnly, force)
-		},
-	}
-
-	flags := cmd.Flags()
-	flags.DurationVarP(&duration, "duration", "d", defaultDuration, "duration of time since validated machine last heartbeat")
-	flags.BoolVar(&notValidOnly, "not-validated-only", false, "only prune machines that are not validated")
-	flags.BoolVar(&force, "force", false, "force prune without asking for confirmation")
-
-	return cmd
-}
-
-func (cli *cliMachines) validate(machineID string) error {
-	if err := cli.db.ValidateMachine(machineID); err != nil {
-		return fmt.Errorf("unable to validate machine '%s': %w", machineID, err)
-	}
-
-	log.Infof("machine '%s' validated successfully", machineID)
-
-	return nil
-}
-
-func (cli *cliMachines) newValidateCmd() *cobra.Command {
-	cmd := &cobra.Command{
+func NewMachinesValidateCmd() *cobra.Command {
+	cmdMachinesValidate := &cobra.Command{
 		Use:               "validate",
 		Short:             "validate a machine to access the local API",
 		Long:              `validate a machine to access the local API.`,
 		Example:           `cscli machines validate "machine_name"`,
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, args []string) error {
-			return cli.validate(args[0])
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			var err error
+			dbClient, err = database.NewClient(csConfig.DbConfig)
+			if err != nil {
+				return fmt.Errorf("unable to create new database client: %s", err)
+			}
+
+			return nil
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			machineID := args[0]
+			if err := dbClient.ValidateMachine(machineID); err != nil {
+				return fmt.Errorf("unable to validate machine '%s': %s", machineID, err)
+			}
+			log.Infof("machine '%s' validated successfully", machineID)
+
+			return nil
 		},
 	}
 
-	return cmd
+	return cmdMachinesValidate
+}
+
+func NewMachinesCmd() *cobra.Command {
+	var cmdMachines = &cobra.Command{
+		Use:   "machines [action]",
+		Short: "Manage local API machines [requires local API]",
+		Long: `To list/add/delete/validate machines.
+Note: This command requires database direct access, so is intended to be run on the local API machine.
+`,
+		Example:           `cscli machines [action]`,
+		DisableAutoGenTag: true,
+		Aliases:           []string{"machine"},
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
+				if err != nil {
+					log.Errorf("local api : %s", err)
+				}
+				return fmt.Errorf("local API is disabled, please run this command on the local API machine")
+			}
+
+			return nil
+		},
+	}
+
+	cmdMachines.AddCommand(NewMachinesListCmd())
+	cmdMachines.AddCommand(NewMachinesAddCmd())
+	cmdMachines.AddCommand(NewMachinesDeleteCmd())
+	cmdMachines.AddCommand(NewMachinesValidateCmd())
+
+	return cmdMachines
 }

cmd/crowdsec-cli/machines_table.go

@@ -5,9 +5,9 @@ import (
 	"time"
 
 	"github.com/aquasecurity/table"
+	"github.com/enescakir/emoji"
 
 	"github.com/crowdsecurity/crowdsec/pkg/database/ent"
-	"github.com/crowdsecurity/crowdsec/pkg/emoji"
 )
 
 func getAgentsTable(out io.Writer, machines []*ent.Machine) {
@@ -17,17 +17,14 @@ func getAgentsTable(out io.Writer, machines []*ent.Machine) {
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	for _, m := range machines {
-		validated := emoji.Prohibited
+		var validated string
 		if m.IsValidated {
-			validated = emoji.CheckMark
+			validated = emoji.CheckMark.String()
+		} else {
+			validated = emoji.Prohibited.String()
 		}
 
-		hb, active := getLastHeartbeat(m)
-		if !active {
-			hb = emoji.Warning + " " + hb
-		}
-
-		t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb)
+		t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, true))
 	}
 
 	t.Render()

cmd/crowdsec-cli/main.go

@@ -3,111 +3,67 @@ package main
 import (
 	"fmt"
 	"os"
+	"path"
 	"path/filepath"
-	"slices"
-	"time"
+	"strings"
 
 	"github.com/fatih/color"
 	cc "github.com/ivanpirog/coloredcobra"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/go-cs-lib/trace"
+	"github.com/spf13/cobra/doc"
+	"golang.org/x/exp/slices"
 
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/fflag"
 )
 
-var (
-	ConfigFilePath string
-	csConfig       *csconfig.Config
-	dbClient       *database.Client
-)
+var trace_lvl, dbg_lvl, nfo_lvl, wrn_lvl, err_lvl bool
 
-type configGetter func() *csconfig.Config
+var ConfigFilePath string
+var csConfig *csconfig.Config
+var dbClient *database.Client
+
+var OutputFormat string
+var OutputColor string
+
+var downloadOnly bool
+var forceAction bool
+var purge bool
+var all bool
+
+var prometheusURL string
 
 var mergedConfig string
 
-type cliRoot struct {
-	logTrace     bool
-	logDebug     bool
-	logInfo      bool
-	logWarn      bool
-	logErr       bool
-	outputColor  string
-	outputFormat string
-	// flagBranch overrides the value in csConfig.Cscli.HubBranch
-	flagBranch string
-}
-
-func newCliRoot() *cliRoot {
-	return &cliRoot{}
-}
-
-// cfg() is a helper function to get the configuration loaded from config.yaml,
-// we pass it to subcommands because the file is not read until the Execute() call
-func (cli *cliRoot) cfg() *csconfig.Config {
-	return csConfig
-}
-
-// wantedLogLevel returns the log level requested in the command line flags.
-func (cli *cliRoot) wantedLogLevel() log.Level {
-	switch {
-	case cli.logTrace:
-		return log.TraceLevel
-	case cli.logDebug:
-		return log.DebugLevel
-	case cli.logInfo:
-		return log.InfoLevel
-	case cli.logWarn:
-		return log.WarnLevel
-	case cli.logErr:
-		return log.ErrorLevel
-	default:
-		return log.InfoLevel
-	}
-}
-
-// loadConfigFor loads the configuration file for the given sub-command.
-// If the sub-command does not need it, it returns a default configuration.
-func loadConfigFor(command string) (*csconfig.Config, string, error) {
-	noNeedConfig := []string{
-		"doc",
-		"help",
-		"completion",
-		"version",
-		"hubtest",
-	}
-
-	if !slices.Contains(noNeedConfig, command) {
-		log.Debugf("Using %s as configuration file", ConfigFilePath)
-
-		config, merged, err := csconfig.NewConfig(ConfigFilePath, false, false, true)
-		if err != nil {
-			return nil, "", err
-		}
-
-		// set up directory for trace files
-		if err := trace.Init(filepath.Join(config.ConfigPaths.DataDir, "trace")); err != nil {
-			return nil, "", fmt.Errorf("while setting up trace directory: %w", err)
-		}
-
-		return config, merged, nil
-	}
-
-	return csconfig.NewDefaultConfig(), "", nil
-}
-
-// initialize is called before the subcommand is executed.
-func (cli *cliRoot) initialize() {
+func initConfig() {
 	var err error
+	if trace_lvl {
+		log.SetLevel(log.TraceLevel)
+	} else if dbg_lvl {
+		log.SetLevel(log.DebugLevel)
+	} else if nfo_lvl {
+		log.SetLevel(log.InfoLevel)
+	} else if wrn_lvl {
+		log.SetLevel(log.WarnLevel)
+	} else if err_lvl {
+		log.SetLevel(log.ErrorLevel)
+	}
 
-	log.SetLevel(cli.wantedLogLevel())
-
-	csConfig, mergedConfig, err = loadConfigFor(os.Args[1])
-	if err != nil {
-		log.Fatal(err)
+	if !slices.Contains(NoNeedConfig, os.Args[1]) {
+		csConfig, mergedConfig, err = csconfig.NewConfig(ConfigFilePath, false, false, true)
+		if err != nil {
+			log.Fatal(err)
+		}
+		log.Debugf("Using %s as configuration file", ConfigFilePath)
+		if err := csConfig.LoadCSCLI(); err != nil {
+			log.Fatal(err)
+		}
+	} else {
+		csConfig = csconfig.NewDefaultConfig()
 	}
 
 	// recap of the enabled feature flags, because logging
@@ -116,22 +72,22 @@ func (cli *cliRoot) initialize() {
 		log.Debugf("Enabled feature flags: %s", fflist)
 	}
 
-	if cli.flagBranch != "" {
-		csConfig.Cscli.HubBranch = cli.flagBranch
+	if csConfig.Cscli == nil {
+		log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath)
 	}
 
-	if cli.outputFormat != "" {
-		csConfig.Cscli.Output = cli.outputFormat
+	if cwhub.HubBranch == "" && csConfig.Cscli.HubBranch != "" {
+		cwhub.HubBranch = csConfig.Cscli.HubBranch
+	}
+	if OutputFormat != "" {
+		csConfig.Cscli.Output = OutputFormat
+		if OutputFormat != "json" && OutputFormat != "raw" && OutputFormat != "human" {
+			log.Fatalf("output format %s unknown", OutputFormat)
+		}
 	}
-
 	if csConfig.Cscli.Output == "" {
 		csConfig.Cscli.Output = "human"
 	}
-
-	if csConfig.Cscli.Output != "human" && csConfig.Cscli.Output != "json" && csConfig.Cscli.Output != "raw" {
-		log.Fatalf("output format '%s' not supported: must be one of human, json, raw", csConfig.Cscli.Output)
-	}
-
 	if csConfig.Cscli.Output == "json" {
 		log.SetFormatter(&log.JSONFormatter{})
 		log.SetLevel(log.ErrorLevel)
@@ -139,44 +95,47 @@ func (cli *cliRoot) initialize() {
 		log.SetLevel(log.ErrorLevel)
 	}
 
-	if cli.outputColor != "" {
-		csConfig.Cscli.Color = cli.outputColor
-
-		if cli.outputColor != "yes" && cli.outputColor != "no" && cli.outputColor != "auto" {
-			log.Fatalf("output color %s unknown", cli.outputColor)
+	if OutputColor != "" {
+		csConfig.Cscli.Color = OutputColor
+		if OutputColor != "yes" && OutputColor != "no" && OutputColor != "auto" {
+			log.Fatalf("output color %s unknown", OutputColor)
 		}
 	}
 }
 
-// list of valid subcommands for the shell completion
 var validArgs = []string{
-	"alerts", "appsec-configs", "appsec-rules", "bouncers", "capi", "collections",
-	"completion", "config", "console", "contexts", "dashboard", "decisions", "explain",
-	"hub", "hubtest", "lapi", "machines", "metrics", "notifications", "parsers",
-	"postoverflows", "scenarios", "simulation", "support", "version",
+	"scenarios", "parsers", "collections", "capi", "lapi", "postoverflows", "machines",
+	"metrics", "bouncers", "alerts", "decisions", "simulation", "hub", "dashboard",
+	"config", "completion", "version", "console", "notifications", "support",
 }
 
-func (cli *cliRoot) colorize(cmd *cobra.Command) {
-	cc.Init(&cc.Config{
-		RootCmd:         cmd,
-		Headings:        cc.Yellow,
-		Commands:        cc.Green + cc.Bold,
-		CmdShortDescr:   cc.Cyan,
-		Example:         cc.Italic,
-		ExecName:        cc.Bold,
-		Aliases:         cc.Bold + cc.Italic,
-		FlagsDataType:   cc.White,
-		Flags:           cc.Green,
-		FlagsDescr:      cc.Cyan,
-		NoExtraNewlines: true,
-		NoBottomNewline: true,
-	})
-	cmd.SetOut(color.Output)
+func prepender(filename string) string {
+	const header = `---
+id: %s
+title: %s
+---
+`
+	name := filepath.Base(filename)
+	base := strings.TrimSuffix(name, path.Ext(name))
+	return fmt.Sprintf(header, base, strings.ReplaceAll(base, "_", " "))
 }
 
-func (cli *cliRoot) NewCommand() *cobra.Command {
+func linkHandler(name string) string {
+	return fmt.Sprintf("/cscli/%s", name)
+}
+
+var (
+	NoNeedConfig = []string{
+		"help",
+		"completion",
+		"version",
+		"hubtest",
+	}
+)
+
+func main() {
 	// set the formatter asap and worry about level later
-	logFormatter := &log.TextFormatter{TimestampFormat: time.RFC3339, FullTimestamp: true}
+	logFormatter := &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}
 	log.SetFormatter(logFormatter)
 
 	if err := fflag.RegisterAllFeatures(); err != nil {
@@ -187,7 +146,7 @@ func (cli *cliRoot) NewCommand() *cobra.Command {
 		log.Fatalf("failed to set feature flags from env: %s", err)
 	}
 
-	cmd := &cobra.Command{
+	var rootCmd = &cobra.Command{
 		Use:   "cscli",
 		Short: "cscli allows you to manage crowdsec",
 		Long: `cscli is the main command to interact with your crowdsec service, scenarios & db.
@@ -199,25 +158,57 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 		/*TBD examples*/
 	}
 
-	cli.colorize(cmd)
+	cc.Init(&cc.Config{
+		RootCmd:       rootCmd,
+		Headings:      cc.Yellow,
+		Commands:      cc.Green + cc.Bold,
+		CmdShortDescr: cc.Cyan,
+		Example:       cc.Italic,
+		ExecName:      cc.Bold,
+		Aliases:       cc.Bold + cc.Italic,
+		FlagsDataType: cc.White,
+		Flags:         cc.Green,
+		FlagsDescr:    cc.Cyan,
+	})
+	rootCmd.SetOut(color.Output)
 
-	/*don't sort flags so we can enforce order*/
-	cmd.Flags().SortFlags = false
+	var cmdDocGen = &cobra.Command{
+		Use:               "doc",
+		Short:             "Generate the documentation in `./doc/`. Directory must exist.",
+		Args:              cobra.ExactArgs(0),
+		Hidden:            true,
+		DisableAutoGenTag: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if err := doc.GenMarkdownTreeCustom(rootCmd, "./doc/", prepender, linkHandler); err != nil {
+				return fmt.Errorf("Failed to generate cobra doc: %s", err)
+			}
+			return nil
+		},
+	}
+	rootCmd.AddCommand(cmdDocGen)
+	/*usage*/
+	var cmdVersion = &cobra.Command{
+		Use:               "version",
+		Short:             "Display version and exit.",
+		Args:              cobra.ExactArgs(0),
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			cwversion.Show()
+		},
+	}
+	rootCmd.AddCommand(cmdVersion)
 
-	pflags := cmd.PersistentFlags()
-	pflags.SortFlags = false
+	rootCmd.PersistentFlags().StringVarP(&ConfigFilePath, "config", "c", csconfig.DefaultConfigPath("config.yaml"), "path to crowdsec config file")
+	rootCmd.PersistentFlags().StringVarP(&OutputFormat, "output", "o", "", "Output format: human, json, raw.")
+	rootCmd.PersistentFlags().StringVarP(&OutputColor, "color", "", "auto", "Output color: yes, no, auto.")
+	rootCmd.PersistentFlags().BoolVar(&dbg_lvl, "debug", false, "Set logging to debug.")
+	rootCmd.PersistentFlags().BoolVar(&nfo_lvl, "info", false, "Set logging to info.")
+	rootCmd.PersistentFlags().BoolVar(&wrn_lvl, "warning", false, "Set logging to warning.")
+	rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error.")
+	rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace.")
 
-	pflags.StringVarP(&ConfigFilePath, "config", "c", csconfig.DefaultConfigPath("config.yaml"), "path to crowdsec config file")
-	pflags.StringVarP(&cli.outputFormat, "output", "o", "", "Output format: human, json, raw")
-	pflags.StringVarP(&cli.outputColor, "color", "", "auto", "Output color: yes, no, auto")
-	pflags.BoolVar(&cli.logDebug, "debug", false, "Set logging to debug")
-	pflags.BoolVar(&cli.logInfo, "info", false, "Set logging to info")
-	pflags.BoolVar(&cli.logWarn, "warning", false, "Set logging to warning")
-	pflags.BoolVar(&cli.logErr, "error", false, "Set logging to error")
-	pflags.BoolVar(&cli.logTrace, "trace", false, "Set logging to trace")
-	pflags.StringVar(&cli.flagBranch, "branch", "", "Override hub branch on github")
-
-	if err := pflags.MarkHidden("branch"); err != nil {
+	rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "", "Override hub branch on github")
+	if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil {
 		log.Fatalf("failed to hide flag: %s", err)
 	}
 
@@ -237,47 +228,45 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	}
 
 	if len(os.Args) > 1 {
-		cobra.OnInitialize(cli.initialize)
+		cobra.OnInitialize(initConfig)
 	}
 
-	cmd.AddCommand(NewCLIDoc().NewCommand(cmd))
-	cmd.AddCommand(NewCLIVersion().NewCommand())
-	cmd.AddCommand(NewCLIConfig(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLIHub(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLIMetrics(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLIDashboard(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLIDecisions(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLIAlerts(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLISimulation(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLIBouncers(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLIMachines(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLICapi(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLILapi(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCompletionCmd())
-	cmd.AddCommand(NewCLIConsole(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLIExplain(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLIHubTest(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLINotifications(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLISupport().NewCommand())
-	cmd.AddCommand(NewCLIPapi(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLICollection(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLIParser(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLIScenario(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLIPostOverflow(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLIContext(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLIAppsecConfig(cli.cfg).NewCommand())
-	cmd.AddCommand(NewCLIAppsecRule(cli.cfg).NewCommand())
+	/*don't sort flags so we can enforce order*/
+	rootCmd.Flags().SortFlags = false
+	rootCmd.PersistentFlags().SortFlags = false
+
+	rootCmd.AddCommand(NewConfigCmd())
+	rootCmd.AddCommand(NewHubCmd())
+	rootCmd.AddCommand(NewMetricsCmd())
+	rootCmd.AddCommand(NewDashboardCmd())
+	rootCmd.AddCommand(NewDecisionsCmd())
+	rootCmd.AddCommand(NewAlertsCmd())
+	rootCmd.AddCommand(NewSimulationCmds())
+	rootCmd.AddCommand(NewBouncersCmd())
+	rootCmd.AddCommand(NewMachinesCmd())
+	rootCmd.AddCommand(NewParsersCmd())
+	rootCmd.AddCommand(NewScenariosCmd())
+	rootCmd.AddCommand(NewCollectionsCmd())
+	rootCmd.AddCommand(NewPostOverflowsCmd())
+	rootCmd.AddCommand(NewCapiCmd())
+	rootCmd.AddCommand(NewLapiCmd())
+	rootCmd.AddCommand(NewCompletionCmd())
+	rootCmd.AddCommand(NewConsoleCmd())
+	rootCmd.AddCommand(NewExplainCmd())
+	rootCmd.AddCommand(NewHubTestCmd())
+	rootCmd.AddCommand(NewNotificationsCmd())
+	rootCmd.AddCommand(NewSupportCmd())
+	rootCmd.AddCommand(NewWafRulesCmd())
 
 	if fflag.CscliSetup.IsEnabled() {
-		cmd.AddCommand(NewSetupCmd())
+		rootCmd.AddCommand(NewSetupCmd())
 	}
 
-	return cmd
-}
+	if fflag.PapiClient.IsEnabled() {
+		rootCmd.AddCommand(NewPapiCmd())
+	}
 
-func main() {
-	cmd := newCliRoot().NewCommand()
-	if err := cmd.Execute(); err != nil {
+	if err := rootCmd.Execute(); err != nil {
 		log.Fatal(err)
 	}
 }

cmd/crowdsec-cli/metrics.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -15,65 +14,13 @@ import (
 	"github.com/prometheus/prom2json"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v3"
+	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/maptools"
-	"github.com/crowdsecurity/go-cs-lib/trace"
+	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
 )
 
-type (
-	statAcquis       map[string]map[string]int
-	statParser       map[string]map[string]int
-	statBucket       map[string]map[string]int
-	statWhitelist    map[string]map[string]map[string]int
-	statLapi         map[string]map[string]int
-	statLapiMachine  map[string]map[string]map[string]int
-	statLapiBouncer  map[string]map[string]map[string]int
-	statLapiDecision map[string]struct {
-		NonEmpty int
-		Empty    int
-	}
-	statDecision     map[string]map[string]map[string]int
-	statAppsecEngine map[string]map[string]int
-	statAppsecRule   map[string]map[string]map[string]int
-	statAlert        map[string]int
-	statStash        map[string]struct {
-		Type  string
-		Count int
-	}
-)
-
-var (
-	ErrMissingConfig   = errors.New("prometheus section missing, can't show metrics")
-	ErrMetricsDisabled = errors.New("prometheus is not enabled, can't show metrics")
-)
-
-type metricSection interface {
-	Table(out io.Writer, noUnit bool, showEmpty bool)
-	Description() (string, string)
-}
-
-type metricStore map[string]metricSection
-
-func NewMetricStore() metricStore {
-	return metricStore{
-		"acquisition":    statAcquis{},
-		"scenarios":      statBucket{},
-		"parsers":        statParser{},
-		"lapi":           statLapi{},
-		"lapi-machine":   statLapiMachine{},
-		"lapi-bouncer":   statLapiBouncer{},
-		"lapi-decisions": statLapiDecision{},
-		"decisions":      statDecision{},
-		"alerts":         statAlert{},
-		"stash":          statStash{},
-		"appsec-engine":  statAppsecEngine{},
-		"appsec-rule":    statAppsecRule{},
-		"whitelists":     statWhitelist{},
-	}
-}
-
-func (ms metricStore) Fetch(url string) error {
+// FormatPrometheusMetrics is a complete rip from prom2json
+func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error {
 	mfChan := make(chan *dto.MetricFamily, 1024)
 	errChan := make(chan error, 1)
 
@@ -86,10 +33,9 @@ func (ms metricStore) Fetch(url string) error {
 	transport.ResponseHeaderTimeout = time.Minute
 	go func() {
 		defer trace.CatchPanic("crowdsec/ShowPrometheus")
-
 		err := prom2json.FetchMetricFamilies(url, mfChan, transport)
 		if err != nil {
-			errChan <- fmt.Errorf("failed to fetch metrics: %w", err)
+			errChan <- fmt.Errorf("failed to fetch prometheus metrics: %w", err)
 			return
 		}
 		errChan <- nil
@@ -104,42 +50,40 @@ func (ms metricStore) Fetch(url string) error {
 		return err
 	}
 
-	log.Debugf("Finished reading metrics output, %d entries", len(result))
+	log.Debugf("Finished reading prometheus output, %d entries", len(result))
 	/*walk*/
-
-	mAcquis := ms["acquisition"].(statAcquis)
-	mParser := ms["parsers"].(statParser)
-	mBucket := ms["scenarios"].(statBucket)
-	mLapi := ms["lapi"].(statLapi)
-	mLapiMachine := ms["lapi-machine"].(statLapiMachine)
-	mLapiBouncer := ms["lapi-bouncer"].(statLapiBouncer)
-	mLapiDecision := ms["lapi-decisions"].(statLapiDecision)
-	mDecision := ms["decisions"].(statDecision)
-	mAppsecEngine := ms["appsec-engine"].(statAppsecEngine)
-	mAppsecRule := ms["appsec-rule"].(statAppsecRule)
-	mAlert := ms["alerts"].(statAlert)
-	mStash := ms["stash"].(statStash)
-	mWhitelist := ms["whitelists"].(statWhitelist)
+	lapi_decisions_stats := map[string]struct {
+		NonEmpty int
+		Empty    int
+	}{}
+	acquis_stats := map[string]map[string]int{}
+	parsers_stats := map[string]map[string]int{}
+	buckets_stats := map[string]map[string]int{}
+	lapi_stats := map[string]map[string]int{}
+	lapi_machine_stats := map[string]map[string]map[string]int{}
+	lapi_bouncer_stats := map[string]map[string]map[string]int{}
+	decisions_stats := map[string]map[string]map[string]int{}
+	alerts_stats := map[string]int{}
+	stash_stats := map[string]struct {
+		Type  string
+		Count int
+	}{}
 
 	for idx, fam := range result {
 		if !strings.HasPrefix(fam.Name, "cs_") {
 			continue
 		}
-
 		log.Tracef("round %d", idx)
-
 		for _, m := range fam.Metrics {
 			metric, ok := m.(prom2json.Metric)
 			if !ok {
 				log.Debugf("failed to convert metric to prom2json.Metric")
 				continue
 			}
-
 			name, ok := metric.Labels["name"]
 			if !ok {
 				log.Debugf("no name in Metric %v", metric.Labels)
 			}
-
 			source, ok := metric.Labels["source"]
 			if !ok {
 				log.Debugf("no source in Metric %v for %s", metric.Labels, fam.Name)
@@ -160,338 +104,212 @@ func (ms metricStore) Fetch(url string) error {
 			origin := metric.Labels["origin"]
 			action := metric.Labels["action"]
 
-			appsecEngine := metric.Labels["appsec_engine"]
-			appsecRule := metric.Labels["rule_name"]
-
 			mtype := metric.Labels["type"]
 
 			fval, err := strconv.ParseFloat(value, 32)
 			if err != nil {
 				log.Errorf("Unexpected int value %s : %s", value, err)
 			}
-
 			ival := int(fval)
-
 			switch fam.Name {
-			//
-			// buckets
-			//
+			/*buckets*/
 			case "cs_bucket_created_total":
-				mBucket.Process(name, "instantiation", ival)
+				if _, ok := buckets_stats[name]; !ok {
+					buckets_stats[name] = make(map[string]int)
+				}
+				buckets_stats[name]["instantiation"] += ival
 			case "cs_buckets":
-				mBucket.Process(name, "curr_count", ival)
+				if _, ok := buckets_stats[name]; !ok {
+					buckets_stats[name] = make(map[string]int)
+				}
+				buckets_stats[name]["curr_count"] += ival
 			case "cs_bucket_overflowed_total":
-				mBucket.Process(name, "overflow", ival)
+				if _, ok := buckets_stats[name]; !ok {
+					buckets_stats[name] = make(map[string]int)
+				}
+				buckets_stats[name]["overflow"] += ival
 			case "cs_bucket_poured_total":
-				mBucket.Process(name, "pour", ival)
-				mAcquis.Process(source, "pour", ival)
+				if _, ok := buckets_stats[name]; !ok {
+					buckets_stats[name] = make(map[string]int)
+				}
+				if _, ok := acquis_stats[source]; !ok {
+					acquis_stats[source] = make(map[string]int)
+				}
+				buckets_stats[name]["pour"] += ival
+				acquis_stats[source]["pour"] += ival
 			case "cs_bucket_underflowed_total":
-				mBucket.Process(name, "underflow", ival)
-			//
-			// parsers
-			//
+				if _, ok := buckets_stats[name]; !ok {
+					buckets_stats[name] = make(map[string]int)
+				}
+				buckets_stats[name]["underflow"] += ival
+				/*acquis*/
 			case "cs_parser_hits_total":
-				mAcquis.Process(source, "reads", ival)
+				if _, ok := acquis_stats[source]; !ok {
+					acquis_stats[source] = make(map[string]int)
+				}
+				acquis_stats[source]["reads"] += ival
 			case "cs_parser_hits_ok_total":
-				mAcquis.Process(source, "parsed", ival)
+				if _, ok := acquis_stats[source]; !ok {
+					acquis_stats[source] = make(map[string]int)
+				}
+				acquis_stats[source]["parsed"] += ival
 			case "cs_parser_hits_ko_total":
-				mAcquis.Process(source, "unparsed", ival)
+				if _, ok := acquis_stats[source]; !ok {
+					acquis_stats[source] = make(map[string]int)
+				}
+				acquis_stats[source]["unparsed"] += ival
 			case "cs_node_hits_total":
-				mParser.Process(name, "hits", ival)
+				if _, ok := parsers_stats[name]; !ok {
+					parsers_stats[name] = make(map[string]int)
+				}
+				parsers_stats[name]["hits"] += ival
 			case "cs_node_hits_ok_total":
-				mParser.Process(name, "parsed", ival)
+				if _, ok := parsers_stats[name]; !ok {
+					parsers_stats[name] = make(map[string]int)
+				}
+				parsers_stats[name]["parsed"] += ival
 			case "cs_node_hits_ko_total":
-				mParser.Process(name, "unparsed", ival)
-			//
-			// whitelists
-			//
-			case "cs_node_wl_hits_total":
-				mWhitelist.Process(name, reason, "hits", ival)
-			case "cs_node_wl_hits_ok_total":
-				mWhitelist.Process(name, reason, "whitelisted", ival)
-				// track as well whitelisted lines at acquis level
-				mAcquis.Process(source, "whitelisted", ival)
-			//
-			// lapi
-			//
+				if _, ok := parsers_stats[name]; !ok {
+					parsers_stats[name] = make(map[string]int)
+				}
+				parsers_stats[name]["unparsed"] += ival
 			case "cs_lapi_route_requests_total":
-				mLapi.Process(route, method, ival)
+				if _, ok := lapi_stats[route]; !ok {
+					lapi_stats[route] = make(map[string]int)
+				}
+				lapi_stats[route][method] += ival
 			case "cs_lapi_machine_requests_total":
-				mLapiMachine.Process(machine, route, method, ival)
+				if _, ok := lapi_machine_stats[machine]; !ok {
+					lapi_machine_stats[machine] = make(map[string]map[string]int)
+				}
+				if _, ok := lapi_machine_stats[machine][route]; !ok {
+					lapi_machine_stats[machine][route] = make(map[string]int)
+				}
+				lapi_machine_stats[machine][route][method] += ival
 			case "cs_lapi_bouncer_requests_total":
-				mLapiBouncer.Process(bouncer, route, method, ival)
+				if _, ok := lapi_bouncer_stats[bouncer]; !ok {
+					lapi_bouncer_stats[bouncer] = make(map[string]map[string]int)
+				}
+				if _, ok := lapi_bouncer_stats[bouncer][route]; !ok {
+					lapi_bouncer_stats[bouncer][route] = make(map[string]int)
+				}
+				lapi_bouncer_stats[bouncer][route][method] += ival
 			case "cs_lapi_decisions_ko_total", "cs_lapi_decisions_ok_total":
-				mLapiDecision.Process(bouncer, fam.Name, ival)
-			//
-			// decisions
-			//
+				if _, ok := lapi_decisions_stats[bouncer]; !ok {
+					lapi_decisions_stats[bouncer] = struct {
+						NonEmpty int
+						Empty    int
+					}{}
+				}
+				x := lapi_decisions_stats[bouncer]
+				if fam.Name == "cs_lapi_decisions_ko_total" {
+					x.Empty += ival
+				} else if fam.Name == "cs_lapi_decisions_ok_total" {
+					x.NonEmpty += ival
+				}
+				lapi_decisions_stats[bouncer] = x
 			case "cs_active_decisions":
-				mDecision.Process(reason, origin, action, ival)
+				if _, ok := decisions_stats[reason]; !ok {
+					decisions_stats[reason] = make(map[string]map[string]int)
+				}
+				if _, ok := decisions_stats[reason][origin]; !ok {
+					decisions_stats[reason][origin] = make(map[string]int)
+				}
+				decisions_stats[reason][origin][action] += ival
 			case "cs_alerts":
-				mAlert.Process(reason, ival)
-			//
-			// stash
-			//
+				/*if _, ok := alerts_stats[scenario]; !ok {
+					alerts_stats[scenario] = make(map[string]int)
+				}*/
+				alerts_stats[reason] += ival
 			case "cs_cache_size":
-				mStash.Process(name, mtype, ival)
-			//
-			// appsec
-			//
-			case "cs_appsec_reqs_total":
-				mAppsecEngine.Process(appsecEngine, "processed", ival)
-			case "cs_appsec_block_total":
-				mAppsecEngine.Process(appsecEngine, "blocked", ival)
-			case "cs_appsec_rule_hits":
-				mAppsecRule.Process(appsecEngine, appsecRule, "triggered", ival)
+				stash_stats[name] = struct {
+					Type  string
+					Count int
+				}{Type: mtype, Count: ival}
 			default:
-				log.Debugf("unknown: %+v", fam.Name)
 				continue
 			}
+
 		}
 	}
 
+	if formatType == "human" {
+		acquisStatsTable(out, acquis_stats)
+		bucketStatsTable(out, buckets_stats)
+		parserStatsTable(out, parsers_stats)
+		lapiStatsTable(out, lapi_stats)
+		lapiMachineStatsTable(out, lapi_machine_stats)
+		lapiBouncerStatsTable(out, lapi_bouncer_stats)
+		lapiDecisionStatsTable(out, lapi_decisions_stats)
+		decisionStatsTable(out, decisions_stats)
+		alertStatsTable(out, alerts_stats)
+		stashStatsTable(out, stash_stats)
+	} else if formatType == "json" {
+		for _, val := range []interface{}{acquis_stats, parsers_stats, buckets_stats, lapi_stats, lapi_bouncer_stats, lapi_machine_stats, lapi_decisions_stats, decisions_stats, alerts_stats, stash_stats} {
+			x, err := json.MarshalIndent(val, "", " ")
+			if err != nil {
+				return fmt.Errorf("failed to unmarshal metrics : %v", err)
+			}
+			out.Write(x)
+		}
+		return nil
+
+	} else if formatType == "raw" {
+		for _, val := range []interface{}{acquis_stats, parsers_stats, buckets_stats, lapi_stats, lapi_bouncer_stats, lapi_machine_stats, lapi_decisions_stats, decisions_stats, alerts_stats, stash_stats} {
+			x, err := yaml.Marshal(val)
+			if err != nil {
+				return fmt.Errorf("failed to unmarshal metrics : %v", err)
+			}
+			out.Write(x)
+		}
+		return nil
+	}
 	return nil
 }
 
-type cliMetrics struct {
-	cfg configGetter
-}
+var noUnit bool
 
-func NewCLIMetrics(cfg configGetter) *cliMetrics {
-	return &cliMetrics{
-		cfg: cfg,
-	}
-}
 
-func (ms metricStore) Format(out io.Writer, sections []string, formatType string, noUnit bool) error {
-	// copy only the sections we want
-	want := map[string]metricSection{}
-
-	// if explicitly asking for sections, we want to show empty tables
-	showEmpty := len(sections) > 0
-
-	// if no sections are specified, we want all of them
-	if len(sections) == 0 {
-		sections = maptools.SortedKeys(ms)
+func runMetrics(cmd *cobra.Command, args []string) error {
+	if err := csConfig.LoadPrometheus(); err != nil {
+		return fmt.Errorf("failed to load prometheus config: %w", err)
 	}
 
-	for _, section := range sections {
-		want[section] = ms[section]
+	if csConfig.Prometheus == nil {
+		return fmt.Errorf("prometheus section missing, can't show metrics")
 	}
 
-	switch formatType {
-	case "human":
-		for _, section := range maptools.SortedKeys(want) {
-			want[section].Table(out, noUnit, showEmpty)
-		}
-	case "json":
-		x, err := json.MarshalIndent(want, "", " ")
-		if err != nil {
-			return fmt.Errorf("failed to marshal metrics: %w", err)
-		}
-		out.Write(x)
-	case "raw":
-		x, err := yaml.Marshal(want)
-		if err != nil {
-			return fmt.Errorf("failed to marshal metrics: %w", err)
-		}
-		out.Write(x)
-	default:
-		return fmt.Errorf("unknown format type %s", formatType)
+	if !csConfig.Prometheus.Enabled {
+		return fmt.Errorf("prometheus is not enabled, can't show metrics")
 	}
 
+	if prometheusURL == "" {
+		prometheusURL = csConfig.Cscli.PrometheusUrl
+	}
+
+	if prometheusURL == "" {
+		return fmt.Errorf("no prometheus url, please specify in %s or via -u", *csConfig.FilePath)
+	}
+
+	err := FormatPrometheusMetrics(color.Output, prometheusURL+"/metrics", csConfig.Cscli.Output)
+	if err != nil {
+		return fmt.Errorf("could not fetch prometheus metrics: %w", err)
+	}
 	return nil
 }
 
-func (cli *cliMetrics) show(sections []string, url string, noUnit bool) error {
-	cfg := cli.cfg()
 
-	if url != "" {
-		cfg.Cscli.PrometheusUrl = url
-	}
-
-	if cfg.Prometheus == nil {
-		return ErrMissingConfig
-	}
-
-	if !cfg.Prometheus.Enabled {
-		return ErrMetricsDisabled
-	}
-
-	ms := NewMetricStore()
-
-	if err := ms.Fetch(cfg.Cscli.PrometheusUrl); err != nil {
-		return err
-	}
-
-	// any section that we don't have in the store is an error
-	for _, section := range sections {
-		if _, ok := ms[section]; !ok {
-			return fmt.Errorf("unknown metrics type: %s", section)
-		}
-	}
-
-	if err := ms.Format(color.Output, sections, cfg.Cscli.Output, noUnit); err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (cli *cliMetrics) NewCommand() *cobra.Command {
-	var (
-		url    string
-		noUnit bool
-	)
-
-	cmd := &cobra.Command{
-		Use:   "metrics",
-		Short: "Display crowdsec prometheus metrics.",
-		Long:  `Fetch metrics from a Local API server and display them`,
-		Example: `# Show all Metrics, skip empty tables (same as "cecli metrics show")
-cscli metrics
-
-# Show only some metrics, connect to a different url
-cscli metrics --url http://lapi.local:6060/metrics show acquisition parsers
-
-# List available metric types
-cscli metrics list`,
+func NewMetricsCmd() *cobra.Command {
+	cmdMetrics := &cobra.Command{
+		Use:               "metrics",
+		Short:             "Display crowdsec prometheus metrics.",
+		Long:              `Fetch metrics from the prometheus server and display them in a human-friendly way`,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			return cli.show(nil, url, noUnit)
-		},
+		RunE: runMetrics,
 	}
+	cmdMetrics.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
+	cmdMetrics.PersistentFlags().BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
 
-	flags := cmd.Flags()
-	flags.StringVarP(&url, "url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
-	flags.BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
-
-	cmd.AddCommand(cli.newShowCmd())
-	cmd.AddCommand(cli.newListCmd())
-
-	return cmd
-}
-
-// expandAlias returns a list of sections. The input can be a list of sections or alias.
-func (cli *cliMetrics) expandAlias(args []string) []string {
-	ret := []string{}
-
-	for _, section := range args {
-		switch section {
-		case "engine":
-			ret = append(ret, "acquisition", "parsers", "scenarios", "stash", "whitelists")
-		case "lapi":
-			ret = append(ret, "alerts", "decisions", "lapi", "lapi-bouncer", "lapi-decisions", "lapi-machine")
-		case "appsec":
-			ret = append(ret, "appsec-engine", "appsec-rule")
-		default:
-			ret = append(ret, section)
-		}
-	}
-
-	return ret
-}
-
-func (cli *cliMetrics) newShowCmd() *cobra.Command {
-	var (
-		url    string
-		noUnit bool
-	)
-
-	cmd := &cobra.Command{
-		Use:   "show [type]...",
-		Short: "Display all or part of the available metrics.",
-		Long:  `Fetch metrics from a Local API server and display them, optionally filtering on specific types.`,
-		Example: `# Show all Metrics, skip empty tables
-cscli metrics show
-
-# Use an alias: "engine", "lapi" or "appsec" to show a group of metrics
-cscli metrics show engine
-
-# Show some specific metrics, show empty tables, connect to a different url
-cscli metrics show acquisition parsers scenarios stash --url http://lapi.local:6060/metrics
-
-# To list available metric types, use "cscli metrics list"
-cscli metrics list; cscli metrics list -o json
-
-# Show metrics in json format
-cscli metrics show acquisition parsers scenarios stash -o json`,
-		// Positional args are optional
-		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, args []string) error {
-			args = cli.expandAlias(args)
-			return cli.show(args, url, noUnit)
-		},
-	}
-
-	flags := cmd.Flags()
-	flags.StringVarP(&url, "url", "u", "", "Metrics url (http://<ip>:<port>/metrics)")
-	flags.BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
-
-	return cmd
-}
-
-func (cli *cliMetrics) list() error {
-	type metricType struct {
-		Type        string `json:"type"        yaml:"type"`
-		Title       string `json:"title"       yaml:"title"`
-		Description string `json:"description" yaml:"description"`
-	}
-
-	var allMetrics []metricType
-
-	ms := NewMetricStore()
-	for _, section := range maptools.SortedKeys(ms) {
-		title, description := ms[section].Description()
-		allMetrics = append(allMetrics, metricType{
-			Type:        section,
-			Title:       title,
-			Description: description,
-		})
-	}
-
-	switch cli.cfg().Cscli.Output {
-	case "human":
-		t := newTable(color.Output)
-		t.SetRowLines(true)
-		t.SetHeaders("Type", "Title", "Description")
-
-		for _, metric := range allMetrics {
-			t.AddRow(metric.Type, metric.Title, metric.Description)
-		}
-
-		t.Render()
-	case "json":
-		x, err := json.MarshalIndent(allMetrics, "", " ")
-		if err != nil {
-			return fmt.Errorf("failed to marshal metric types: %w", err)
-		}
-
-		fmt.Println(string(x))
-	case "raw":
-		x, err := yaml.Marshal(allMetrics)
-		if err != nil {
-			return fmt.Errorf("failed to marshal metric types: %w", err)
-		}
-
-		fmt.Println(string(x))
-	}
-
-	return nil
-}
-
-func (cli *cliMetrics) newListCmd() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:               "list",
-		Short:             "List available types of metrics.",
-		Long:              `List available types of metrics.`,
-		Args:              cobra.ExactArgs(0),
-		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			return cli.list()
-		},
-	}
-
-	return cmd
+	return cmdMetrics
 }

cmd/crowdsec-cli/metrics_table.go

@@ -1,33 +1,25 @@
 package main
 
 import (
-	"errors"
 	"fmt"
 	"io"
 	"sort"
-	"strconv"
 
 	"github.com/aquasecurity/table"
 	log "github.com/sirupsen/logrus"
-
-	"github.com/crowdsecurity/go-cs-lib/maptools"
 )
 
-// ErrNilTable means a nil pointer was passed instead of a table instance. This is a programming error.
-var ErrNilTable = errors.New("nil table")
-
 func lapiMetricsToTable(t *table.Table, stats map[string]map[string]map[string]int) int {
 	// stats: machine -> route -> method -> count
+
 	// sort keys to keep consistent order when printing
 	machineKeys := []string{}
 	for k := range stats {
 		machineKeys = append(machineKeys, k)
 	}
-
 	sort.Strings(machineKeys)
 
 	numRows := 0
-
 	for _, machine := range machineKeys {
 		// oneRow: route -> method -> count
 		machineRow := stats[machine]
@@ -39,79 +31,41 @@ func lapiMetricsToTable(t *table.Table, stats map[string]map[string]map[string]i
 					methodName,
 				}
 				if count != 0 {
-					row = append(row, strconv.Itoa(count))
+					row = append(row, fmt.Sprintf("%d", count))
 				} else {
 					row = append(row, "-")
 				}
-
 				t.AddRow(row...)
-
 				numRows++
 			}
 		}
 	}
-
 	return numRows
 }
 
-func wlMetricsToTable(t *table.Table, stats map[string]map[string]map[string]int, noUnit bool) (int, error) {
+func metricsToTable(t *table.Table, stats map[string]map[string]int, keys []string) (int, error) {
 	if t == nil {
-		return 0, ErrNilTable
+		return 0, fmt.Errorf("nil table")
 	}
+	// sort keys to keep consistent order when printing
+	sortedKeys := []string{}
+	for k := range stats {
+		sortedKeys = append(sortedKeys, k)
+	}
+	sort.Strings(sortedKeys)
 
 	numRows := 0
-
-	for _, name := range maptools.SortedKeys(stats) {
-		for _, reason := range maptools.SortedKeys(stats[name]) {
-			row := []string{
-				name,
-				reason,
-				"-",
-				"-",
-			}
-
-			for _, action := range maptools.SortedKeys(stats[name][reason]) {
-				value := stats[name][reason][action]
-
-				switch action {
-				case "whitelisted":
-					row[3] = strconv.Itoa(value)
-				case "hits":
-					row[2] = strconv.Itoa(value)
-				default:
-					log.Debugf("unexpected counter '%s' for whitelists = %d", action, value)
-				}
-			}
-
-			t.AddRow(row...)
-
-			numRows++
-		}
-	}
-
-	return numRows, nil
-}
-
-func metricsToTable(t *table.Table, stats map[string]map[string]int, keys []string, noUnit bool) (int, error) {
-	if t == nil {
-		return 0, ErrNilTable
-	}
-
-	numRows := 0
-
-	for _, alabel := range maptools.SortedKeys(stats) {
+	for _, alabel := range sortedKeys {
 		astats, ok := stats[alabel]
 		if !ok {
 			continue
 		}
-
 		row := []string{
 			alabel,
 		}
-
 		for _, sl := range keys {
 			if v, ok := astats[sl]; ok && v != 0 {
-				numberToShow := strconv.Itoa(v)
+				numberToShow := fmt.Sprintf("%d", v)
 				if !noUnit {
 					numberToShow = formatNumber(v)
 				}
@@ -121,192 +75,45 @@ func metricsToTable(t *table.Table, stats map[string]map[string]int, keys []stri
 				row = append(row, "-")
 			}
 		}
-
 		t.AddRow(row...)
-
 		numRows++
 	}
-
 	return numRows, nil
 }
 
-func (s statBucket) Description() (string, string) {
-	return "Scenario Metrics",
-		`Measure events in different scenarios. Current count is the number of buckets during metrics collection. ` +
-			`Overflows are past event-producing buckets, while Expired are the ones that didn’t receive enough events to Overflow.`
-}
-
-func (s statBucket) Process(bucket, metric string, val int) {
-	if _, ok := s[bucket]; !ok {
-		s[bucket] = make(map[string]int)
-	}
-
-	s[bucket][metric] += val
-}
-
-func (s statBucket) Table(out io.Writer, noUnit bool, showEmpty bool) {
+func bucketStatsTable(out io.Writer, stats map[string]map[string]int) {
 	t := newTable(out)
 	t.SetRowLines(false)
-	t.SetHeaders("Scenario", "Current Count", "Overflows", "Instantiated", "Poured", "Expired")
+	t.SetHeaders("Bucket", "Current Count", "Overflows", "Instantiated", "Poured", "Expired")
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	keys := []string{"curr_count", "overflow", "instantiation", "pour", "underflow"}
 
-	if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
-		log.Warningf("while collecting scenario stats: %s", err)
-	} else if numRows > 0 || showEmpty {
-		title, _ := s.Description()
-		renderTableTitle(out, "\n"+title+":")
+	if numRows, err := metricsToTable(t, stats, keys); err != nil {
+		log.Warningf("while collecting acquis stats: %s", err)
+	} else if numRows > 0 {
+		renderTableTitle(out, "\nBucket Metrics:")
 		t.Render()
 	}
 }
 
-func (s statAcquis) Description() (string, string) {
-	return "Acquisition Metrics",
-		`Measures the lines read, parsed, and unparsed per datasource. ` +
-			`Zero read lines indicate a misconfigured or inactive datasource. ` +
-			`Zero parsed lines mean the parser(s) failed. ` +
-			`Non-zero parsed lines are fine as crowdsec selects relevant lines.`
-}
-
-func (s statAcquis) Process(source, metric string, val int) {
-	if _, ok := s[source]; !ok {
-		s[source] = make(map[string]int)
-	}
-
-	s[source][metric] += val
-}
-
-func (s statAcquis) Table(out io.Writer, noUnit bool, showEmpty bool) {
+func acquisStatsTable(out io.Writer, stats map[string]map[string]int) {
 	t := newTable(out)
 	t.SetRowLines(false)
-	t.SetHeaders("Source", "Lines read", "Lines parsed", "Lines unparsed", "Lines poured to bucket", "Lines whitelisted")
+	t.SetHeaders("Source", "Lines read", "Lines parsed", "Lines unparsed", "Lines poured to bucket")
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
-	keys := []string{"reads", "parsed", "unparsed", "pour", "whitelisted"}
+	keys := []string{"reads", "parsed", "unparsed", "pour"}
 
-	if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
+	if numRows, err := metricsToTable(t, stats, keys); err != nil {
 		log.Warningf("while collecting acquis stats: %s", err)
-	} else if numRows > 0 || showEmpty {
-		title, _ := s.Description()
-		renderTableTitle(out, "\n"+title+":")
+	} else if numRows > 0 {
+		renderTableTitle(out, "\nAcquisition Metrics:")
 		t.Render()
 	}
 }
 
-func (s statAppsecEngine) Description() (string, string) {
-	return "Appsec Metrics",
-		`Measures the number of parsed and blocked requests by the AppSec Component.`
-}
-
-func (s statAppsecEngine) Process(appsecEngine, metric string, val int) {
-	if _, ok := s[appsecEngine]; !ok {
-		s[appsecEngine] = make(map[string]int)
-	}
-
-	s[appsecEngine][metric] += val
-}
-
-func (s statAppsecEngine) Table(out io.Writer, noUnit bool, showEmpty bool) {
-	t := newTable(out)
-	t.SetRowLines(false)
-	t.SetHeaders("Appsec Engine", "Processed", "Blocked")
-	t.SetAlignment(table.AlignLeft, table.AlignLeft)
-
-	keys := []string{"processed", "blocked"}
-
-	if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
-		log.Warningf("while collecting appsec stats: %s", err)
-	} else if numRows > 0 || showEmpty {
-		title, _ := s.Description()
-		renderTableTitle(out, "\n"+title+":")
-		t.Render()
-	}
-}
-
-func (s statAppsecRule) Description() (string, string) {
-	return "Appsec Rule Metrics",
-		`Provides “per AppSec Component” information about the number of matches for loaded AppSec Rules.`
-}
-
-func (s statAppsecRule) Process(appsecEngine, appsecRule string, metric string, val int) {
-	if _, ok := s[appsecEngine]; !ok {
-		s[appsecEngine] = make(map[string]map[string]int)
-	}
-
-	if _, ok := s[appsecEngine][appsecRule]; !ok {
-		s[appsecEngine][appsecRule] = make(map[string]int)
-	}
-
-	s[appsecEngine][appsecRule][metric] += val
-}
-
-func (s statAppsecRule) Table(out io.Writer, noUnit bool, showEmpty bool) {
-	for appsecEngine, appsecEngineRulesStats := range s {
-		t := newTable(out)
-		t.SetRowLines(false)
-		t.SetHeaders("Rule ID", "Triggered")
-		t.SetAlignment(table.AlignLeft, table.AlignLeft)
-
-		keys := []string{"triggered"}
-
-		if numRows, err := metricsToTable(t, appsecEngineRulesStats, keys, noUnit); err != nil {
-			log.Warningf("while collecting appsec rules stats: %s", err)
-		} else if numRows > 0 || showEmpty {
-			renderTableTitle(out, fmt.Sprintf("\nAppsec '%s' Rules Metrics:", appsecEngine))
-			t.Render()
-		}
-	}
-}
-
-func (s statWhitelist) Description() (string, string) {
-	return "Whitelist Metrics",
-		`Tracks the number of events processed and possibly whitelisted by each parser whitelist.`
-}
-
-func (s statWhitelist) Process(whitelist, reason, metric string, val int) {
-	if _, ok := s[whitelist]; !ok {
-		s[whitelist] = make(map[string]map[string]int)
-	}
-
-	if _, ok := s[whitelist][reason]; !ok {
-		s[whitelist][reason] = make(map[string]int)
-	}
-
-	s[whitelist][reason][metric] += val
-}
-
-func (s statWhitelist) Table(out io.Writer, noUnit bool, showEmpty bool) {
-	t := newTable(out)
-	t.SetRowLines(false)
-	t.SetHeaders("Whitelist", "Reason", "Hits", "Whitelisted")
-	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
-
-	if numRows, err := wlMetricsToTable(t, s, noUnit); err != nil {
-		log.Warningf("while collecting parsers stats: %s", err)
-	} else if numRows > 0 || showEmpty {
-		title, _ := s.Description()
-		renderTableTitle(out, "\n"+title+":")
-		t.Render()
-	}
-}
-
-func (s statParser) Description() (string, string) {
-	return "Parser Metrics",
-		`Tracks the number of events processed by each parser and indicates success of failure. ` +
-			`Zero parsed lines means the parer(s) failed. ` +
-			`Non-zero unparsed lines are fine as crowdsec select relevant lines.`
-}
-
-func (s statParser) Process(parser, metric string, val int) {
-	if _, ok := s[parser]; !ok {
-		s[parser] = make(map[string]int)
-	}
-
-	s[parser][metric] += val
-}
-
-func (s statParser) Table(out io.Writer, noUnit bool, showEmpty bool) {
+func parserStatsTable(out io.Writer, stats map[string]map[string]int) {
 	t := newTable(out)
 	t.SetRowLines(false)
 	t.SetHeaders("Parsers", "Hits", "Parsed", "Unparsed")
@@ -314,302 +121,187 @@ func (s statParser) Table(out io.Writer, noUnit bool, showEmpty bool) {
 
 	keys := []string{"hits", "parsed", "unparsed"}
 
-	if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
-		log.Warningf("while collecting parsers stats: %s", err)
-	} else if numRows > 0 || showEmpty {
-		title, _ := s.Description()
-		renderTableTitle(out, "\n"+title+":")
+	if numRows, err := metricsToTable(t, stats, keys); err != nil {
+		log.Warningf("while collecting acquis stats: %s", err)
+	} else if numRows > 0 {
+		renderTableTitle(out, "\nParser Metrics:")
 		t.Render()
 	}
 }
 
-func (s statStash) Description() (string, string) {
-	return "Parser Stash Metrics",
-		`Tracks the status of stashes that might be created by various parsers and scenarios.`
-}
+func stashStatsTable(out io.Writer, stats map[string]struct {
+	Type  string
+	Count int
+}) {
 
-func (s statStash) Process(name, mtype string, val int) {
-	s[name] = struct {
-		Type  string
-		Count int
-	}{
-		Type:  mtype,
-		Count: val,
-	}
-}
-
-func (s statStash) Table(out io.Writer, noUnit bool, showEmpty bool) {
 	t := newTable(out)
 	t.SetRowLines(false)
 	t.SetHeaders("Name", "Type", "Items")
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	// unfortunately, we can't reuse metricsToTable as the structure is too different :/
-	numRows := 0
+	sortedKeys := []string{}
+	for k := range stats {
+		sortedKeys = append(sortedKeys, k)
+	}
+	sort.Strings(sortedKeys)
 
-	for _, alabel := range maptools.SortedKeys(s) {
-		astats := s[alabel]
+	numRows := 0
+	for _, alabel := range sortedKeys {
+		astats := stats[alabel]
 
 		row := []string{
 			alabel,
 			astats.Type,
-			strconv.Itoa(astats.Count),
+			fmt.Sprintf("%d", astats.Count),
 		}
 		t.AddRow(row...)
-
 		numRows++
 	}
-
-	if numRows > 0 || showEmpty {
-		title, _ := s.Description()
-		renderTableTitle(out, "\n"+title+":")
+	if numRows > 0 {
+		renderTableTitle(out, "\nParser Stash Metrics:")
 		t.Render()
 	}
 }
 
-func (s statLapi) Description() (string, string) {
-	return "Local API Metrics",
-		`Monitors the requests made to local API routes.`
-}
-
-func (s statLapi) Process(route, method string, val int) {
-	if _, ok := s[route]; !ok {
-		s[route] = make(map[string]int)
-	}
-
-	s[route][method] += val
-}
-
-func (s statLapi) Table(out io.Writer, noUnit bool, showEmpty bool) {
+func lapiStatsTable(out io.Writer, stats map[string]map[string]int) {
 	t := newTable(out)
 	t.SetRowLines(false)
 	t.SetHeaders("Route", "Method", "Hits")
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	// unfortunately, we can't reuse metricsToTable as the structure is too different :/
-	numRows := 0
+	sortedKeys := []string{}
+	for k := range stats {
+		sortedKeys = append(sortedKeys, k)
+	}
+	sort.Strings(sortedKeys)
 
-	for _, alabel := range maptools.SortedKeys(s) {
-		astats := s[alabel]
+	numRows := 0
+	for _, alabel := range sortedKeys {
+		astats := stats[alabel]
 
 		subKeys := []string{}
 		for skey := range astats {
 			subKeys = append(subKeys, skey)
 		}
-
 		sort.Strings(subKeys)
 
 		for _, sl := range subKeys {
 			row := []string{
 				alabel,
 				sl,
-				strconv.Itoa(astats[sl]),
+				fmt.Sprintf("%d", astats[sl]),
 			}
-
 			t.AddRow(row...)
-
 			numRows++
 		}
 	}
 
-	if numRows > 0 || showEmpty {
-		title, _ := s.Description()
-		renderTableTitle(out, "\n"+title+":")
+	if numRows > 0 {
+		renderTableTitle(out, "\nLocal Api Metrics:")
 		t.Render()
 	}
 }
 
-func (s statLapiMachine) Description() (string, string) {
-	return "Local API Machines Metrics",
-		`Tracks the number of calls to the local API from each registered machine.`
-}
-
-func (s statLapiMachine) Process(machine, route, method string, val int) {
-	if _, ok := s[machine]; !ok {
-		s[machine] = make(map[string]map[string]int)
-	}
-
-	if _, ok := s[machine][route]; !ok {
-		s[machine][route] = make(map[string]int)
-	}
-
-	s[machine][route][method] += val
-}
-
-func (s statLapiMachine) Table(out io.Writer, noUnit bool, showEmpty bool) {
+func lapiMachineStatsTable(out io.Writer, stats map[string]map[string]map[string]int) {
 	t := newTable(out)
 	t.SetRowLines(false)
 	t.SetHeaders("Machine", "Route", "Method", "Hits")
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
-	numRows := lapiMetricsToTable(t, s)
+	numRows := lapiMetricsToTable(t, stats)
 
-	if numRows > 0 || showEmpty {
-		title, _ := s.Description()
-		renderTableTitle(out, "\n"+title+":")
+	if numRows > 0 {
+		renderTableTitle(out, "\nLocal Api Machines Metrics:")
 		t.Render()
 	}
 }
 
-func (s statLapiBouncer) Description() (string, string) {
-	return "Local API Bouncers Metrics",
-		`Tracks total hits to remediation component related API routes.`
-}
-
-func (s statLapiBouncer) Process(bouncer, route, method string, val int) {
-	if _, ok := s[bouncer]; !ok {
-		s[bouncer] = make(map[string]map[string]int)
-	}
-
-	if _, ok := s[bouncer][route]; !ok {
-		s[bouncer][route] = make(map[string]int)
-	}
-
-	s[bouncer][route][method] += val
-}
-
-func (s statLapiBouncer) Table(out io.Writer, noUnit bool, showEmpty bool) {
+func lapiBouncerStatsTable(out io.Writer, stats map[string]map[string]map[string]int) {
 	t := newTable(out)
 	t.SetRowLines(false)
 	t.SetHeaders("Bouncer", "Route", "Method", "Hits")
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
-	numRows := lapiMetricsToTable(t, s)
+	numRows := lapiMetricsToTable(t, stats)
 
-	if numRows > 0 || showEmpty {
-		title, _ := s.Description()
-		renderTableTitle(out, "\n"+title+":")
+	if numRows > 0 {
+		renderTableTitle(out, "\nLocal Api Bouncers Metrics:")
 		t.Render()
 	}
 }
 
-func (s statLapiDecision) Description() (string, string) {
-	return "Local API Bouncers Decisions",
-		`Tracks the number of empty/non-empty answers from LAPI to bouncers that are working in "live" mode.`
-}
-
-func (s statLapiDecision) Process(bouncer, fam string, val int) {
-	if _, ok := s[bouncer]; !ok {
-		s[bouncer] = struct {
-			NonEmpty int
-			Empty    int
-		}{}
-	}
-
-	x := s[bouncer]
-
-	switch fam {
-	case "cs_lapi_decisions_ko_total":
-		x.Empty += val
-	case "cs_lapi_decisions_ok_total":
-		x.NonEmpty += val
-	}
-
-	s[bouncer] = x
-}
-
-func (s statLapiDecision) Table(out io.Writer, noUnit bool, showEmpty bool) {
+func lapiDecisionStatsTable(out io.Writer, stats map[string]struct {
+	NonEmpty int
+	Empty    int
+},
+) {
 	t := newTable(out)
 	t.SetRowLines(false)
 	t.SetHeaders("Bouncer", "Empty answers", "Non-empty answers")
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	numRows := 0
-
-	for bouncer, hits := range s {
+	for bouncer, hits := range stats {
 		t.AddRow(
 			bouncer,
-			strconv.Itoa(hits.Empty),
-			strconv.Itoa(hits.NonEmpty),
+			fmt.Sprintf("%d", hits.Empty),
+			fmt.Sprintf("%d", hits.NonEmpty),
 		)
-
 		numRows++
 	}
 
-	if numRows > 0 || showEmpty {
-		title, _ := s.Description()
-		renderTableTitle(out, "\n"+title+":")
+	if numRows > 0 {
+		renderTableTitle(out, "\nLocal Api Bouncers Decisions:")
 		t.Render()
 	}
 }
 
-func (s statDecision) Description() (string, string) {
-	return "Local API Decisions",
-		`Provides information about all currently active decisions. ` +
-			`Includes both local (crowdsec) and global decisions (CAPI), and lists subscriptions (lists).`
-}
-
-func (s statDecision) Process(reason, origin, action string, val int) {
-	if _, ok := s[reason]; !ok {
-		s[reason] = make(map[string]map[string]int)
-	}
-
-	if _, ok := s[reason][origin]; !ok {
-		s[reason][origin] = make(map[string]int)
-	}
-
-	s[reason][origin][action] += val
-}
-
-func (s statDecision) Table(out io.Writer, noUnit bool, showEmpty bool) {
+func decisionStatsTable(out io.Writer, stats map[string]map[string]map[string]int) {
 	t := newTable(out)
 	t.SetRowLines(false)
 	t.SetHeaders("Reason", "Origin", "Action", "Count")
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
 	numRows := 0
-
-	for reason, origins := range s {
+	for reason, origins := range stats {
 		for origin, actions := range origins {
 			for action, hits := range actions {
 				t.AddRow(
 					reason,
 					origin,
 					action,
-					strconv.Itoa(hits),
+					fmt.Sprintf("%d", hits),
 				)
-
 				numRows++
 			}
 		}
 	}
 
-	if numRows > 0 || showEmpty {
-		title, _ := s.Description()
-		renderTableTitle(out, "\n"+title+":")
+	if numRows > 0 {
+		renderTableTitle(out, "\nLocal Api Decisions:")
 		t.Render()
 	}
 }
 
-func (s statAlert) Description() (string, string) {
-	return "Local API Alerts",
-		`Tracks the total number of past and present alerts for the installed scenarios.`
-}
-
-func (s statAlert) Process(reason string, val int) {
-	s[reason] += val
-}
-
-func (s statAlert) Table(out io.Writer, noUnit bool, showEmpty bool) {
+func alertStatsTable(out io.Writer, stats map[string]int) {
 	t := newTable(out)
 	t.SetRowLines(false)
 	t.SetHeaders("Reason", "Count")
 	t.SetAlignment(table.AlignLeft, table.AlignLeft)
 
 	numRows := 0
-
-	for scenario, hits := range s {
+	for scenario, hits := range stats {
 		t.AddRow(
 			scenario,
-			strconv.Itoa(hits),
+			fmt.Sprintf("%d", hits),
 		)
-
 		numRows++
 	}
 
-	if numRows > 0 || showEmpty {
-		title, _ := s.Description()
-		renderTableTitle(out, "\n"+title+":")
+	if numRows > 0 {
+		renderTableTitle(out, "\nLocal Api Alerts:")
 		t.Render()
 	}
 }

cmd/crowdsec-cli/notifications.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"encoding/csv"
 	"encoding/json"
-	"errors"
 	"fmt"
 	"io/fs"
 	"net/url"
@@ -16,171 +15,148 @@ import (
 
 	"github.com/fatih/color"
 	"github.com/go-openapi/strfmt"
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"gopkg.in/tomb.v2"
-	"gopkg.in/yaml.v3"
 
-	"github.com/crowdsecurity/go-cs-lib/ptr"
-	"github.com/crowdsecurity/go-cs-lib/version"
+	"github.com/crowdsecurity/go-cs-lib/pkg/version"
 
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/csplugin"
 	"github.com/crowdsecurity/crowdsec/pkg/csprofiles"
-	"github.com/crowdsecurity/crowdsec/pkg/models"
-	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
+
 type NotificationsCfg struct {
 	Config   csplugin.PluginConfig  `json:"plugin_config"`
 	Profiles []*csconfig.ProfileCfg `json:"associated_profiles"`
 	ids      []uint
 }
 
-type cliNotifications struct {
-	cfg configGetter
-}
 
-func NewCLINotifications(cfg configGetter) *cliNotifications {
-	return &cliNotifications{
-		cfg: cfg,
-	}
-}
-
-func (cli *cliNotifications) NewCommand() *cobra.Command {
-	cmd := &cobra.Command{
+func NewNotificationsCmd() *cobra.Command {
+	var cmdNotifications = &cobra.Command{
 		Use:               "notifications [action]",
 		Short:             "Helper for notification plugin configuration",
 		Long:              "To list/inspect/test notification template",
 		Args:              cobra.MinimumNArgs(1),
 		Aliases:           []string{"notifications", "notification"},
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
-			cfg := cli.cfg()
-			if err := require.LAPI(cfg); err != nil {
-				return err
+		PersistentPreRun: func(cmd *cobra.Command, args []string) {
+			var (
+				err error
+			)
+			if err = csConfig.API.Server.LoadProfiles(); err != nil {
+				log.Fatal(err)
 			}
-			if err := cfg.LoadAPIClient(); err != nil {
-				return fmt.Errorf("loading api client: %w", err)
+			if csConfig.ConfigPaths.NotificationDir == "" {
+				log.Fatalf("config_paths.notification_dir is not set in crowdsec config")
 			}
-			if err := require.Notifications(cfg); err != nil {
-				return err
-			}
-
-			return nil
 		},
 	}
 
-	cmd.AddCommand(cli.NewListCmd())
-	cmd.AddCommand(cli.NewInspectCmd())
-	cmd.AddCommand(cli.NewReinjectCmd())
-	cmd.AddCommand(cli.NewTestCmd())
 
-	return cmd
+	cmdNotifications.AddCommand(NewNotificationsListCmd())
+	cmdNotifications.AddCommand(NewNotificationsInspectCmd())
+	cmdNotifications.AddCommand(NewNotificationsReinjectCmd())
+
+	return cmdNotifications
 }
 
-func (cli *cliNotifications) getPluginConfigs() (map[string]csplugin.PluginConfig, error) {
-	cfg := cli.cfg()
+
+func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
 	pcfgs := map[string]csplugin.PluginConfig{}
 	wf := func(path string, info fs.FileInfo, err error) error {
 		if info == nil {
-			return fmt.Errorf("error while traversing directory %s: %w", path, err)
+			return errors.Wrapf(err, "error while traversing directory %s", path)
 		}
-
-		name := filepath.Join(cfg.ConfigPaths.NotificationDir, info.Name()) // Avoid calling info.Name() twice
+		name := filepath.Join(csConfig.ConfigPaths.NotificationDir, info.Name()) //Avoid calling info.Name() twice
 		if (strings.HasSuffix(name, "yaml") || strings.HasSuffix(name, "yml")) && !(info.IsDir()) {
 			ts, err := csplugin.ParsePluginConfigFile(name)
 			if err != nil {
-				return fmt.Errorf("loading notifification plugin configuration with %s: %w", name, err)
+				return errors.Wrapf(err, "Loading notifification plugin configuration with %s", name)
 			}
-
 			for _, t := range ts {
-				csplugin.SetRequiredFields(&t)
 				pcfgs[t.Name] = t
 			}
 		}
-
 		return nil
 	}
 
-	if err := filepath.Walk(cfg.ConfigPaths.NotificationDir, wf); err != nil {
-		return nil, fmt.Errorf("while loading notifification plugin configuration: %w", err)
+	if err := filepath.Walk(csConfig.ConfigPaths.NotificationDir, wf); err != nil {
+		return nil, errors.Wrap(err, "Loading notifification plugin configuration")
 	}
 
-	return pcfgs, nil
-}
-
-func (cli *cliNotifications) getProfilesConfigs() (map[string]NotificationsCfg, error) {
-	cfg := cli.cfg()
 	// A bit of a tricky stuf now: reconcile profiles and notification plugins
-	pcfgs, err := cli.getPluginConfigs()
-	if err != nil {
-		return nil, err
-	}
-
 	ncfgs := map[string]NotificationsCfg{}
-	for _, pc := range pcfgs {
-		ncfgs[pc.Name] = NotificationsCfg{
-			Config: pc,
-		}
-	}
-
-	profiles, err := csprofiles.NewProfile(cfg.API.Server.Profiles)
+	profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
 	if err != nil {
-		return nil, fmt.Errorf("while extracting profiles from configuration: %w", err)
+		return nil, errors.Wrap(err, "Cannot extract profiles from configuration")
 	}
-
 	for profileID, profile := range profiles {
+	loop:
 		for _, notif := range profile.Cfg.Notifications {
-			pc, ok := pcfgs[notif]
-			if !ok {
-				return nil, fmt.Errorf("notification plugin '%s' does not exist", notif)
+			for name, pc := range pcfgs {
+				if notif == name {
+					if _, ok := ncfgs[pc.Name]; !ok {
+						ncfgs[pc.Name] = NotificationsCfg{
+							Config:   pc,
+							Profiles: []*csconfig.ProfileCfg{profile.Cfg},
+							ids:      []uint{uint(profileID)},
+						}
+						continue loop
+					}
+					tmp := ncfgs[pc.Name]
+					for _, pr := range tmp.Profiles {
+						var profiles []*csconfig.ProfileCfg
+						if pr.Name == profile.Cfg.Name {
+							continue
+						}
+						profiles = append(tmp.Profiles, profile.Cfg)
+						ids := append(tmp.ids, uint(profileID))
+						ncfgs[pc.Name] = NotificationsCfg{
+							Config:   tmp.Config,
+							Profiles: profiles,
+							ids:      ids,
+						}
+					}
+				}
 			}
-
-			tmp, ok := ncfgs[pc.Name]
-			if !ok {
-				return nil, fmt.Errorf("notification plugin '%s' does not exist", pc.Name)
-			}
-
-			tmp.Profiles = append(tmp.Profiles, profile.Cfg)
-			tmp.ids = append(tmp.ids, uint(profileID))
-			ncfgs[pc.Name] = tmp
 		}
 	}
-
 	return ncfgs, nil
 }
 
-func (cli *cliNotifications) NewListCmd() *cobra.Command {
-	cmd := &cobra.Command{
+
+func NewNotificationsListCmd() *cobra.Command {
+	var cmdNotificationsList = &cobra.Command{
 		Use:               "list",
-		Short:             "list active notifications plugins",
-		Long:              `list active notifications plugins`,
+		Short:             "List active notifications plugins",
+		Long:              `List active notifications plugins`,
 		Example:           `cscli notifications list`,
 		Args:              cobra.ExactArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
-			cfg := cli.cfg()
-			ncfgs, err := cli.getProfilesConfigs()
+		RunE: func(cmd *cobra.Command, arg []string) error {
+			ncfgs, err := getNotificationsConfiguration()
 			if err != nil {
-				return fmt.Errorf("can't build profiles configuration: %w", err)
+				return errors.Wrap(err, "Can't build profiles configuration")
 			}
 
-			if cfg.Cscli.Output == "human" {
+			if csConfig.Cscli.Output == "human" {
 				notificationListTable(color.Output, ncfgs)
-			} else if cfg.Cscli.Output == "json" {
+			} else if csConfig.Cscli.Output == "json" {
 				x, err := json.MarshalIndent(ncfgs, "", " ")
 				if err != nil {
-					return fmt.Errorf("failed to marshal notification configuration: %w", err)
+					return errors.New("failed to marshal notification configuration")
 				}
 				fmt.Printf("%s", string(x))
-			} else if cfg.Cscli.Output == "raw" {
+			} else if csConfig.Cscli.Output == "raw" {
 				csvwriter := csv.NewWriter(os.Stdout)
 				err := csvwriter.Write([]string{"Name", "Type", "Profile name"})
 				if err != nil {
-					return fmt.Errorf("failed to write raw header: %w", err)
+					return errors.Wrap(err, "failed to write raw header")
 				}
 				for _, b := range ncfgs {
 					profilesList := []string{}
@@ -189,193 +165,140 @@ func (cli *cliNotifications) NewListCmd() *cobra.Command {
 					}
 					err := csvwriter.Write([]string{b.Config.Name, b.Config.Type, strings.Join(profilesList, ", ")})
 					if err != nil {
-						return fmt.Errorf("failed to write raw content: %w", err)
+						return errors.Wrap(err, "failed to write raw content")
 					}
 				}
 				csvwriter.Flush()
 			}
-
 			return nil
 		},
 	}
 
-	return cmd
+	return cmdNotificationsList
 }
 
-func (cli *cliNotifications) NewInspectCmd() *cobra.Command {
-	cmd := &cobra.Command{
+
+func NewNotificationsInspectCmd() *cobra.Command {
+	var cmdNotificationsInspect = &cobra.Command{
 		Use:               "inspect",
 		Short:             "Inspect active notifications plugin configuration",
 		Long:              `Inspect active notifications plugin and show configuration`,
 		Example:           `cscli notifications inspect <plugin_name>`,
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, args []string) error {
-			cfg := cli.cfg()
-			ncfgs, err := cli.getProfilesConfigs()
+		RunE: func(cmd *cobra.Command, arg []string) error {
+			var (
+				cfg NotificationsCfg
+				ok  bool
+			)
+
+			pluginName := arg[0]
+
+			if pluginName == "" {
+				errors.New("Please provide a plugin name to inspect")
+			}
+			ncfgs, err := getNotificationsConfiguration()
 			if err != nil {
-				return fmt.Errorf("can't build profiles configuration: %w", err)
+				return errors.Wrap(err, "Can't build profiles configuration")
 			}
-			ncfg, ok := ncfgs[args[0]]
-			if !ok {
-				return fmt.Errorf("plugin '%s' does not exist or is not active", args[0])
+			if cfg, ok = ncfgs[pluginName]; !ok {
+				return errors.New("The provided plugin name doesn't exist or isn't active")
 			}
-			if cfg.Cscli.Output == "human" || cfg.Cscli.Output == "raw" {
-				fmt.Printf(" - %15s: %15s\n", "Type", ncfg.Config.Type)
-				fmt.Printf(" - %15s: %15s\n", "Name", ncfg.Config.Name)
-				fmt.Printf(" - %15s: %15s\n", "Timeout", ncfg.Config.TimeOut)
-				fmt.Printf(" - %15s: %15s\n", "Format", ncfg.Config.Format)
-				for k, v := range ncfg.Config.Config {
+
+			if csConfig.Cscli.Output == "human" || csConfig.Cscli.Output == "raw" {
+				fmt.Printf(" - %15s: %15s\n", "Type", cfg.Config.Type)
+				fmt.Printf(" - %15s: %15s\n", "Name", cfg.Config.Name)
+				fmt.Printf(" - %15s: %15s\n", "Timeout", cfg.Config.TimeOut)
+				fmt.Printf(" - %15s: %15s\n", "Format", cfg.Config.Format)
+				for k, v := range cfg.Config.Config {
 					fmt.Printf(" - %15s: %15v\n", k, v)
 				}
-			} else if cfg.Cscli.Output == "json" {
+			} else if csConfig.Cscli.Output == "json" {
 				x, err := json.MarshalIndent(cfg, "", " ")
 				if err != nil {
-					return fmt.Errorf("failed to marshal notification configuration: %w", err)
+					return errors.New("failed to marshal notification configuration")
 				}
 				fmt.Printf("%s", string(x))
 			}
-
 			return nil
 		},
 	}
 
-	return cmd
+	return cmdNotificationsInspect
 }
 
-func (cli *cliNotifications) NewTestCmd() *cobra.Command {
-	var (
-		pluginBroker  csplugin.PluginBroker
-		pluginTomb    tomb.Tomb
-		alertOverride string
-	)
 
-	cmd := &cobra.Command{
-		Use:               "test [plugin name]",
-		Short:             "send a generic test alert to notification plugin",
-		Long:              `send a generic test alert to a notification plugin to test configuration even if is not active`,
-		Example:           `cscli notifications test [plugin_name]`,
-		Args:              cobra.ExactArgs(1),
-		DisableAutoGenTag: true,
-		PreRunE: func(_ *cobra.Command, args []string) error {
-			cfg := cli.cfg()
-			pconfigs, err := cli.getPluginConfigs()
-			if err != nil {
-				return fmt.Errorf("can't build profiles configuration: %w", err)
-			}
-			pcfg, ok := pconfigs[args[0]]
-			if !ok {
-				return fmt.Errorf("plugin name: '%s' does not exist", args[0])
-			}
-			// Create a single profile with plugin name as notification name
-			return pluginBroker.Init(cfg.PluginConfig, []*csconfig.ProfileCfg{
-				{
-					Notifications: []string{
-						pcfg.Name,
-					},
-				},
-			}, cfg.ConfigPaths)
-		},
-		RunE: func(_ *cobra.Command, _ []string) error {
-			pluginTomb.Go(func() error {
-				pluginBroker.Run(&pluginTomb)
-				return nil
-			})
-			alert := &models.Alert{
-				Capacity: ptr.Of(int32(0)),
-				Decisions: []*models.Decision{{
-					Duration: ptr.Of("4h"),
-					Scope:    ptr.Of("Ip"),
-					Value:    ptr.Of("10.10.10.10"),
-					Type:     ptr.Of("ban"),
-					Scenario: ptr.Of("test alert"),
-					Origin:   ptr.Of(types.CscliOrigin),
-				}},
-				Events:          []*models.Event{},
-				EventsCount:     ptr.Of(int32(1)),
-				Leakspeed:       ptr.Of("0"),
-				Message:         ptr.Of("test alert"),
-				ScenarioHash:    ptr.Of(""),
-				Scenario:        ptr.Of("test alert"),
-				ScenarioVersion: ptr.Of(""),
-				Simulated:       ptr.Of(false),
-				Source: &models.Source{
-					AsName:   "",
-					AsNumber: "",
-					Cn:       "",
-					IP:       "10.10.10.10",
-					Range:    "",
-					Scope:    ptr.Of("Ip"),
-					Value:    ptr.Of("10.10.10.10"),
-				},
-				StartAt:   ptr.Of(time.Now().UTC().Format(time.RFC3339)),
-				StopAt:    ptr.Of(time.Now().UTC().Format(time.RFC3339)),
-				CreatedAt: time.Now().UTC().Format(time.RFC3339),
-			}
-			if err := yaml.Unmarshal([]byte(alertOverride), alert); err != nil {
-				return fmt.Errorf("failed to unmarshal alert override: %w", err)
-			}
+func NewNotificationsReinjectCmd() *cobra.Command {
+	var remediation bool
+	var alertOverride string
 
-			pluginBroker.PluginChannel <- csplugin.ProfileAlert{
-				ProfileID: uint(0),
-				Alert:     alert,
-			}
-
-			// time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
-			pluginTomb.Kill(errors.New("terminating"))
-			pluginTomb.Wait()
-
-			return nil
-		},
-	}
-	cmd.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the generic alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
-
-	return cmd
-}
-
-func (cli *cliNotifications) NewReinjectCmd() *cobra.Command {
-	var (
-		alertOverride string
-		alert         *models.Alert
-	)
-
-	cmd := &cobra.Command{
+	var cmdNotificationsReinject = &cobra.Command{
 		Use:   "reinject",
-		Short: "reinject an alert into profiles to trigger notifications",
-		Long:  `reinject an alert into profiles to be evaluated by the filter and sent to matched notifications plugins`,
+		Short: "reinject alert into notifications system",
+		Long:  `Reinject alert into notifications system`,
 		Example: `
 cscli notifications reinject <alert_id>
-cscli notifications reinject <alert_id> -a '{"remediation": false,"scenario":"notification/test"}'
+cscli notifications reinject <alert_id> --remediation
 cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"notification/test"}'
 `,
 		Args:              cobra.ExactArgs(1),
 		DisableAutoGenTag: true,
-		PreRunE: func(_ *cobra.Command, args []string) error {
-			var err error
-			alert, err = cli.fetchAlertFromArgString(args[0])
-			if err != nil {
-				return err
-			}
-
-			return nil
-		},
-		RunE: func(_ *cobra.Command, _ []string) error {
+		RunE: func(cmd *cobra.Command, args []string) error {
 			var (
 				pluginBroker csplugin.PluginBroker
 				pluginTomb   tomb.Tomb
 			)
-
-			cfg := cli.cfg()
-
-			if alertOverride != "" {
-				if err := json.Unmarshal([]byte(alertOverride), alert); err != nil {
-					return fmt.Errorf("can't unmarshal data in the alert flag: %w", err)
-				}
+			if len(args) != 1 {
+				printHelp(cmd)
+				return errors.New("Wrong number of argument: there should be one argument")
 			}
 
-			err := pluginBroker.Init(cfg.PluginConfig, cfg.API.Server.Profiles, cfg.ConfigPaths)
+			//first: get the alert
+			id, err := strconv.Atoi(args[0])
 			if err != nil {
-				return fmt.Errorf("can't initialize plugins: %w", err)
+				return errors.New(fmt.Sprintf("bad alert id %s", args[0]))
+			}
+			if err := csConfig.LoadAPIClient(); err != nil {
+				return errors.Wrapf(err, "loading api client")
+			}
+			if csConfig.API.Client == nil {
+				return errors.New("There is no configuration on 'api_client:'")
+			}
+			if csConfig.API.Client.Credentials == nil {
+				return errors.New(fmt.Sprintf("Please provide credentials for the API in '%s'", csConfig.API.Client.CredentialsFilePath))
+			}
+			apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
+			if err != nil {
+				return errors.Wrapf(err, "error parsing the URL of the API")
+			}
+			client, err := apiclient.NewClient(&apiclient.Config{
+				MachineID:     csConfig.API.Client.Credentials.Login,
+				Password:      strfmt.Password(csConfig.API.Client.Credentials.Password),
+				UserAgent:     fmt.Sprintf("crowdsec/%s", version.String()),
+				URL:           apiURL,
+				VersionPrefix: "v1",
+			})
+			if err != nil {
+				return errors.Wrapf(err, "error creating the client for the API")
+			}
+			alert, _, err := client.Alerts.GetByID(context.Background(), id)
+			if err != nil {
+				return errors.Wrapf(err, fmt.Sprintf("can't find alert with id %s", args[0]))
+			}
+
+			if alertOverride != "" {
+				if err = json.Unmarshal([]byte(alertOverride), alert); err != nil {
+					return errors.Wrapf(err, "Can't unmarshal the data given in the alert flag")
+				}
+			}
+			if !remediation {
+				alert.Remediation = true
+			}
+
+			// second we start plugins
+			err = pluginBroker.Init(csConfig.PluginConfig, csConfig.API.Server.Profiles, csConfig.ConfigPaths)
+			if err != nil {
+				return errors.Wrapf(err, "Can't initialize plugins")
 			}
 
 			pluginTomb.Go(func() error {
@@ -383,15 +306,17 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
 				return nil
 			})
 
-			profiles, err := csprofiles.NewProfile(cfg.API.Server.Profiles)
+			//third: get the profile(s), and process the whole stuff
+
+			profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
 			if err != nil {
-				return fmt.Errorf("cannot extract profiles from configuration: %w", err)
+				return errors.Wrap(err, "Cannot extract profiles from configuration")
 			}
 
 			for id, profile := range profiles {
 				_, matched, err := profile.EvaluateProfile(alert)
 				if err != nil {
-					return fmt.Errorf("can't evaluate profile %s: %w", profile.Cfg.Name, err)
+					return errors.Wrapf(err, "can't evaluate profile %s", profile.Cfg.Name)
 				}
 				if !matched {
 					log.Infof("The profile %s didn't match", profile.Cfg.Name)
@@ -409,54 +334,23 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
 					default:
 						time.Sleep(50 * time.Millisecond)
 						log.Info("sleeping\n")
+
 					}
 				}
-
 				if profile.Cfg.OnSuccess == "break" {
 					log.Infof("The profile %s contains a 'on_success: break' so bailing out", profile.Cfg.Name)
 					break
 				}
 			}
-			// time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
+
+			//			time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
 			pluginTomb.Kill(errors.New("terminating"))
 			pluginTomb.Wait()
-
 			return nil
 		},
 	}
-	cmd.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the reinjected alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
+	cmdNotificationsReinject.Flags().BoolVarP(&remediation, "remediation", "r", false, "Set Alert.Remediation to false in the reinjected alert (see your profile filter configuration)")
+	cmdNotificationsReinject.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the reinjected alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
 
-	return cmd
-}
-
-func (cli *cliNotifications) fetchAlertFromArgString(toParse string) (*models.Alert, error) {
-	cfg := cli.cfg()
-
-	id, err := strconv.Atoi(toParse)
-	if err != nil {
-		return nil, fmt.Errorf("bad alert id %s", toParse)
-	}
-
-	apiURL, err := url.Parse(cfg.API.Client.Credentials.URL)
-	if err != nil {
-		return nil, fmt.Errorf("error parsing the URL of the API: %w", err)
-	}
-
-	client, err := apiclient.NewClient(&apiclient.Config{
-		MachineID:     cfg.API.Client.Credentials.Login,
-		Password:      strfmt.Password(cfg.API.Client.Credentials.Password),
-		UserAgent:     fmt.Sprintf("crowdsec/%s", version.String()),
-		URL:           apiURL,
-		VersionPrefix: "v1",
-	})
-	if err != nil {
-		return nil, fmt.Errorf("error creating the client for the API: %w", err)
-	}
-
-	alert, _, err := client.Alerts.GetByID(context.Background(), id)
-	if err != nil {
-		return nil, fmt.Errorf("can't find alert with id %d: %w", id, err)
-	}
-
-	return alert, nil
+	return cmdNotificationsReinject
 }

cmd/crowdsec-cli/notifications_table.go

@@ -2,43 +2,23 @@ package main
 
 import (
 	"io"
-	"sort"
 	"strings"
 
 	"github.com/aquasecurity/table"
-
-	"github.com/crowdsecurity/crowdsec/pkg/emoji"
 )
 
 func notificationListTable(out io.Writer, ncfgs map[string]NotificationsCfg) {
 	t := newLightTable(out)
-	t.SetHeaders("Active", "Name", "Type", "Profile name")
-	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
-	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
+	t.SetHeaders("Name", "Type", "Profile name")
+	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
+	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
-	keys := make([]string, 0, len(ncfgs))
-	for k := range ncfgs {
-		keys = append(keys, k)
-	}
-
-	sort.Slice(keys, func(i, j int) bool {
-		return len(ncfgs[keys[i]].Profiles) > len(ncfgs[keys[j]].Profiles)
-	})
-
-	for _, k := range keys {
-		b := ncfgs[k]
+	for _, b := range ncfgs {
 		profilesList := []string{}
-
 		for _, p := range b.Profiles {
 			profilesList = append(profilesList, p.Name)
 		}
-
-		active := emoji.CheckMark
-		if len(profilesList) == 0 {
-			active = emoji.Prohibited
-		}
-
-		t.AddRow(active, b.Config.Name, b.Config.Type, strings.Join(profilesList, ", "))
+		t.AddRow(b.Config.Name, b.Config.Type, strings.Join(profilesList, ", "))
 	}
 
 	t.Render()

cmd/crowdsec-cli/papi.go

@@ -8,78 +8,67 @@ import (
 	"github.com/spf13/cobra"
 	"gopkg.in/tomb.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/ptr"
+	"github.com/crowdsecurity/go-cs-lib/pkg/ptr"
 
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/apiserver"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 )
 
-type cliPapi struct {
-	cfg configGetter
-}
-
-func NewCLIPapi(cfg configGetter) *cliPapi {
-	return &cliPapi{
-		cfg: cfg,
-	}
-}
-
-func (cli *cliPapi) NewCommand() *cobra.Command {
-	cmd := &cobra.Command{
+func NewPapiCmd() *cobra.Command {
+	var cmdLapi = &cobra.Command{
 		Use:               "papi [action]",
 		Short:             "Manage interaction with Polling API (PAPI)",
 		Args:              cobra.MinimumNArgs(1),
 		DisableAutoGenTag: true,
-		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
-			cfg := cli.cfg()
-			if err := require.LAPI(cfg); err != nil {
-				return err
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
+				return fmt.Errorf("Local API is disabled, please run this command on the local API machine: %w", err)
 			}
-			if err := require.CAPI(cfg); err != nil {
-				return err
+			if csConfig.API.Server.OnlineClient == nil {
+				log.Fatalf("no configuration for Central API in '%s'", *csConfig.FilePath)
 			}
-			if err := require.PAPI(cfg); err != nil {
-				return err
+			if csConfig.API.Server.OnlineClient.Credentials.PapiURL == "" {
+				log.Fatalf("no PAPI URL in configuration")
 			}
-
 			return nil
 		},
 	}
 
-	cmd.AddCommand(cli.NewStatusCmd())
-	cmd.AddCommand(cli.NewSyncCmd())
+	cmdLapi.AddCommand(NewPapiStatusCmd())
+	cmdLapi.AddCommand(NewPapiSyncCmd())
 
-	return cmd
+	return cmdLapi
 }
 
-func (cli *cliPapi) NewStatusCmd() *cobra.Command {
-	cmd := &cobra.Command{
+func NewPapiStatusCmd() *cobra.Command {
+	cmdCapiStatus := &cobra.Command{
 		Use:               "status",
 		Short:             "Get status of the Polling API",
 		Args:              cobra.MinimumNArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
+		Run: func(cmd *cobra.Command, args []string) {
 			var err error
-			cfg := cli.cfg()
-			dbClient, err = database.NewClient(cfg.DbConfig)
+			dbClient, err = database.NewClient(csConfig.DbConfig)
 			if err != nil {
-				return fmt.Errorf("unable to initialize database client: %w", err)
+				log.Fatalf("unable to initialize database client : %s", err)
 			}
 
-			apic, err := apiserver.NewAPIC(cfg.API.Server.OnlineClient, dbClient, cfg.API.Server.ConsoleConfig, cfg.API.Server.CapiWhitelists)
+			apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig, csConfig.API.Server.CapiWhitelists)
+
 			if err != nil {
-				return fmt.Errorf("unable to initialize API client: %w", err)
+				log.Fatalf("unable to initialize API client : %s", err)
 			}
 
-			papi, err := apiserver.NewPAPI(apic, dbClient, cfg.API.Server.ConsoleConfig, log.GetLevel())
+			papi, err := apiserver.NewPAPI(apic, dbClient, csConfig.API.Server.ConsoleConfig, log.GetLevel())
+
 			if err != nil {
-				return fmt.Errorf("unable to initialize PAPI client: %w", err)
+				log.Fatalf("unable to initialize PAPI client : %s", err)
 			}
 
 			perms, err := papi.GetPermissions()
+
 			if err != nil {
-				return fmt.Errorf("unable to get PAPI permissions: %w", err)
+				log.Fatalf("unable to get PAPI permissions: %s", err)
 			}
 			var lastTimestampStr *string
 			lastTimestampStr, err = dbClient.GetConfigItem(apiserver.PapiPullKey)
@@ -94,47 +83,45 @@ func (cli *cliPapi) NewStatusCmd() *cobra.Command {
 			for _, sub := range perms.Categories {
 				log.Infof(" - %s", sub)
 			}
-
-			return nil
 		},
 	}
 
-	return cmd
+	return cmdCapiStatus
 }
 
-func (cli *cliPapi) NewSyncCmd() *cobra.Command {
-	cmd := &cobra.Command{
+func NewPapiSyncCmd() *cobra.Command {
+	cmdCapiSync := &cobra.Command{
 		Use:               "sync",
 		Short:             "Sync with the Polling API, pulling all non-expired orders for the instance",
 		Args:              cobra.MinimumNArgs(0),
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
+		Run: func(cmd *cobra.Command, args []string) {
 			var err error
-			cfg := cli.cfg()
 			t := tomb.Tomb{}
-
-			dbClient, err = database.NewClient(cfg.DbConfig)
+			dbClient, err = database.NewClient(csConfig.DbConfig)
 			if err != nil {
-				return fmt.Errorf("unable to initialize database client: %w", err)
+				log.Fatalf("unable to initialize database client : %s", err)
 			}
 
-			apic, err := apiserver.NewAPIC(cfg.API.Server.OnlineClient, dbClient, cfg.API.Server.ConsoleConfig, cfg.API.Server.CapiWhitelists)
+			apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig, csConfig.API.Server.CapiWhitelists)
+
 			if err != nil {
-				return fmt.Errorf("unable to initialize API client: %w", err)
+				log.Fatalf("unable to initialize API client : %s", err)
 			}
 
 			t.Go(apic.Push)
 
-			papi, err := apiserver.NewPAPI(apic, dbClient, cfg.API.Server.ConsoleConfig, log.GetLevel())
-			if err != nil {
-				return fmt.Errorf("unable to initialize PAPI client: %w", err)
-			}
+			papi, err := apiserver.NewPAPI(apic, dbClient, csConfig.API.Server.ConsoleConfig, log.GetLevel())
 
+			if err != nil {
+				log.Fatalf("unable to initialize PAPI client : %s", err)
+			}
 			t.Go(papi.SyncDecisions)
 
 			err = papi.PullOnce(time.Time{}, true)
+
 			if err != nil {
-				return fmt.Errorf("unable to sync decisions: %w", err)
+				log.Fatalf("unable to sync decisions: %s", err)
 			}
 
 			log.Infof("Sending acknowledgements to CAPI")
@@ -144,9 +131,8 @@ func (cli *cliPapi) NewSyncCmd() *cobra.Command {
 			t.Wait()
 			time.Sleep(5 * time.Second) //FIXME: the push done by apic.Push is run inside a sub goroutine, sleep to make sure it's done
 
-			return nil
 		},
 	}
 
-	return cmd
+	return cmdCapiSync
 }

cmd/crowdsec-cli/parsers.go

@@ -0,0 +1,202 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/fatih/color"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+
+func NewParsersCmd() *cobra.Command {
+	var cmdParsers = &cobra.Command{
+		Use:   "parsers [action] [config]",
+		Short: "Install/Remove/Upgrade/Inspect parser(s) from hub",
+		Example: `cscli parsers install crowdsecurity/sshd-logs
+cscli parsers inspect crowdsecurity/sshd-logs
+cscli parsers upgrade crowdsecurity/sshd-logs
+cscli parsers list
+cscli parsers remove crowdsecurity/sshd-logs
+`,
+		Args:              cobra.MinimumNArgs(1),
+		Aliases:           []string{"parser"},
+		DisableAutoGenTag: true,
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatal(err)
+			}
+			if csConfig.Hub == nil {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+
+			if err := cwhub.SetHubBranch(); err != nil {
+				return fmt.Errorf("error while setting hub branch: %s", err)
+			}
+
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+				log.Info("Run 'sudo cscli hub update' to get the hub index")
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			return nil
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			if cmd.Name() == "inspect" || cmd.Name() == "list" {
+				return
+			}
+			log.Infof(ReloadMessage())
+		},
+	}
+
+	cmdParsers.AddCommand(NewParsersInstallCmd())
+	cmdParsers.AddCommand(NewParsersRemoveCmd())
+	cmdParsers.AddCommand(NewParsersUpgradeCmd())
+	cmdParsers.AddCommand(NewParsersInspectCmd())
+	cmdParsers.AddCommand(NewParsersListCmd())
+
+	return cmdParsers
+}
+
+
+func NewParsersInstallCmd() *cobra.Command {
+	var ignoreError bool
+
+	var cmdParsersInstall = &cobra.Command{
+		Use:               "install [config]",
+		Short:             "Install given parser(s)",
+		Long:              `Fetch and install given parser(s) from hub`,
+		Example:           `cscli parsers install crowdsec/xxx crowdsec/xyz`,
+		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compAllItems(cwhub.PARSERS, args, toComplete)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			for _, name := range args {
+				t := cwhub.GetItem(cwhub.PARSERS, name)
+				if t == nil {
+					nearestItem, score := GetDistance(cwhub.PARSERS, name)
+					Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError)
+					continue
+				}
+				if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
+					if ignoreError {
+						log.Errorf("Error while installing '%s': %s", name, err)
+					} else {
+						log.Fatalf("Error while installing '%s': %s", name, err)
+					}
+				}
+			}
+		},
+	}
+	cmdParsersInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
+	cmdParsersInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
+	cmdParsersInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple parsers")
+
+	return cmdParsersInstall
+}
+
+
+func NewParsersRemoveCmd() *cobra.Command {
+	var cmdParsersRemove = &cobra.Command{
+		Use:               "remove [config]",
+		Short:             "Remove given parser(s)",
+		Long:              `Remove given parse(s) from hub`,
+		Aliases:           []string{"delete"},
+		Example:           `cscli parsers remove crowdsec/xxx crowdsec/xyz`,
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(cwhub.PARSERS, args, toComplete)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			if all {
+				cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, forceAction)
+				return
+			}
+
+			if len(args) == 0 {
+				log.Fatalf("Specify at least one parser to remove or '--all' flag.")
+			}
+
+			for _, name := range args {
+				cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, forceAction)
+			}
+		},
+	}
+	cmdParsersRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
+	cmdParsersRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
+	cmdParsersRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the parsers")
+
+	return cmdParsersRemove
+}
+
+
+func NewParsersUpgradeCmd() *cobra.Command {
+	var cmdParsersUpgrade = &cobra.Command{
+		Use:               "upgrade [config]",
+		Short:             "Upgrade given parser(s)",
+		Long:              `Fetch and upgrade given parser(s) from hub`,
+		Example:           `cscli parsers upgrade crowdsec/xxx crowdsec/xyz`,
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(cwhub.PARSERS, args, toComplete)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			if all {
+				cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
+			} else {
+				if len(args) == 0 {
+					log.Fatalf("no target parser to upgrade")
+				}
+				for _, name := range args {
+					cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, forceAction)
+				}
+			}
+		},
+	}
+	cmdParsersUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the parsers")
+	cmdParsersUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
+
+	return cmdParsersUpgrade
+}
+
+
+func NewParsersInspectCmd() *cobra.Command {
+	var cmdParsersInspect = &cobra.Command{
+		Use:               "inspect [name]",
+		Short:             "Inspect given parser",
+		Long:              `Inspect given parser`,
+		Example:           `cscli parsers inspect crowdsec/xxx`,
+		DisableAutoGenTag: true,
+		Args:              cobra.MinimumNArgs(1),
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(cwhub.PARSERS, args, toComplete)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			InspectItem(args[0], cwhub.PARSERS)
+		},
+	}
+	cmdParsersInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
+
+	return cmdParsersInspect
+}
+
+
+func NewParsersListCmd() *cobra.Command {
+	var cmdParsersList = &cobra.Command{
+		Use:   "list [name]",
+		Short: "List all parsers or given one",
+		Long:  `List all parsers or given one`,
+		Example: `cscli parsers list
+cscli parser list crowdsecurity/xxx`,
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all)
+		},
+	}
+	cmdParsersList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
+
+	return cmdParsersList
+}

cmd/crowdsec-cli/postoverflows.go

@@ -0,0 +1,200 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/fatih/color"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func NewPostOverflowsInstallCmd() *cobra.Command {
+	var ignoreError bool
+
+	cmdPostOverflowsInstall := &cobra.Command{
+		Use:               "install [config]",
+		Short:             "Install given postoverflow(s)",
+		Long:              `Fetch and install given postoverflow(s) from hub`,
+		Example:           `cscli postoverflows install crowdsec/xxx crowdsec/xyz`,
+		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compAllItems(cwhub.PARSERS_OVFLW, args, toComplete)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			for _, name := range args {
+				t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name)
+				if t == nil {
+					nearestItem, score := GetDistance(cwhub.PARSERS_OVFLW, name)
+					Suggest(cwhub.PARSERS_OVFLW, name, nearestItem.Name, score, ignoreError)
+					continue
+				}
+				if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil {
+					if ignoreError {
+						log.Errorf("Error while installing '%s': %s", name, err)
+					} else {
+						log.Fatalf("Error while installing '%s': %s", name, err)
+					}
+				}
+			}
+		},
+	}
+
+	cmdPostOverflowsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
+	cmdPostOverflowsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
+	cmdPostOverflowsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple postoverflows")
+
+	return cmdPostOverflowsInstall
+}
+
+func NewPostOverflowsRemoveCmd() *cobra.Command {
+	cmdPostOverflowsRemove := &cobra.Command{
+		Use:               "remove [config]",
+		Short:             "Remove given postoverflow(s)",
+		Long:              `remove given postoverflow(s)`,
+		Example:           `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`,
+		DisableAutoGenTag: true,
+		Aliases:           []string{"delete"},
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			if all {
+				cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, forceAction)
+				return
+			}
+
+			if len(args) == 0 {
+				log.Fatalf("Specify at least one postoverflow to remove or '--all' flag.")
+			}
+
+			for _, name := range args {
+				cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, forceAction)
+			}
+		},
+	}
+
+	cmdPostOverflowsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
+	cmdPostOverflowsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
+	cmdPostOverflowsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the postoverflows")
+
+	return cmdPostOverflowsRemove
+}
+
+func NewPostOverflowsUpgradeCmd() *cobra.Command {
+	cmdPostOverflowsUpgrade := &cobra.Command{
+		Use:               "upgrade [config]",
+		Short:             "Upgrade given postoverflow(s)",
+		Long:              `Fetch and Upgrade given postoverflow(s) from hub`,
+		Example:           `cscli postoverflows upgrade crowdsec/xxx crowdsec/xyz`,
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			if all {
+				cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
+			} else {
+				if len(args) == 0 {
+					log.Fatalf("no target postoverflow to upgrade")
+				}
+				for _, name := range args {
+					cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, forceAction)
+				}
+			}
+		},
+	}
+
+	cmdPostOverflowsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the postoverflows")
+	cmdPostOverflowsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
+
+	return cmdPostOverflowsUpgrade
+}
+
+func NewPostOverflowsInspectCmd() *cobra.Command {
+	cmdPostOverflowsInspect := &cobra.Command{
+		Use:               "inspect [config]",
+		Short:             "Inspect given postoverflow",
+		Long:              `Inspect given postoverflow`,
+		Example:           `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`,
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
+		},
+		Args: cobra.MinimumNArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			InspectItem(args[0], cwhub.PARSERS_OVFLW)
+		},
+	}
+
+	return cmdPostOverflowsInspect
+}
+
+func NewPostOverflowsListCmd() *cobra.Command {
+	cmdPostOverflowsList := &cobra.Command{
+		Use:   "list [config]",
+		Short: "List all postoverflows or given one",
+		Long:  `List all postoverflows or given one`,
+		Example: `cscli postoverflows list
+cscli postoverflows list crowdsecurity/xxx`,
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all)
+		},
+	}
+
+	cmdPostOverflowsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
+
+	return cmdPostOverflowsList
+}
+
+
+
+func NewPostOverflowsCmd() *cobra.Command {
+	cmdPostOverflows := &cobra.Command{
+		Use:   "postoverflows [action] [config]",
+		Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub",
+		Example: `cscli postoverflows install crowdsecurity/cdn-whitelist
+		cscli postoverflows inspect crowdsecurity/cdn-whitelist
+		cscli postoverflows upgrade crowdsecurity/cdn-whitelist
+		cscli postoverflows list
+		cscli postoverflows remove crowdsecurity/cdn-whitelist`,
+		Args:              cobra.MinimumNArgs(1),
+		Aliases:           []string{"postoverflow"},
+		DisableAutoGenTag: true,
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatal(err)
+			}
+			if csConfig.Hub == nil {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+
+			if err := cwhub.SetHubBranch(); err != nil {
+				return fmt.Errorf("error while setting hub branch: %s", err)
+			}
+
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+				log.Info("Run 'sudo cscli hub update' to get the hub index")
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			return nil
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			if cmd.Name() == "inspect" || cmd.Name() == "list" {
+				return
+			}
+			log.Infof(ReloadMessage())
+		},
+	}
+
+	cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd())
+	cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd())
+	cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd())
+	cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd())
+	cmdPostOverflows.AddCommand(NewPostOverflowsListCmd())
+
+	return cmdPostOverflows
+}

cmd/crowdsec-cli/require/branch.go

@@ -1,62 +0,0 @@
-package require
-
-// Set the appropriate hub branch according to config settings and crowdsec version
-
-import (
-	log "github.com/sirupsen/logrus"
-	"golang.org/x/mod/semver"
-
-	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
-	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
-)
-
-func chooseBranch(cfg *csconfig.Config) string {
-	// this was set from config.yaml or flag
-	if cfg.Cscli.HubBranch != "" {
-		log.Debugf("Hub override from config: branch '%s'", cfg.Cscli.HubBranch)
-		return cfg.Cscli.HubBranch
-	}
-
-	latest, err := cwversion.Latest()
-	if err != nil {
-		log.Warningf("Unable to retrieve latest crowdsec version: %s, using hub branch 'master'", err)
-		return "master"
-	}
-
-	csVersion := cwversion.VersionStrip()
-	if csVersion == latest {
-		log.Debugf("Latest crowdsec version (%s), using hub branch 'master'", csVersion)
-		return "master"
-	}
-
-	// if current version is greater than the latest we are in pre-release
-	if semver.Compare(csVersion, latest) == 1 {
-		log.Debugf("Your current crowdsec version seems to be a pre-release (%s), using hub branch 'master'", csVersion)
-		return "master"
-	}
-
-	if csVersion == "" {
-		log.Warning("Crowdsec version is not set, using hub branch 'master'")
-		return "master"
-	}
-
-	log.Warnf("A new CrowdSec release is available (%s). "+
-		"Your version is '%s'. Please update it to use new parsers/scenarios/collections.",
-		latest, csVersion)
-	return csVersion
-}
-
-
-// HubBranch sets the branch (in cscli config) and returns its value
-// It can be "master", or the branch corresponding to the current crowdsec version, or the value overridden in config/flag
-func HubBranch(cfg *csconfig.Config) string {
-	branch := chooseBranch(cfg)
-
-	cfg.Cscli.HubBranch = branch
-
-	return branch
-}
-
-func HubURLTemplate(cfg *csconfig.Config) string {
-	return cfg.Cscli.HubURLTemplate
-}

cmd/crowdsec-cli/require/require.go

@@ -1,100 +0,0 @@
-package require
-
-import (
-	"errors"
-	"fmt"
-	"io"
-
-	"github.com/sirupsen/logrus"
-
-	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-)
-
-func LAPI(c *csconfig.Config) error {
-	if err := c.LoadAPIServer(true); err != nil {
-		return fmt.Errorf("failed to load Local API: %w", err)
-	}
-
-	if c.DisableAPI {
-		return errors.New("local API is disabled -- this command must be run on the local API machine")
-	}
-
-	return nil
-}
-
-func CAPI(c *csconfig.Config) error {
-	if c.API.Server.OnlineClient == nil {
-		return fmt.Errorf("no configuration for Central API (CAPI) in '%s'", *c.FilePath)
-	}
-
-	return nil
-}
-
-func PAPI(c *csconfig.Config) error {
-	if c.API.Server.OnlineClient.Credentials.PapiURL == "" {
-		return errors.New("no PAPI URL in configuration")
-	}
-
-	return nil
-}
-
-func CAPIRegistered(c *csconfig.Config) error {
-	if c.API.Server.OnlineClient.Credentials == nil {
-		return errors.New("the Central API (CAPI) must be configured with 'cscli capi register'")
-	}
-
-	return nil
-}
-
-func DB(c *csconfig.Config) error {
-	if err := c.LoadDBConfig(true); err != nil {
-		return fmt.Errorf("this command requires direct database access (must be run on the local API machine): %w", err)
-	}
-
-	return nil
-}
-
-func Notifications(c *csconfig.Config) error {
-	if c.ConfigPaths.NotificationDir == "" {
-		return errors.New("config_paths.notification_dir is not set in crowdsec config")
-	}
-
-	return nil
-}
-
-// RemoteHub returns the configuration required to download hub index and items: url, branch, etc.
-func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg {
-	// set branch in config, and log if necessary
-	branch := HubBranch(c)
-	urlTemplate := HubURLTemplate(c)
-	remote := &cwhub.RemoteHubCfg{
-		Branch:      branch,
-		URLTemplate: urlTemplate,
-		IndexPath: ".index.json",
-	}
-
-	return remote
-}
-
-// Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items.
-// If no remote parameter is provided, the hub can only be used for local operations.
-func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg, logger *logrus.Logger) (*cwhub.Hub, error) {
-	local := c.Hub
-
-	if local == nil {
-		return nil, errors.New("you must configure cli before interacting with hub")
-	}
-
-	if logger == nil {
-		logger = logrus.New()
-		logger.SetOutput(io.Discard)
-	}
-
-	hub, err := cwhub.NewHub(local, remote, false, logger)
-	if err != nil {
-		return nil, fmt.Errorf("failed to read Hub index: %w. Run 'sudo cscli hub update' to download the index again", err)
-	}
-
-	return hub, nil
-}

cmd/crowdsec-cli/scenarios.go

@@ -0,0 +1,198 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/fatih/color"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func NewScenariosCmd() *cobra.Command {
+	var cmdScenarios = &cobra.Command{
+		Use:   "scenarios [action] [config]",
+		Short: "Install/Remove/Upgrade/Inspect scenario(s) from hub",
+		Example: `cscli scenarios list [-a]
+cscli scenarios install crowdsecurity/ssh-bf
+cscli scenarios inspect crowdsecurity/ssh-bf
+cscli scenarios upgrade crowdsecurity/ssh-bf
+cscli scenarios remove crowdsecurity/ssh-bf
+`,
+		Args:              cobra.MinimumNArgs(1),
+		Aliases:           []string{"scenario"},
+		DisableAutoGenTag: true,
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatal(err)
+			}
+			if csConfig.Hub == nil {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+
+			if err := cwhub.SetHubBranch(); err != nil {
+				return errors.Wrap(err, "while setting hub branch")
+			}
+
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+				log.Info("Run 'sudo cscli hub update' to get the hub index")
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+
+			return nil
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			if cmd.Name() == "inspect" || cmd.Name() == "list" {
+				return
+			}
+			log.Infof(ReloadMessage())
+		},
+	}
+
+	cmdScenarios.AddCommand(NewCmdScenariosInstall())
+	cmdScenarios.AddCommand(NewCmdScenariosRemove())
+	cmdScenarios.AddCommand(NewCmdScenariosUpgrade())
+	cmdScenarios.AddCommand(NewCmdScenariosInspect())
+	cmdScenarios.AddCommand(NewCmdScenariosList())
+
+	return cmdScenarios
+}
+
+func NewCmdScenariosInstall() *cobra.Command {
+	var ignoreError bool
+
+	var cmdScenariosInstall = &cobra.Command{
+		Use:     "install [config]",
+		Short:   "Install given scenario(s)",
+		Long:    `Fetch and install given scenario(s) from hub`,
+		Example: `cscli scenarios install crowdsec/xxx crowdsec/xyz`,
+		Args:    cobra.MinimumNArgs(1),
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compAllItems(cwhub.SCENARIOS, args, toComplete)
+		},
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			for _, name := range args {
+				t := cwhub.GetItem(cwhub.SCENARIOS, name)
+				if t == nil {
+					nearestItem, score := GetDistance(cwhub.SCENARIOS, name)
+					Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError)
+					continue
+				}
+				if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
+					if ignoreError {
+						log.Errorf("Error while installing '%s': %s", name, err)
+					} else {
+						log.Fatalf("Error while installing '%s': %s", name, err)
+					}
+				}
+			}
+		},
+	}
+	cmdScenariosInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
+	cmdScenariosInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
+	cmdScenariosInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple scenarios")
+
+	return cmdScenariosInstall
+}
+
+func NewCmdScenariosRemove() *cobra.Command {
+	var cmdScenariosRemove = &cobra.Command{
+		Use:     "remove [config]",
+		Short:   "Remove given scenario(s)",
+		Long:    `remove given scenario(s)`,
+		Example: `cscli scenarios remove crowdsec/xxx crowdsec/xyz`,
+		Aliases: []string{"delete"},
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
+		},
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			if all {
+				cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, forceAction)
+				return
+			}
+
+			if len(args) == 0 {
+				log.Fatalf("Specify at least one scenario to remove or '--all' flag.")
+			}
+
+			for _, name := range args {
+				cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, forceAction)
+			}
+		},
+	}
+	cmdScenariosRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
+	cmdScenariosRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
+	cmdScenariosRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the scenarios")
+
+	return cmdScenariosRemove
+}
+
+func NewCmdScenariosUpgrade() *cobra.Command {
+	var cmdScenariosUpgrade = &cobra.Command{
+		Use:     "upgrade [config]",
+		Short:   "Upgrade given scenario(s)",
+		Long:    `Fetch and Upgrade given scenario(s) from hub`,
+		Example: `cscli scenarios upgrade crowdsec/xxx crowdsec/xyz`,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
+		},
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			if all {
+				cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
+			} else {
+				if len(args) == 0 {
+					log.Fatalf("no target scenario to upgrade")
+				}
+				for _, name := range args {
+					cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, forceAction)
+				}
+			}
+		},
+	}
+	cmdScenariosUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the scenarios")
+	cmdScenariosUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
+
+	return cmdScenariosUpgrade
+}
+
+func NewCmdScenariosInspect() *cobra.Command {
+	var cmdScenariosInspect = &cobra.Command{
+		Use:     "inspect [config]",
+		Short:   "Inspect given scenario",
+		Long:    `Inspect given scenario`,
+		Example: `cscli scenarios inspect crowdsec/xxx`,
+		Args:    cobra.MinimumNArgs(1),
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
+		},
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			InspectItem(args[0], cwhub.SCENARIOS)
+		},
+	}
+	cmdScenariosInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
+
+	return cmdScenariosInspect
+}
+
+func NewCmdScenariosList() *cobra.Command {
+	var cmdScenariosList = &cobra.Command{
+		Use:   "list [config]",
+		Short: "List all scenario(s) or given one",
+		Long:  `List all scenario(s) or given one`,
+		Example: `cscli scenarios list
+cscli scenarios list crowdsecurity/xxx`,
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all)
+		},
+	}
+	cmdScenariosList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
+
+	return cmdScenariosList
+}

cmd/crowdsec-cli/setup.go

@@ -2,17 +2,15 @@ package main
 
 import (
 	"bytes"
-	"errors"
 	"fmt"
 	"os"
 	"os/exec"
 
-	goccyyaml "github.com/goccy/go-yaml"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"gopkg.in/yaml.v3"
+	goccyyaml "github.com/goccy/go-yaml"
 
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/setup"
 )
@@ -114,22 +112,6 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
 		return err
 	}
 
-	var detectReader *os.File
-
-	switch detectConfigFile {
-	case "-":
-		log.Tracef("Reading detection rules from stdin")
-
-		detectReader = os.Stdin
-	default:
-		log.Tracef("Reading detection rules: %s", detectConfigFile)
-
-		detectReader, err = os.Open(detectConfigFile)
-		if err != nil {
-			return err
-		}
-	}
-
 	listSupportedServices, err := flags.GetBool("list-supported-services")
 	if err != nil {
 		return err
@@ -174,7 +156,6 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
 		_, err := exec.LookPath("systemctl")
 		if err != nil {
 			log.Debug("systemctl not available: snubbing systemd")
-
 			snubSystemd = true
 		}
 	}
@@ -186,12 +167,11 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
 
 	if forcedOSFamily == "" && forcedOSID != "" {
 		log.Debug("force-os-id is set: force-os-family defaults to 'linux'")
-
 		forcedOSFamily = "linux"
 	}
 
 	if listSupportedServices {
-		supported, err := setup.ListSupported(detectReader)
+		supported, err := setup.ListSupported(detectConfigFile)
 		if err != nil {
 			return err
 		}
@@ -215,7 +195,7 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
 		SnubSystemd:  snubSystemd,
 	}
 
-	hubSetup, err := setup.Detect(detectReader, opts)
+	hubSetup, err := setup.Detect(detectConfigFile, opts)
 	if err != nil {
 		return fmt.Errorf("detecting services: %w", err)
 	}
@@ -224,7 +204,6 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
-
 	fmt.Println(setup)
 
 	return nil
@@ -310,12 +289,7 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("while reading file %s: %w", fromFile, err)
 	}
 
-	hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), log.StandardLogger())
-	if err != nil {
-		return err
-	}
-
-	if err = setup.InstallHubItems(hub, input, dryRun); err != nil {
+	if err = setup.InstallHubItems(csConfig, input, dryRun); err != nil {
 		return err
 	}
 
@@ -324,7 +298,6 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error {
 
 func runSetupValidate(cmd *cobra.Command, args []string) error {
 	fromFile := args[0]
-
 	input, err := os.ReadFile(fromFile)
 	if err != nil {
 		return fmt.Errorf("while reading stdin: %w", err)
@@ -332,7 +305,7 @@ func runSetupValidate(cmd *cobra.Command, args []string) error {
 
 	if err = setup.Validate(input); err != nil {
 		fmt.Printf("%v\n", err)
-		return errors.New("invalid setup file")
+		return fmt.Errorf("invalid setup file")
 	}
 
 	return nil

cmd/crowdsec-cli/simulation.go

@@ -1,209 +1,40 @@
 package main
 
 import (
-	"errors"
 	"fmt"
 	"os"
-	"slices"
 
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
-	"gopkg.in/yaml.v3"
+	"golang.org/x/exp/slices"
+	"gopkg.in/yaml.v2"
 
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 )
 
-type cliSimulation struct {
-	cfg configGetter
+func addToExclusion(name string) error {
+	csConfig.Cscli.SimulationConfig.Exclusions = append(csConfig.Cscli.SimulationConfig.Exclusions, name)
+	return nil
 }
 
-func NewCLISimulation(cfg configGetter) *cliSimulation {
-	return &cliSimulation{
-		cfg: cfg,
-	}
-}
-
-func (cli *cliSimulation) NewCommand() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:   "simulation [command]",
-		Short: "Manage simulation status of scenarios",
-		Example: `cscli simulation status
-cscli simulation enable crowdsecurity/ssh-bf
-cscli simulation disable crowdsecurity/ssh-bf`,
-		DisableAutoGenTag: true,
-		PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
-			if err := cli.cfg().LoadSimulation(); err != nil {
-				return err
-			}
-			if cli.cfg().Cscli.SimulationConfig == nil {
-				return errors.New("no simulation configured")
-			}
-
-			return nil
-		},
-		PersistentPostRun: func(cmd *cobra.Command, _ []string) {
-			if cmd.Name() != "status" {
-				log.Infof(ReloadMessage())
-			}
-		},
-	}
-	cmd.Flags().SortFlags = false
-	cmd.PersistentFlags().SortFlags = false
-
-	cmd.AddCommand(cli.NewEnableCmd())
-	cmd.AddCommand(cli.NewDisableCmd())
-	cmd.AddCommand(cli.NewStatusCmd())
-
-	return cmd
-}
-
-func (cli *cliSimulation) NewEnableCmd() *cobra.Command {
-	var forceGlobalSimulation bool
-
-	cmd := &cobra.Command{
-		Use:               "enable [scenario] [-global]",
-		Short:             "Enable the simulation, globally or on specified scenarios",
-		Example:           `cscli simulation enable`,
-		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			hub, err := require.Hub(cli.cfg(), nil, nil)
-			if err != nil {
-				return err
-			}
-
-			if len(args) > 0 {
-				for _, scenario := range args {
-					item := hub.GetItem(cwhub.SCENARIOS, scenario)
-					if item == nil {
-						log.Errorf("'%s' doesn't exist or is not a scenario", scenario)
-						continue
-					}
-					if !item.State.Installed {
-						log.Warningf("'%s' isn't enabled", scenario)
-					}
-					isExcluded := slices.Contains(cli.cfg().Cscli.SimulationConfig.Exclusions, scenario)
-					if *cli.cfg().Cscli.SimulationConfig.Simulation && !isExcluded {
-						log.Warning("global simulation is already enabled")
-						continue
-					}
-					if !*cli.cfg().Cscli.SimulationConfig.Simulation && isExcluded {
-						log.Warningf("simulation for '%s' already enabled", scenario)
-						continue
-					}
-					if *cli.cfg().Cscli.SimulationConfig.Simulation && isExcluded {
-						cli.removeFromExclusion(scenario)
-						log.Printf("simulation enabled for '%s'", scenario)
-						continue
-					}
-					cli.addToExclusion(scenario)
-					log.Printf("simulation mode for '%s' enabled", scenario)
-				}
-				if err := cli.dumpSimulationFile(); err != nil {
-					return fmt.Errorf("simulation enable: %w", err)
-				}
-			} else if forceGlobalSimulation {
-				if err := cli.enableGlobalSimulation(); err != nil {
-					return fmt.Errorf("unable to enable global simulation mode: %w", err)
-				}
-			} else {
-				printHelp(cmd)
-			}
-
-			return nil
-		},
-	}
-	cmd.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Enable global simulation (reverse mode)")
-
-	return cmd
-}
-
-func (cli *cliSimulation) NewDisableCmd() *cobra.Command {
-	var forceGlobalSimulation bool
-
-	cmd := &cobra.Command{
-		Use:               "disable [scenario]",
-		Short:             "Disable the simulation mode. Disable only specified scenarios",
-		Example:           `cscli simulation disable`,
-		DisableAutoGenTag: true,
-		RunE: func(cmd *cobra.Command, args []string) error {
-			if len(args) > 0 {
-				for _, scenario := range args {
-					isExcluded := slices.Contains(cli.cfg().Cscli.SimulationConfig.Exclusions, scenario)
-					if !*cli.cfg().Cscli.SimulationConfig.Simulation && !isExcluded {
-						log.Warningf("%s isn't in simulation mode", scenario)
-						continue
-					}
-					if !*cli.cfg().Cscli.SimulationConfig.Simulation && isExcluded {
-						cli.removeFromExclusion(scenario)
-						log.Printf("simulation mode for '%s' disabled", scenario)
-						continue
-					}
-					if isExcluded {
-						log.Warningf("simulation mode is enabled but is already disable for '%s'", scenario)
-						continue
-					}
-					cli.addToExclusion(scenario)
-					log.Printf("simulation mode for '%s' disabled", scenario)
-				}
-				if err := cli.dumpSimulationFile(); err != nil {
-					return fmt.Errorf("simulation disable: %w", err)
-				}
-			} else if forceGlobalSimulation {
-				if err := cli.disableGlobalSimulation(); err != nil {
-					return fmt.Errorf("unable to disable global simulation mode: %w", err)
-				}
-			} else {
-				printHelp(cmd)
-			}
-
-			return nil
-		},
-	}
-	cmd.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Disable global simulation (reverse mode)")
-
-	return cmd
-}
-
-func (cli *cliSimulation) NewStatusCmd() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:               "status",
-		Short:             "Show simulation mode status",
-		Example:           `cscli simulation status`,
-		DisableAutoGenTag: true,
-		Run: func(_ *cobra.Command, _ []string) {
-			cli.status()
-		},
-		PersistentPostRun: func(cmd *cobra.Command, args []string) {
-		},
-	}
-
-	return cmd
-}
-
-func (cli *cliSimulation) addToExclusion(name string) {
-	cfg := cli.cfg()
-	cfg.Cscli.SimulationConfig.Exclusions = append(cfg.Cscli.SimulationConfig.Exclusions, name)
-}
-
-func (cli *cliSimulation) removeFromExclusion(name string) {
-	cfg := cli.cfg()
-	index := slices.Index(cfg.Cscli.SimulationConfig.Exclusions, name)
+func removeFromExclusion(name string) error {
+	index := indexOf(name, csConfig.Cscli.SimulationConfig.Exclusions)
 
 	// Remove element from the slice
-	cfg.Cscli.SimulationConfig.Exclusions[index] = cfg.Cscli.SimulationConfig.Exclusions[len(cfg.Cscli.SimulationConfig.Exclusions)-1]
-	cfg.Cscli.SimulationConfig.Exclusions[len(cfg.Cscli.SimulationConfig.Exclusions)-1] = ""
-	cfg.Cscli.SimulationConfig.Exclusions = cfg.Cscli.SimulationConfig.Exclusions[:len(cfg.Cscli.SimulationConfig.Exclusions)-1]
+	csConfig.Cscli.SimulationConfig.Exclusions[index] = csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
+	csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1] = ""
+	csConfig.Cscli.SimulationConfig.Exclusions = csConfig.Cscli.SimulationConfig.Exclusions[:len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
+
+	return nil
 }
 
-func (cli *cliSimulation) enableGlobalSimulation() error {
-	cfg := cli.cfg()
-	cfg.Cscli.SimulationConfig.Simulation = new(bool)
-	*cfg.Cscli.SimulationConfig.Simulation = true
-	cfg.Cscli.SimulationConfig.Exclusions = []string{}
+func enableGlobalSimulation() error {
+	csConfig.Cscli.SimulationConfig.Simulation = new(bool)
+	*csConfig.Cscli.SimulationConfig.Simulation = true
+	csConfig.Cscli.SimulationConfig.Exclusions = []string{}
 
-	if err := cli.dumpSimulationFile(); err != nil {
-		return fmt.Errorf("unable to dump simulation file: %w", err)
+	if err := dumpSimulationFile(); err != nil {
+		log.Fatalf("unable to dump simulation file: %s", err)
 	}
 
 	log.Printf("global simulation: enabled")
@@ -211,72 +42,227 @@ func (cli *cliSimulation) enableGlobalSimulation() error {
 	return nil
 }
 
-func (cli *cliSimulation) dumpSimulationFile() error {
-	cfg := cli.cfg()
-
-	newConfigSim, err := yaml.Marshal(cfg.Cscli.SimulationConfig)
+func dumpSimulationFile() error {
+	newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
 	if err != nil {
-		return fmt.Errorf("unable to marshal simulation configuration: %w", err)
+		return fmt.Errorf("unable to marshal simulation configuration: %s", err)
 	}
-
-	err = os.WriteFile(cfg.ConfigPaths.SimulationFilePath, newConfigSim, 0o644)
+	err = os.WriteFile(csConfig.ConfigPaths.SimulationFilePath, newConfigSim, 0644)
 	if err != nil {
-		return fmt.Errorf("write simulation config in '%s' failed: %w", cfg.ConfigPaths.SimulationFilePath, err)
+		return fmt.Errorf("write simulation config in '%s' failed: %s", csConfig.ConfigPaths.SimulationFilePath, err)
 	}
-
-	log.Debugf("updated simulation file %s", cfg.ConfigPaths.SimulationFilePath)
+	log.Debugf("updated simulation file %s", csConfig.ConfigPaths.SimulationFilePath)
 
 	return nil
 }
 
-func (cli *cliSimulation) disableGlobalSimulation() error {
-	cfg := cli.cfg()
-	cfg.Cscli.SimulationConfig.Simulation = new(bool)
-	*cfg.Cscli.SimulationConfig.Simulation = false
+func disableGlobalSimulation() error {
+	csConfig.Cscli.SimulationConfig.Simulation = new(bool)
+	*csConfig.Cscli.SimulationConfig.Simulation = false
 
-	cfg.Cscli.SimulationConfig.Exclusions = []string{}
-
-	newConfigSim, err := yaml.Marshal(cfg.Cscli.SimulationConfig)
+	csConfig.Cscli.SimulationConfig.Exclusions = []string{}
+	newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
 	if err != nil {
-		return fmt.Errorf("unable to marshal new simulation configuration: %w", err)
+		return fmt.Errorf("unable to marshal new simulation configuration: %s", err)
 	}
-
-	err = os.WriteFile(cfg.ConfigPaths.SimulationFilePath, newConfigSim, 0o644)
+	err = os.WriteFile(csConfig.ConfigPaths.SimulationFilePath, newConfigSim, 0644)
 	if err != nil {
-		return fmt.Errorf("unable to write new simulation config in '%s': %w", cfg.ConfigPaths.SimulationFilePath, err)
+		return fmt.Errorf("unable to write new simulation config in '%s' : %s", csConfig.ConfigPaths.SimulationFilePath, err)
 	}
 
 	log.Printf("global simulation: disabled")
-
 	return nil
 }
 
-func (cli *cliSimulation) status() {
-	cfg := cli.cfg()
-	if cfg.Cscli.SimulationConfig == nil {
+func simulationStatus() error {
+	if csConfig.Cscli.SimulationConfig == nil {
 		log.Printf("global simulation: disabled (configuration file is missing)")
-		return
+		return nil
 	}
-
-	if *cfg.Cscli.SimulationConfig.Simulation {
+	if *csConfig.Cscli.SimulationConfig.Simulation {
 		log.Println("global simulation: enabled")
-
-		if len(cfg.Cscli.SimulationConfig.Exclusions) > 0 {
+		if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
 			log.Println("Scenarios not in simulation mode :")
-
-			for _, scenario := range cfg.Cscli.SimulationConfig.Exclusions {
+			for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
 				log.Printf("  - %s", scenario)
 			}
 		}
 	} else {
 		log.Println("global simulation: disabled")
-
-		if len(cfg.Cscli.SimulationConfig.Exclusions) > 0 {
+		if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
 			log.Println("Scenarios in simulation mode :")
-
-			for _, scenario := range cfg.Cscli.SimulationConfig.Exclusions {
+			for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
 				log.Printf("  - %s", scenario)
 			}
 		}
 	}
+	return nil
+}
+
+func NewSimulationCmds() *cobra.Command {
+	var cmdSimulation = &cobra.Command{
+		Use:   "simulation [command]",
+		Short: "Manage simulation status of scenarios",
+		Example: `cscli simulation status
+cscli simulation enable crowdsecurity/ssh-bf
+cscli simulation disable crowdsecurity/ssh-bf`,
+		DisableAutoGenTag: true,
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := csConfig.LoadSimulation(); err != nil {
+				log.Fatal(err)
+			}
+			if csConfig.Cscli == nil {
+				return fmt.Errorf("you must configure cli before using simulation")
+			}
+			if csConfig.Cscli.SimulationConfig == nil {
+				return fmt.Errorf("no simulation configured")
+			}
+			return nil
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			if cmd.Name() != "status" {
+				log.Infof(ReloadMessage())
+			}
+		},
+	}
+	cmdSimulation.Flags().SortFlags = false
+	cmdSimulation.PersistentFlags().SortFlags = false
+
+	cmdSimulation.AddCommand(NewSimulationEnableCmd())
+	cmdSimulation.AddCommand(NewSimulationDisableCmd())
+	cmdSimulation.AddCommand(NewSimulationStatusCmd())
+
+	return cmdSimulation
+}
+
+func NewSimulationEnableCmd() *cobra.Command {
+	var forceGlobalSimulation bool
+
+	var cmdSimulationEnable = &cobra.Command{
+		Use:               "enable [scenario] [-global]",
+		Short:             "Enable the simulation, globally or on specified scenarios",
+		Example:           `cscli simulation enable`,
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatal(err)
+			}
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+				log.Info("Run 'sudo cscli hub update' to get the hub index")
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+
+			if len(args) > 0 {
+				for _, scenario := range args {
+					var item = cwhub.GetItem(cwhub.SCENARIOS, scenario)
+					if item == nil {
+						log.Errorf("'%s' doesn't exist or is not a scenario", scenario)
+						continue
+					}
+					if !item.Installed {
+						log.Warningf("'%s' isn't enabled", scenario)
+					}
+					isExcluded := slices.Contains(csConfig.Cscli.SimulationConfig.Exclusions, scenario)
+					if *csConfig.Cscli.SimulationConfig.Simulation && !isExcluded {
+						log.Warning("global simulation is already enabled")
+						continue
+					}
+					if !*csConfig.Cscli.SimulationConfig.Simulation && isExcluded {
+						log.Warningf("simulation for '%s' already enabled", scenario)
+						continue
+					}
+					if *csConfig.Cscli.SimulationConfig.Simulation && isExcluded {
+						if err := removeFromExclusion(scenario); err != nil {
+							log.Fatal(err)
+						}
+						log.Printf("simulation enabled for '%s'", scenario)
+						continue
+					}
+					if err := addToExclusion(scenario); err != nil {
+						log.Fatal(err)
+					}
+					log.Printf("simulation mode for '%s' enabled", scenario)
+				}
+				if err := dumpSimulationFile(); err != nil {
+					log.Fatalf("simulation enable: %s", err)
+				}
+			} else if forceGlobalSimulation {
+				if err := enableGlobalSimulation(); err != nil {
+					log.Fatalf("unable to enable global simulation mode : %s", err)
+				}
+			} else {
+				printHelp(cmd)
+			}
+		},
+	}
+	cmdSimulationEnable.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Enable global simulation (reverse mode)")
+
+	return cmdSimulationEnable
+}
+
+func NewSimulationDisableCmd() *cobra.Command {
+	var forceGlobalSimulation bool
+
+	var cmdSimulationDisable = &cobra.Command{
+		Use:               "disable [scenario]",
+		Short:             "Disable the simulation mode. Disable only specified scenarios",
+		Example:           `cscli simulation disable`,
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			if len(args) > 0 {
+				for _, scenario := range args {
+					isExcluded := slices.Contains(csConfig.Cscli.SimulationConfig.Exclusions, scenario)
+					if !*csConfig.Cscli.SimulationConfig.Simulation && !isExcluded {
+						log.Warningf("%s isn't in simulation mode", scenario)
+						continue
+					}
+					if !*csConfig.Cscli.SimulationConfig.Simulation && isExcluded {
+						if err := removeFromExclusion(scenario); err != nil {
+							log.Fatal(err)
+						}
+						log.Printf("simulation mode for '%s' disabled", scenario)
+						continue
+					}
+					if isExcluded {
+						log.Warningf("simulation mode is enabled but is already disable for '%s'", scenario)
+						continue
+					}
+					if err := addToExclusion(scenario); err != nil {
+						log.Fatal(err)
+					}
+					log.Printf("simulation mode for '%s' disabled", scenario)
+				}
+				if err := dumpSimulationFile(); err != nil {
+					log.Fatalf("simulation disable: %s", err)
+				}
+			} else if forceGlobalSimulation {
+				if err := disableGlobalSimulation(); err != nil {
+					log.Fatalf("unable to disable global simulation mode : %s", err)
+				}
+			} else {
+				printHelp(cmd)
+			}
+		},
+	}
+	cmdSimulationDisable.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Disable global simulation (reverse mode)")
+
+	return cmdSimulationDisable
+}
+
+func NewSimulationStatusCmd() *cobra.Command {
+	var cmdSimulationStatus = &cobra.Command{
+		Use:               "status",
+		Short:             "Show simulation mode status",
+		Example:           `cscli simulation status`,
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			if err := simulationStatus(); err != nil {
+				log.Fatal(err)
+			}
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+		},
+	}
+
+	return cmdSimulationStatus
 }

cmd/crowdsec-cli/support.go

@@ -4,7 +4,6 @@ import (
 	"archive/zip"
 	"bytes"
 	"context"
-	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -13,23 +12,21 @@ import (
 	"path/filepath"
 	"regexp"
 	"strings"
-	"time"
 
 	"github.com/blackfireio/osinfo"
 	"github.com/go-openapi/strfmt"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 
-	"github.com/crowdsecurity/go-cs-lib/trace"
-	"github.com/crowdsecurity/go-cs-lib/version"
+	"github.com/crowdsecurity/go-cs-lib/pkg/version"
 
-	"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/fflag"
 	"github.com/crowdsecurity/crowdsec/pkg/models"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
 const (
@@ -40,7 +37,6 @@ const (
 	SUPPORT_OS_INFO_PATH                 = "osinfo.txt"
 	SUPPORT_PARSERS_PATH                 = "hub/parsers.txt"
 	SUPPORT_SCENARIOS_PATH               = "hub/scenarios.txt"
-	SUPPORT_CONTEXTS_PATH                = "hub/scenarios.txt"
 	SUPPORT_COLLECTIONS_PATH             = "hub/collections.txt"
 	SUPPORT_POSTOVERFLOWS_PATH           = "hub/postoverflows.txt"
 	SUPPORT_BOUNCERS_PATH                = "lapi/bouncers.txt"
@@ -50,54 +46,43 @@ const (
 	SUPPORT_CAPI_STATUS_PATH             = "capi_status.txt"
 	SUPPORT_ACQUISITION_CONFIG_BASE_PATH = "config/acquis/"
 	SUPPORT_CROWDSEC_PROFILE_PATH        = "config/profiles.yaml"
-	SUPPORT_CRASH_PATH                   = "crash/"
 )
 
-// from https://github.com/acarl005/stripansi
-var reStripAnsi = regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))")
-
-func stripAnsiString(str string) string {
-	// the byte version doesn't strip correctly
-	return reStripAnsi.ReplaceAllString(str, "")
-}
-
 func collectMetrics() ([]byte, []byte, error) {
 	log.Info("Collecting prometheus metrics")
-
-	if csConfig.Cscli.PrometheusUrl == "" {
-		log.Warn("No Prometheus URL configured, metrics will not be collected")
-		return nil, nil, errors.New("prometheus_uri is not set")
-	}
-
-	humanMetrics := bytes.NewBuffer(nil)
-
-	ms := NewMetricStore()
-
-	if err := ms.Fetch(csConfig.Cscli.PrometheusUrl); err != nil {
-		return nil, nil, fmt.Errorf("could not fetch prometheus metrics: %w", err)
-	}
-
-	if err := ms.Format(humanMetrics, nil, "human", false); err != nil {
+	err := csConfig.LoadPrometheus()
+	if err != nil {
 		return nil, nil, err
 	}
 
-	req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl, nil)
-	if err != nil {
-		return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %w", err)
+	if csConfig.Cscli.PrometheusUrl == "" {
+		log.Warn("No Prometheus URL configured, metrics will not be collected")
+		return nil, nil, fmt.Errorf("prometheus_uri is not set")
 	}
 
-	client := &http.Client{}
+	humanMetrics := bytes.NewBuffer(nil)
+	err = FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl+"/metrics", "human")
 
-	resp, err := client.Do(req)
 	if err != nil {
-		return nil, nil, fmt.Errorf("could not get metrics from prometheus endpoint: %w", err)
+		return nil, nil, fmt.Errorf("could not fetch promtheus metrics: %s", err)
+	}
+
+	req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl+"/metrics", nil)
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err)
+	}
+	client := &http.Client{}
+	resp, err := client.Do(req)
+
+	if err != nil {
+		return nil, nil, fmt.Errorf("could not get metrics from prometheus endpoint: %s", err)
 	}
 
 	defer resp.Body.Close()
 
 	body, err := io.ReadAll(resp.Body)
 	if err != nil {
-		return nil, nil, fmt.Errorf("could not read metrics from prometheus endpoint: %w", err)
+		return nil, nil, fmt.Errorf("could not read metrics from prometheus endpoint: %s", err)
 	}
 
 	return humanMetrics.Bytes(), body, nil
@@ -110,96 +95,89 @@ func collectVersion() []byte {
 
 func collectFeatures() []byte {
 	log.Info("Collecting feature flags")
-
 	enabledFeatures := fflag.Crowdsec.GetEnabledFeatures()
 
 	w := bytes.NewBuffer(nil)
 	for _, k := range enabledFeatures {
 		fmt.Fprintf(w, "%s\n", k)
 	}
-
 	return w.Bytes()
 }
 
 func collectOSInfo() ([]byte, error) {
 	log.Info("Collecting OS info")
-
 	info, err := osinfo.GetOSInfo()
+
 	if err != nil {
 		return nil, err
 	}
 
 	w := bytes.NewBuffer(nil)
-	fmt.Fprintf(w, "Architecture: %s\n", info.Architecture)
-	fmt.Fprintf(w, "Family: %s\n", info.Family)
-	fmt.Fprintf(w, "ID: %s\n", info.ID)
-	fmt.Fprintf(w, "Name: %s\n", info.Name)
-	fmt.Fprintf(w, "Codename: %s\n", info.Codename)
-	fmt.Fprintf(w, "Version: %s\n", info.Version)
-	fmt.Fprintf(w, "Build: %s\n", info.Build)
+	w.WriteString(fmt.Sprintf("Architecture: %s\n", info.Architecture))
+	w.WriteString(fmt.Sprintf("Family: %s\n", info.Family))
+	w.WriteString(fmt.Sprintf("ID: %s\n", info.ID))
+	w.WriteString(fmt.Sprintf("Name: %s\n", info.Name))
+	w.WriteString(fmt.Sprintf("Codename: %s\n", info.Codename))
+	w.WriteString(fmt.Sprintf("Version: %s\n", info.Version))
+	w.WriteString(fmt.Sprintf("Build: %s\n", info.Build))
 
 	return w.Bytes(), nil
 }
 
-func collectHubItems(hub *cwhub.Hub, itemType string) []byte {
-	var err error
+func initHub() error {
+	if err := csConfig.LoadHub(); err != nil {
+		return fmt.Errorf("cannot load hub: %s", err)
+	}
+	if csConfig.Hub == nil {
+		return fmt.Errorf("hub not configured")
+	}
 
+	if err := cwhub.SetHubBranch(); err != nil {
+		return fmt.Errorf("cannot set hub branch: %s", err)
+	}
+
+	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+		return fmt.Errorf("no hub index found: %s", err)
+	}
+	return nil
+}
+
+func collectHubItems(itemType string) []byte {
 	out := bytes.NewBuffer(nil)
-
 	log.Infof("Collecting %s list", itemType)
-
-	items := make(map[string][]*cwhub.Item)
-
-	if items[itemType], err = selectItems(hub, itemType, nil, true); err != nil {
-		log.Warnf("could not collect %s list: %s", itemType, err)
-	}
-
-	if err := listItems(out, []string{itemType}, items, false); err != nil {
-		log.Warnf("could not collect %s list: %s", itemType, err)
-	}
-
+	ListItems(out, []string{itemType}, []string{}, false, true, all)
 	return out.Bytes()
 }
 
 func collectBouncers(dbClient *database.Client) ([]byte, error) {
 	out := bytes.NewBuffer(nil)
-
-	bouncers, err := dbClient.ListBouncers()
+	err := getBouncers(out, dbClient)
 	if err != nil {
-		return nil, fmt.Errorf("unable to list bouncers: %w", err)
+		return nil, err
 	}
-
-	getBouncersTable(out, bouncers)
-
 	return out.Bytes(), nil
 }
 
 func collectAgents(dbClient *database.Client) ([]byte, error) {
 	out := bytes.NewBuffer(nil)
-
-	machines, err := dbClient.ListMachines()
+	err := getAgents(out, dbClient)
 	if err != nil {
-		return nil, fmt.Errorf("unable to list machines: %w", err)
+		return nil, err
 	}
-
-	getAgentsTable(out, machines)
-
 	return out.Bytes(), nil
 }
 
-func collectAPIStatus(login string, password string, endpoint string, prefix string, hub *cwhub.Hub) []byte {
+func collectAPIStatus(login string, password string, endpoint string, prefix string) []byte {
 	if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil {
 		return []byte("No agent credentials found, are we LAPI ?")
 	}
-
 	pwd := strfmt.Password(password)
-
 	apiurl, err := url.Parse(endpoint)
+
 	if err != nil {
 		return []byte(fmt.Sprintf("cannot parse API URL: %s", err))
 	}
-
-	scenarios, err := hub.GetInstalledNamesByType(cwhub.SCENARIOS)
+	scenarios, err := cwhub.GetInstalledScenariosAsString()
 	if err != nil {
 		return []byte(fmt.Sprintf("could not collect scenarios: %s", err))
 	}
@@ -211,7 +189,6 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
 	if err != nil {
 		return []byte(fmt.Sprintf("could not init client: %s", err))
 	}
-
 	t := models.WatcherAuthRequest{
 		MachineID: &login,
 		Password:  &pwd,
@@ -228,7 +205,6 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
 
 func collectCrowdsecConfig() []byte {
 	log.Info("Collecting crowdsec config")
-
 	config, err := os.ReadFile(*csConfig.FilePath)
 	if err != nil {
 		return []byte(fmt.Sprintf("could not read config file: %s", err))
@@ -241,18 +217,15 @@ func collectCrowdsecConfig() []byte {
 
 func collectCrowdsecProfile() []byte {
 	log.Info("Collecting crowdsec profile")
-
 	config, err := os.ReadFile(csConfig.API.Server.ProfilesPath)
 	if err != nil {
 		return []byte(fmt.Sprintf("could not read profile file: %s", err))
 	}
-
 	return config
 }
 
 func collectAcquisitionConfig() map[string][]byte {
 	log.Info("Collecting acquisition config")
-
 	ret := make(map[string][]byte)
 
 	for _, filename := range csConfig.Crowdsec.AcquisitionFiles {
@@ -267,19 +240,8 @@ func collectAcquisitionConfig() map[string][]byte {
 	return ret
 }
 
-func collectCrash() ([]string, error) {
-	log.Info("Collecting crash dumps")
-	return trace.List()
-}
-
-type cliSupport struct{}
-
-func NewCLISupport() *cliSupport {
-	return &cliSupport{}
-}
-
-func (cli cliSupport) NewCommand() *cobra.Command {
-	cmd := &cobra.Command{
+func NewSupportCmd() *cobra.Command {
+	var cmdSupport = &cobra.Command{
 		Use:               "support [action]",
 		Short:             "Provide commands to help during support",
 		Args:              cobra.MinimumNArgs(1),
@@ -289,15 +251,9 @@ func (cli cliSupport) NewCommand() *cobra.Command {
 		},
 	}
 
-	cmd.AddCommand(cli.NewDumpCmd())
-
-	return cmd
-}
-
-func (cli cliSupport) NewDumpCmd() *cobra.Command {
 	var outFile string
 
-	cmd := &cobra.Command{
+	cmdDump := &cobra.Command{
 		Use:   "dump",
 		Short: "Dump all your configuration to a zip file for easier support",
 		Long: `Dump the following informations:
@@ -307,7 +263,6 @@ func (cli cliSupport) NewDumpCmd() *cobra.Command {
 - Installed parsers list
 - Installed scenarios list
 - Installed postoverflows list
-- Installed context list
 - Bouncers list
 - Machines list
 - CAPI status
@@ -319,7 +274,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
 `,
 		Args:              cobra.NoArgs,
 		DisableAutoGenTag: true,
-		RunE: func(_ *cobra.Command, _ []string) error {
+		Run: func(cmd *cobra.Command, args []string) {
 			var err error
 			var skipHub, skipDB, skipCAPI, skipLAPI, skipAgent bool
 			infos := map[string][]byte{
@@ -339,25 +294,24 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				infos[SUPPORT_AGENTS_PATH] = []byte(err.Error())
 			}
 
-			if err = csConfig.LoadAPIServer(true); err != nil {
+			if err := csConfig.LoadAPIServer(); err != nil {
 				log.Warnf("could not load LAPI, skipping CAPI check")
 				skipLAPI = true
 				infos[SUPPORT_CAPI_STATUS_PATH] = []byte(err.Error())
 			}
 
-			if err = csConfig.LoadCrowdsec(); err != nil {
+			if err := csConfig.LoadCrowdsec(); err != nil {
 				log.Warnf("could not load agent config, skipping crowdsec config check")
 				skipAgent = true
 			}
 
-			hub, err := require.Hub(csConfig, nil, nil)
+			err = initHub()
 			if err != nil {
 				log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
 				skipHub = true
 				infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())
 				infos[SUPPORT_SCENARIOS_PATH] = []byte(err.Error())
 				infos[SUPPORT_POSTOVERFLOWS_PATH] = []byte(err.Error())
-				infos[SUPPORT_CONTEXTS_PATH] = []byte(err.Error())
 				infos[SUPPORT_COLLECTIONS_PATH] = []byte(err.Error())
 			}
 
@@ -390,11 +344,10 @@ cscli support dump -f /tmp/crowdsec-support.zip
 			infos[SUPPORT_CROWDSEC_CONFIG_PATH] = collectCrowdsecConfig()
 
 			if !skipHub {
-				infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS)
-				infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS)
-				infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
-				infos[SUPPORT_CONTEXTS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
-				infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS)
+				infos[SUPPORT_PARSERS_PATH] = collectHubItems(cwhub.PARSERS)
+				infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(cwhub.SCENARIOS)
+				infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.PARSERS_OVFLW)
+				infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(cwhub.COLLECTIONS)
 			}
 
 			if !skipDB {
@@ -416,8 +369,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				infos[SUPPORT_CAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Server.OnlineClient.Credentials.Login,
 					csConfig.API.Server.OnlineClient.Credentials.Password,
 					csConfig.API.Server.OnlineClient.Credentials.URL,
-					CAPIURLPrefix,
-					hub)
+					CAPIURLPrefix)
 			}
 
 			if !skipLAPI {
@@ -425,12 +377,12 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				infos[SUPPORT_LAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Client.Credentials.Login,
 					csConfig.API.Client.Credentials.Password,
 					csConfig.API.Client.Credentials.URL,
-					LAPIURLPrefix,
-					hub)
+					LAPIURLPrefix)
 				infos[SUPPORT_CROWDSEC_PROFILE_PATH] = collectCrowdsecProfile()
 			}
 
 			if !skipAgent {
+
 				acquis := collectAcquisitionConfig()
 
 				for filename, content := range acquis {
@@ -439,57 +391,33 @@ cscli support dump -f /tmp/crowdsec-support.zip
 				}
 			}
 
-			crash, err := collectCrash()
-			if err != nil {
-				log.Errorf("could not collect crash dumps: %s", err)
-			}
-
-			for _, filename := range crash {
-				content, err := os.ReadFile(filename)
-				if err != nil {
-					log.Errorf("could not read crash dump %s: %s", filename, err)
-				}
-
-				infos[SUPPORT_CRASH_PATH+filepath.Base(filename)] = content
-			}
-
 			w := bytes.NewBuffer(nil)
 			zipWriter := zip.NewWriter(w)
 
 			for filename, data := range infos {
-				header := &zip.FileHeader{
-					Name:   filename,
-					Method: zip.Deflate,
-					// TODO: retain mtime where possible (esp. trace)
-					Modified: time.Now(),
-				}
-				fw, err := zipWriter.CreateHeader(header)
+				fw, err := zipWriter.Create(filename)
 				if err != nil {
 					log.Errorf("Could not add zip entry for %s: %s", filename, err)
 					continue
 				}
-				fw.Write([]byte(stripAnsiString(string(data))))
+				fw.Write([]byte(types.StripAnsiString(string(data))))
 			}
 
 			err = zipWriter.Close()
 			if err != nil {
-				return fmt.Errorf("could not finalize zip file: %s", err)
+				log.Fatalf("could not finalize zip file: %s", err)
 			}
 
-			if outFile == "-" {
-				_, err = os.Stdout.Write(w.Bytes())
-				return err
-			}
-			err = os.WriteFile(outFile, w.Bytes(), 0o600)
+			err = os.WriteFile(outFile, w.Bytes(), 0600)
 			if err != nil {
-				return fmt.Errorf("could not write zip file to %s: %s", outFile, err)
+				log.Fatalf("could not write zip file to %s: %s", outFile, err)
 			}
+
 			log.Infof("Written zip file to %s", outFile)
-			return nil
 		},
 	}
+	cmdDump.Flags().StringVarP(&outFile, "outFile", "f", "", "File to dump the information to")
+	cmdSupport.AddCommand(cmdDump)
 
-	cmd.Flags().StringVarP(&outFile, "outFile", "f", "", "File to dump the information to")
-
-	return cmd
+	return cmdSupport
 }

cmd/crowdsec-cli/utils.go

@@ -1,23 +1,279 @@
 package main
 
 import (
+	"encoding/csv"
+	"encoding/json"
 	"fmt"
+	"io"
+	"math"
 	"net"
+	"net/http"
+	"os"
+	"strconv"
 	"strings"
+	"time"
 
+	"github.com/fatih/color"
+	dto "github.com/prometheus/client_model/go"
+	"github.com/prometheus/prom2json"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
+	"github.com/texttheater/golang-levenshtein/levenshtein"
+	"golang.org/x/exp/slices"
+	"gopkg.in/yaml.v2"
 
+	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+	"github.com/crowdsecurity/crowdsec/pkg/database"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
+const MaxDistance = 7
+
 func printHelp(cmd *cobra.Command) {
-	if err := cmd.Help(); err != nil {
+	err := cmd.Help()
+	if err != nil {
 		log.Fatalf("unable to print help(): %s", err)
 	}
 }
 
+func indexOf(s string, slice []string) int {
+	for i, elem := range slice {
+		if s == elem {
+			return i
+		}
+	}
+	return -1
+}
+
+func LoadHub() error {
+	if err := csConfig.LoadHub(); err != nil {
+		log.Fatal(err)
+	}
+	if csConfig.Hub == nil {
+		return fmt.Errorf("unable to load hub")
+	}
+
+	if err := cwhub.SetHubBranch(); err != nil {
+		log.Warningf("unable to set hub branch (%s), default to master", err)
+	}
+
+	if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+		return fmt.Errorf("Failed to get Hub index : '%w'. Run 'sudo cscli hub update' to get the hub index", err)
+	}
+
+	return nil
+}
+
+func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
+	errMsg := ""
+	if score < MaxDistance {
+		errMsg = fmt.Sprintf("unable to find %s '%s', did you mean %s ?", itemType, baseItem, suggestItem)
+	} else {
+		errMsg = fmt.Sprintf("unable to find %s '%s'", itemType, baseItem)
+	}
+	if ignoreErr {
+		log.Error(errMsg)
+	} else {
+		log.Fatalf(errMsg)
+	}
+}
+
+func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
+	allItems := make([]string, 0)
+	nearestScore := 100
+	nearestItem := &cwhub.Item{}
+	hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
+	for _, item := range hubItems {
+		allItems = append(allItems, item.Name)
+	}
+
+	for _, s := range allItems {
+		d := levenshtein.DistanceForStrings([]rune(itemName), []rune(s), levenshtein.DefaultOptions)
+		if d < nearestScore {
+			nearestScore = d
+			nearestItem = cwhub.GetItem(itemType, s)
+		}
+	}
+	return nearestItem, nearestScore
+}
+
+func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+	if err := LoadHub(); err != nil {
+		return nil, cobra.ShellCompDirectiveDefault
+	}
+
+	comp := make([]string, 0)
+	hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
+	for _, item := range hubItems {
+		if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
+			comp = append(comp, item.Name)
+		}
+	}
+	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
+	return comp, cobra.ShellCompDirectiveNoFileComp
+}
+
+func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+	if err := LoadHub(); err != nil {
+		return nil, cobra.ShellCompDirectiveDefault
+	}
+
+	var items []string
+	var err error
+	switch itemType {
+	case cwhub.PARSERS:
+		items, err = cwhub.GetInstalledParsersAsString()
+	case cwhub.SCENARIOS:
+		items, err = cwhub.GetInstalledScenariosAsString()
+	case cwhub.PARSERS_OVFLW:
+		items, err = cwhub.GetInstalledPostOverflowsAsString()
+	case cwhub.COLLECTIONS:
+		items, err = cwhub.GetInstalledCollectionsAsString()
+	case cwhub.WAF_RULES:
+		items, err = cwhub.GetInstalledWafRulesAsString()
+	default:
+		return nil, cobra.ShellCompDirectiveDefault
+	}
+
+	if err != nil {
+		cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
+		return nil, cobra.ShellCompDirectiveDefault
+	}
+	comp := make([]string, 0)
+
+	if toComplete != "" {
+		for _, item := range items {
+			if strings.Contains(item, toComplete) {
+				comp = append(comp, item)
+			}
+		}
+	} else {
+		comp = items
+	}
+
+	cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
+
+	return comp, cobra.ShellCompDirectiveNoFileComp
+}
+
+func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) {
+	var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus)
+
+	for _, itemType := range itemTypes {
+		itemName := ""
+		if len(args) == 1 {
+			itemName = args[0]
+		}
+		hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all)
+	}
+
+	if csConfig.Cscli.Output == "human" {
+		for _, itemType := range itemTypes {
+			var statuses []cwhub.ItemHubStatus
+			var ok bool
+			if statuses, ok = hubStatusByItemType[itemType]; !ok {
+				log.Errorf("unknown item type: %s", itemType)
+				continue
+			}
+			listHubItemTable(out, "\n"+strings.ToUpper(itemType), statuses)
+		}
+	} else if csConfig.Cscli.Output == "json" {
+		x, err := json.MarshalIndent(hubStatusByItemType, "", " ")
+		if err != nil {
+			log.Fatalf("failed to unmarshal")
+		}
+		out.Write(x)
+	} else if csConfig.Cscli.Output == "raw" {
+		csvwriter := csv.NewWriter(out)
+		if showHeader {
+			header := []string{"name", "status", "version", "description"}
+			if showType {
+				header = append(header, "type")
+			}
+			err := csvwriter.Write(header)
+			if err != nil {
+				log.Fatalf("failed to write header: %s", err)
+			}
+
+		}
+		for _, itemType := range itemTypes {
+			var statuses []cwhub.ItemHubStatus
+			var ok bool
+			if statuses, ok = hubStatusByItemType[itemType]; !ok {
+				log.Errorf("unknown item type: %s", itemType)
+				continue
+			}
+			for _, status := range statuses {
+				if status.LocalVersion == "" {
+					status.LocalVersion = "n/a"
+				}
+				row := []string{
+					status.Name,
+					status.Status,
+					status.LocalVersion,
+					status.Description,
+				}
+				if showType {
+					row = append(row, itemType)
+				}
+				err := csvwriter.Write(row)
+				if err != nil {
+					log.Fatalf("failed to write raw output : %s", err)
+				}
+			}
+		}
+		csvwriter.Flush()
+	}
+}
+
+func InspectItem(name string, objecitemType string) {
+
+	hubItem := cwhub.GetItem(objecitemType, name)
+	if hubItem == nil {
+		log.Fatalf("unable to retrieve item.")
+	}
+	var b []byte
+	var err error
+	switch csConfig.Cscli.Output {
+	case "human", "raw":
+		b, err = yaml.Marshal(*hubItem)
+		if err != nil {
+			log.Fatalf("unable to marshal item : %s", err)
+		}
+	case "json":
+		b, err = json.MarshalIndent(*hubItem, "", " ")
+		if err != nil {
+			log.Fatalf("unable to marshal item : %s", err)
+		}
+	}
+	fmt.Printf("%s", string(b))
+	if csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
+		return
+	}
+
+	if prometheusURL == "" {
+		//This is technically wrong to do this, as the prometheus section contains a listen address, not an URL to query prometheus
+		//But for ease of use, we will use the listen address as the prometheus URL because it will be 127.0.0.1 in the default case
+		listenAddr := csConfig.Prometheus.ListenAddr
+		if listenAddr == "" {
+			listenAddr = "127.0.0.1"
+		}
+		listenPort := csConfig.Prometheus.ListenPort
+		if listenPort == 0 {
+			listenPort = 6060
+		}
+		prometheusURL = fmt.Sprintf("http://%s:%d/metrics", listenAddr, listenPort)
+		log.Debugf("No prometheus URL provided using: %s", prometheusURL)
+	}
+
+	fmt.Printf("\nCurrent metrics : \n")
+	ShowMetrics(hubItem)
+}
+
 func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
+
 	/*if a range is provided, change the scope*/
 	if *ipRange != "" {
 		_, _, err := net.ParseCIDR(*ipRange)
@@ -25,7 +281,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
 			return fmt.Errorf("%s isn't a valid range", *ipRange)
 		}
 	}
-
 	if *ip != "" {
 		ipRepr := net.ParseIP(*ip)
 		if ipRepr == nil {
@@ -33,7 +288,7 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
 		}
 	}
 
-	// avoid confusion on scope (ip vs Ip and range vs Range)
+	//avoid confusion on scope (ip vs Ip and range vs Range)
 	switch strings.ToLower(*scope) {
 	case "ip":
 		*scope = types.Ip
@@ -44,10 +299,434 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
 	case "as":
 		*scope = types.AS
 	}
+	return nil
+}
+
+func ShowMetrics(hubItem *cwhub.Item) {
+	switch hubItem.Type {
+	case cwhub.PARSERS:
+		metrics := GetParserMetric(prometheusURL, hubItem.Name)
+		parserMetricsTable(color.Output, hubItem.Name, metrics)
+	case cwhub.SCENARIOS:
+		metrics := GetScenarioMetric(prometheusURL, hubItem.Name)
+		scenarioMetricsTable(color.Output, hubItem.Name, metrics)
+	case cwhub.COLLECTIONS:
+		for _, item := range hubItem.Parsers {
+			metrics := GetParserMetric(prometheusURL, item)
+			parserMetricsTable(color.Output, item, metrics)
+		}
+		for _, item := range hubItem.Scenarios {
+			metrics := GetScenarioMetric(prometheusURL, item)
+			scenarioMetricsTable(color.Output, item, metrics)
+		}
+		for _, item := range hubItem.Collections {
+			hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
+			if hubItem == nil {
+				log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
+			}
+			ShowMetrics(hubItem)
+		}
+	case cwhub.WAF_RULES:
+		log.Fatalf("FIXME: not implemented yet")
+	default:
+		log.Errorf("item of type '%s' is unknown", hubItem.Type)
+	}
+}
+
+// GetParserMetric is a complete rip from prom2json
+func GetParserMetric(url string, itemName string) map[string]map[string]int {
+	stats := make(map[string]map[string]int)
+
+	result := GetPrometheusMetric(url)
+	for idx, fam := range result {
+		if !strings.HasPrefix(fam.Name, "cs_") {
+			continue
+		}
+		log.Tracef("round %d", idx)
+		for _, m := range fam.Metrics {
+			metric, ok := m.(prom2json.Metric)
+			if !ok {
+				log.Debugf("failed to convert metric to prom2json.Metric")
+				continue
+			}
+			name, ok := metric.Labels["name"]
+			if !ok {
+				log.Debugf("no name in Metric %v", metric.Labels)
+			}
+			if name != itemName {
+				continue
+			}
+			source, ok := metric.Labels["source"]
+			if !ok {
+				log.Debugf("no source in Metric %v", metric.Labels)
+			} else {
+				if srctype, ok := metric.Labels["type"]; ok {
+					source = srctype + ":" + source
+				}
+			}
+			value := m.(prom2json.Metric).Value
+			fval, err := strconv.ParseFloat(value, 32)
+			if err != nil {
+				log.Errorf("Unexpected int value %s : %s", value, err)
+				continue
+			}
+			ival := int(fval)
+
+			switch fam.Name {
+			case "cs_reader_hits_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+					stats[source]["parsed"] = 0
+					stats[source]["reads"] = 0
+					stats[source]["unparsed"] = 0
+					stats[source]["hits"] = 0
+				}
+				stats[source]["reads"] += ival
+			case "cs_parser_hits_ok_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["parsed"] += ival
+			case "cs_parser_hits_ko_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["unparsed"] += ival
+			case "cs_node_hits_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["hits"] += ival
+			case "cs_node_hits_ok_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["parsed"] += ival
+			case "cs_node_hits_ko_total":
+				if _, ok := stats[source]; !ok {
+					stats[source] = make(map[string]int)
+				}
+				stats[source]["unparsed"] += ival
+			default:
+				continue
+			}
+		}
+	}
+	return stats
+}
+
+func GetScenarioMetric(url string, itemName string) map[string]int {
+	stats := make(map[string]int)
+
+	stats["instantiation"] = 0
+	stats["curr_count"] = 0
+	stats["overflow"] = 0
+	stats["pour"] = 0
+	stats["underflow"] = 0
+
+	result := GetPrometheusMetric(url)
+	for idx, fam := range result {
+		if !strings.HasPrefix(fam.Name, "cs_") {
+			continue
+		}
+		log.Tracef("round %d", idx)
+		for _, m := range fam.Metrics {
+			metric, ok := m.(prom2json.Metric)
+			if !ok {
+				log.Debugf("failed to convert metric to prom2json.Metric")
+				continue
+			}
+			name, ok := metric.Labels["name"]
+			if !ok {
+				log.Debugf("no name in Metric %v", metric.Labels)
+			}
+			if name != itemName {
+				continue
+			}
+			value := m.(prom2json.Metric).Value
+			fval, err := strconv.ParseFloat(value, 32)
+			if err != nil {
+				log.Errorf("Unexpected int value %s : %s", value, err)
+				continue
+			}
+			ival := int(fval)
+
+			switch fam.Name {
+			case "cs_bucket_created_total":
+				stats["instantiation"] += ival
+			case "cs_buckets":
+				stats["curr_count"] += ival
+			case "cs_bucket_overflowed_total":
+				stats["overflow"] += ival
+			case "cs_bucket_poured_total":
+				stats["pour"] += ival
+			case "cs_bucket_underflowed_total":
+				stats["underflow"] += ival
+			default:
+				continue
+			}
+		}
+	}
+	return stats
+}
+
+// it's a rip of the cli version, but in silent-mode
+func silenceInstallItem(name string, obtype string) (string, error) {
+	var item = cwhub.GetItem(obtype, name)
+	if item == nil {
+		return "", fmt.Errorf("error retrieving item")
+	}
+	it := *item
+	if downloadOnly && it.Downloaded && it.UpToDate {
+		return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
+	}
+	it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction, false)
+	if err != nil {
+		return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
+	}
+	if err := cwhub.AddItem(obtype, it); err != nil {
+		return "", err
+	}
+
+	if downloadOnly {
+		return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
+	}
+	it, err = cwhub.EnableItem(csConfig.Hub, it)
+	if err != nil {
+		return "", fmt.Errorf("error while enabling %s : %v", it.Name, err)
+	}
+	if err := cwhub.AddItem(obtype, it); err != nil {
+		return "", err
+	}
+	return fmt.Sprintf("Enabled %s", it.Name), nil
+}
+
+func GetPrometheusMetric(url string) []*prom2json.Family {
+	mfChan := make(chan *dto.MetricFamily, 1024)
+
+	// Start with the DefaultTransport for sane defaults.
+	transport := http.DefaultTransport.(*http.Transport).Clone()
+	// Conservatively disable HTTP keep-alives as this program will only
+	// ever need a single HTTP request.
+	transport.DisableKeepAlives = true
+	// Timeout early if the server doesn't even return the headers.
+	transport.ResponseHeaderTimeout = time.Minute
+
+	go func() {
+		defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
+		err := prom2json.FetchMetricFamilies(url, mfChan, transport)
+		if err != nil {
+			log.Fatalf("failed to fetch prometheus metrics : %v", err)
+		}
+	}()
+
+	result := []*prom2json.Family{}
+	for mf := range mfChan {
+		result = append(result, prom2json.NewFamily(mf))
+	}
+	log.Debugf("Finished reading prometheus output, %d entries", len(result))
+
+	return result
+}
+
+func RestoreHub(dirPath string) error {
+	var err error
+
+	if err := csConfig.LoadHub(); err != nil {
+		return err
+	}
+	if err := cwhub.SetHubBranch(); err != nil {
+		return fmt.Errorf("error while setting hub branch: %s", err)
+	}
+
+	for _, itype := range cwhub.ItemTypes {
+		itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
+		if _, err = os.Stat(itemDirectory); err != nil {
+			log.Infof("no %s in backup", itype)
+			continue
+		}
+		/*restore the upstream items*/
+		upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
+		file, err := os.ReadFile(upstreamListFN)
+		if err != nil {
+			return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
+		}
+		var upstreamList []string
+		err = json.Unmarshal(file, &upstreamList)
+		if err != nil {
+			return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
+		}
+		for _, toinstall := range upstreamList {
+			label, err := silenceInstallItem(toinstall, itype)
+			if err != nil {
+				log.Errorf("Error while installing %s : %s", toinstall, err)
+			} else if label != "" {
+				log.Infof("Installed %s : %s", toinstall, label)
+			} else {
+				log.Printf("Installed %s : ok", toinstall)
+			}
+		}
+
+		/*restore the local and tainted items*/
+		files, err := os.ReadDir(itemDirectory)
+		if err != nil {
+			return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
+		}
+		for _, file := range files {
+			//this was the upstream data
+			if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
+				continue
+			}
+			if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
+				//we expect a stage here
+				if !file.IsDir() {
+					continue
+				}
+				stage := file.Name()
+				stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
+				log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
+				if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
+					return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
+				}
+				/*find items*/
+				ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
+				if err != nil {
+					return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
+				}
+				//finally copy item
+				for _, tfile := range ifiles {
+					log.Infof("Going to restore local/tainted [%s]", tfile.Name())
+					sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
+					destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
+					if err = types.CopyFile(sourceFile, destinationFile); err != nil {
+						return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
+					}
+					log.Infof("restored %s to %s", sourceFile, destinationFile)
+				}
+			} else {
+				log.Infof("Going to restore local/tainted [%s]", file.Name())
+				sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
+				destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
+				if err = types.CopyFile(sourceFile, destinationFile); err != nil {
+					return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
+				}
+				log.Infof("restored %s to %s", sourceFile, destinationFile)
+			}
+
+		}
+	}
+	return nil
+}
+
+func BackupHub(dirPath string) error {
+	var err error
+	var itemDirectory string
+	var upstreamParsers []string
+
+	for _, itemType := range cwhub.ItemTypes {
+		clog := log.WithFields(log.Fields{
+			"type": itemType,
+		})
+		itemMap := cwhub.GetItemMap(itemType)
+		if itemMap == nil {
+			clog.Infof("No %s to backup.", itemType)
+			continue
+		}
+		itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
+		if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
+			return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
+		}
+		upstreamParsers = []string{}
+		for k, v := range itemMap {
+			clog = clog.WithFields(log.Fields{
+				"file": v.Name,
+			})
+			if !v.Installed { //only backup installed ones
+				clog.Debugf("[%s] : not installed", k)
+				continue
+			}
+
+			//for the local/tainted ones, we backup the full file
+			if v.Tainted || v.Local || !v.UpToDate {
+				//we need to backup stages for parsers
+				if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
+					fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
+					if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
+						return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
+					}
+				}
+				clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
+				tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
+				if err = types.CopyFile(v.LocalPath, tfile); err != nil {
+					return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
+				}
+				clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
+				continue
+			}
+			clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
+			clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
+			upstreamParsers = append(upstreamParsers, v.Name)
+		}
+		//write the upstream items
+		upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
+		upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
+		if err != nil {
+			return fmt.Errorf("failed marshaling upstream parsers : %s", err)
+		}
+		err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
+		if err != nil {
+			return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
+		}
+		clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
+	}
 
 	return nil
 }
 
+type unit struct {
+	value  int64
+	symbol string
+}
+
+var ranges = []unit{
+	{value: 1e18, symbol: "E"},
+	{value: 1e15, symbol: "P"},
+	{value: 1e12, symbol: "T"},
+	{value: 1e9,  symbol: "G"},
+	{value: 1e6,  symbol: "M"},
+	{value: 1e3,  symbol: "k"},
+	{value: 1,    symbol: ""},
+}
+
+func formatNumber(num int) string {
+	goodUnit := unit{}
+	for _, u := range ranges {
+		if int64(num) >= u.value {
+			goodUnit = u
+			break
+		}
+	}
+
+	if goodUnit.value == 1 {
+		return fmt.Sprintf("%d%s", num, goodUnit.symbol)
+	}
+
+	res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
+	return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
+}
+
+func getDBClient() (*database.Client, error) {
+	var err error
+	if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
+		return nil, err
+	}
+	ret, err := database.NewClient(csConfig.DbConfig)
+	if err != nil {
+		return nil, err
+	}
+	return ret, nil
+}
+
 func removeFromSlice(val string, slice []string) []string {
 	var i int
 	var value string
@@ -69,4 +748,5 @@ func removeFromSlice(val string, slice []string) []string {
 	}
 
 	return slice
+
 }

cmd/crowdsec-cli/utils_table.go

@@ -3,56 +3,39 @@ package main
 import (
 	"fmt"
 	"io"
-	"strconv"
 
 	"github.com/aquasecurity/table"
+	"github.com/enescakir/emoji"
 
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-	"github.com/crowdsecurity/crowdsec/pkg/emoji"
 )
 
-func listHubItemTable(out io.Writer, title string, items []*cwhub.Item) {
+func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatus) {
 	t := newLightTable(out)
 	t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path")
 	t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 	t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
 
-	for _, item := range items {
-		status := fmt.Sprintf("%v  %s", item.State.Emoji(), item.State.Text())
-		t.AddRow(item.Name, status, item.State.LocalVersion, item.State.LocalPath)
+	for _, status := range statuses {
+		t.AddRow(status.Name, status.UTF8_Status, status.LocalVersion, status.LocalPath)
 	}
-
 	renderTableTitle(out, title)
 	t.Render()
 }
 
-func appsecMetricsTable(out io.Writer, itemName string, metrics map[string]int) {
-	t := newTable(out)
-	t.SetHeaders("Inband Hits", "Outband Hits")
-
-	t.AddRow(
-		strconv.Itoa(metrics["inband_hits"]),
-		strconv.Itoa(metrics["outband_hits"]),
-	)
-
-	renderTableTitle(out, fmt.Sprintf("\n - (AppSec Rule) %s:", itemName))
-	t.Render()
-}
-
 func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int) {
 	if metrics["instantiation"] == 0 {
 		return
 	}
-
 	t := newTable(out)
 	t.SetHeaders("Current Count", "Overflows", "Instantiated", "Poured", "Expired")
 
 	t.AddRow(
-		strconv.Itoa(metrics["curr_count"]),
-		strconv.Itoa(metrics["overflow"]),
-		strconv.Itoa(metrics["instantiation"]),
-		strconv.Itoa(metrics["pour"]),
-		strconv.Itoa(metrics["underflow"]),
+		fmt.Sprintf("%d", metrics["curr_count"]),
+		fmt.Sprintf("%d", metrics["overflow"]),
+		fmt.Sprintf("%d", metrics["instantiation"]),
+		fmt.Sprintf("%d", metrics["pour"]),
+		fmt.Sprintf("%d", metrics["underflow"]),
 	)
 
 	renderTableTitle(out, fmt.Sprintf("\n - (Scenario) %s:", itemName))
@@ -60,26 +43,23 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
 }
 
 func parserMetricsTable(out io.Writer, itemName string, metrics map[string]map[string]int) {
+	skip := true
 	t := newTable(out)
 	t.SetHeaders("Parsers", "Hits", "Parsed", "Unparsed")
 
-	// don't show table if no hits
-	showTable := false
-
 	for source, stats := range metrics {
 		if stats["hits"] > 0 {
 			t.AddRow(
 				source,
-				strconv.Itoa(stats["hits"]),
-				strconv.Itoa(stats["parsed"]),
-				strconv.Itoa(stats["unparsed"]),
+				fmt.Sprintf("%d", stats["hits"]),
+				fmt.Sprintf("%d", stats["parsed"]),
+				fmt.Sprintf("%d", stats["unparsed"]),
 			)
-
-			showTable = true
+			skip = false
 		}
 	}
 
-	if showTable {
+	if !skip {
 		renderTableTitle(out, fmt.Sprintf("\n - (Parser) %s:", itemName))
 		t.Render()
 	}

cmd/crowdsec-cli/version.go

@@ -1,27 +0,0 @@
-package main
-
-import (
-	"github.com/spf13/cobra"
-
-	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
-)
-
-type cliVersion struct{}
-
-func NewCLIVersion() *cliVersion {
-	return &cliVersion{}
-}
-
-func (cli cliVersion) NewCommand() *cobra.Command {
-	cmd := &cobra.Command{
-		Use:               "version",
-		Short:             "Display version",
-		Args:              cobra.ExactArgs(0),
-		DisableAutoGenTag: true,
-		Run: func(_ *cobra.Command, _ []string) {
-			cwversion.Show()
-		},
-	}
-
-	return cmd
-}

cmd/crowdsec-cli/waf_rules.go

@@ -0,0 +1,196 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/fatih/color"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
+)
+
+func NewWafRulesCmd() *cobra.Command {
+	var cmdWafRules = &cobra.Command{
+		Use:   "waf-rules [action] [config]",
+		Short: "Install/Remove/Upgrade/Inspect waf-rule(s) from hub",
+		Example: `cscli waf-rules install crowdsecurity/core-rule-set
+cscli waf-rules inspect crowdsecurity/core-rule-set
+cscli waf-rules upgrade crowdsecurity/core-rule-set
+cscli waf-rules list
+cscli waf-rules remove crowdsecurity/core-rule-set
+`,
+		Args:              cobra.MinimumNArgs(1),
+		Aliases:           []string{"waf-rule"},
+		DisableAutoGenTag: true,
+		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
+			if err := csConfig.LoadHub(); err != nil {
+				log.Fatal(err)
+			}
+			if csConfig.Hub == nil {
+				return fmt.Errorf("you must configure cli before interacting with hub")
+			}
+
+			if err := cwhub.SetHubBranch(); err != nil {
+				return fmt.Errorf("error while setting hub branch: %s", err)
+			}
+
+			if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
+				log.Info("Run 'sudo cscli hub update' to get the hub index")
+				log.Fatalf("Failed to get Hub index : %v", err)
+			}
+			return nil
+		},
+		PersistentPostRun: func(cmd *cobra.Command, args []string) {
+			if cmd.Name() == "inspect" || cmd.Name() == "list" {
+				return
+			}
+			log.Infof(ReloadMessage())
+		},
+	}
+
+	cmdWafRules.AddCommand(NewWafRulesInstallCmd())
+	cmdWafRules.AddCommand(NewWafRulesRemoveCmd())
+	cmdWafRules.AddCommand(NewWafRulesUpgradeCmd())
+	cmdWafRules.AddCommand(NewWafRulesInspectCmd())
+	cmdWafRules.AddCommand(NewWafRulesListCmd())
+
+	return cmdWafRules
+}
+
+func NewWafRulesInstallCmd() *cobra.Command {
+	var ignoreError bool
+
+	var cmdWafRulesInstall = &cobra.Command{
+		Use:               "install [config]",
+		Short:             "Install given waf-rule(s)",
+		Long:              `Fetch and install given waf-rule(s) from hub`,
+		Example:           `cscli waf-rules install crowdsec/xxx crowdsec/xyz`,
+		Args:              cobra.MinimumNArgs(1),
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compAllItems(cwhub.WAF_RULES, args, toComplete)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			for _, name := range args {
+				t := cwhub.GetItem(cwhub.WAF_RULES, name)
+				if t == nil {
+					nearestItem, score := GetDistance(cwhub.WAF_RULES, name)
+					Suggest(cwhub.WAF_RULES, name, nearestItem.Name, score, ignoreError)
+					continue
+				}
+				if err := cwhub.InstallItem(csConfig, name, cwhub.WAF_RULES, forceAction, downloadOnly); err != nil {
+					if ignoreError {
+						log.Errorf("Error while installing '%s': %s", name, err)
+					} else {
+						log.Fatalf("Error while installing '%s': %s", name, err)
+					}
+				}
+			}
+		},
+	}
+	cmdWafRulesInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
+	cmdWafRulesInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
+	cmdWafRulesInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple waf rules")
+
+	return cmdWafRulesInstall
+}
+
+func NewWafRulesRemoveCmd() *cobra.Command {
+	var cmdWafRulesRemove = &cobra.Command{
+		Use:               "remove [config]",
+		Short:             "Remove given waf-rule(s)",
+		Long:              `Remove given waf-rule(s) from hub`,
+		Aliases:           []string{"delete"},
+		Example:           `cscli waf-rules remove crowdsec/xxx crowdsec/xyz`,
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(cwhub.WAF_RULES, args, toComplete)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			if all {
+				cwhub.RemoveMany(csConfig, cwhub.WAF_RULES, "", all, purge, forceAction)
+				return
+			}
+
+			if len(args) == 0 {
+				log.Fatalf("Specify at least one waf rule to remove or '--all' flag.")
+			}
+
+			for _, name := range args {
+				cwhub.RemoveMany(csConfig, cwhub.WAF_RULES, name, all, purge, forceAction)
+			}
+		},
+	}
+	cmdWafRulesRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
+	cmdWafRulesRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
+	cmdWafRulesRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the waf rules")
+
+	return cmdWafRulesRemove
+}
+
+func NewWafRulesUpgradeCmd() *cobra.Command {
+	var cmdWafRulesUpgrade = &cobra.Command{
+		Use:               "upgrade [config]",
+		Short:             "Upgrade given waf-rule(s)",
+		Long:              `Fetch and upgrade given waf-rule(s) from hub`,
+		Example:           `cscli waf-rules upgrade crowdsec/xxx crowdsec/xyz`,
+		DisableAutoGenTag: true,
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(cwhub.WAF_RULES, args, toComplete)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			if all {
+				cwhub.UpgradeConfig(csConfig, cwhub.WAF_RULES, "", forceAction)
+			} else {
+				if len(args) == 0 {
+					log.Fatalf("no target waf rule to upgrade")
+				}
+				for _, name := range args {
+					cwhub.UpgradeConfig(csConfig, cwhub.WAF_RULES, name, forceAction)
+				}
+			}
+		},
+	}
+	cmdWafRulesUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the waf rules")
+	cmdWafRulesUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
+
+	return cmdWafRulesUpgrade
+}
+
+func NewWafRulesInspectCmd() *cobra.Command {
+	var cmdWafRulesInspect = &cobra.Command{
+		Use:               "inspect [name]",
+		Short:             "Inspect given waf rule",
+		Long:              `Inspect given waf rule`,
+		Example:           `cscli waf-rules inspect crowdsec/xxx`,
+		DisableAutoGenTag: true,
+		Args:              cobra.MinimumNArgs(1),
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			return compInstalledItems(cwhub.WAF_RULES, args, toComplete)
+		},
+		Run: func(cmd *cobra.Command, args []string) {
+			InspectItem(args[0], cwhub.WAF_RULES)
+		},
+	}
+	cmdWafRulesInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
+
+	return cmdWafRulesInspect
+}
+
+func NewWafRulesListCmd() *cobra.Command {
+	var cmdWafRulesList = &cobra.Command{
+		Use:   "list [name]",
+		Short: "List all waf rules or given one",
+		Long:  `List all waf rules or given one`,
+		Example: `cscli waf-rules list
+cscli waf-rules list crowdsecurity/xxx`,
+		DisableAutoGenTag: true,
+		Run: func(cmd *cobra.Command, args []string) {
+			ListItems(color.Output, []string{cwhub.WAF_RULES}, args, false, true, all)
+		},
+	}
+	cmdWafRulesList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
+
+	return cmdWafRulesList
+}

cmd/crowdsec/Makefile

@@ -4,9 +4,10 @@ ifeq ($(OS), Windows_NT)
 	EXT = .exe
 endif
 
-GO = go
-GOBUILD = $(GO) build
-GOTEST = $(GO) test
+# Go parameters
+GOCMD = go
+GOBUILD = $(GOCMD) build
+GOTEST = $(GOCMD) test
 
 CROWDSEC_BIN = crowdsec$(EXT)
 # names longer than 15 chars break 'pgrep'
@@ -22,7 +23,7 @@ SYSTEMD_PATH_FILE = "/etc/systemd/system/crowdsec.service"
 all: clean test build
 
 build: clean
-	$(GOBUILD) $(LD_OPTS) -o $(CROWDSEC_BIN)
+	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(CROWDSEC_BIN)
 
 test:
 	$(GOTEST) $(LD_OPTS) -v ./...

cmd/crowdsec/api.go

@@ -1,14 +1,13 @@
 package main
 
 import (
-	"errors"
-	"fmt"
 	"runtime"
 	"time"
 
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/crowdsecurity/go-cs-lib/trace"
+	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/apiserver"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
@@ -21,7 +20,7 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) {
 
 	apiServer, err := apiserver.NewServer(cConfig.API.Server)
 	if err != nil {
-		return nil, fmt.Errorf("unable to run local API: %w", err)
+		return nil, errors.Wrap(err, "unable to run local API")
 	}
 
 	if hasPlugins(cConfig.API.Server.Profiles) {
@@ -30,34 +29,29 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) {
 		if cConfig.PluginConfig == nil && runtime.GOOS != "windows" {
 			return nil, errors.New("plugins are enabled, but the plugin_config section is missing in the configuration")
 		}
-
 		if cConfig.ConfigPaths.NotificationDir == "" {
 			return nil, errors.New("plugins are enabled, but config_paths.notification_dir is not defined")
 		}
-
 		if cConfig.ConfigPaths.PluginDir == "" {
 			return nil, errors.New("plugins are enabled, but config_paths.plugin_dir is not defined")
 		}
-
 		err = pluginBroker.Init(cConfig.PluginConfig, cConfig.API.Server.Profiles, cConfig.ConfigPaths)
 		if err != nil {
-			return nil, fmt.Errorf("unable to run plugin broker: %w", err)
+			return nil, errors.Wrap(err, "unable to run local API")
 		}
-
 		log.Info("initiated plugin broker")
 		apiServer.AttachPluginBroker(&pluginBroker)
 	}
 
 	err = apiServer.InitController()
 	if err != nil {
-		return nil, fmt.Errorf("unable to run local API: %w", err)
+		return nil, errors.Wrap(err, "unable to run local API")
 	}
 
 	return apiServer, nil
 }
 
-func serveAPIServer(apiServer *apiserver.APIServer) {
-	apiReady := make(chan bool, 1)
+func serveAPIServer(apiServer *apiserver.APIServer, apiReady chan bool) {
 	apiTomb.Go(func() error {
 		defer trace.CatchPanic("crowdsec/serveAPIServer")
 		go func() {
@@ -81,7 +75,6 @@ func serveAPIServer(apiServer *apiserver.APIServer) {
 		}
 		return nil
 	})
-	<-apiReady
 }
 
 func hasPlugins(profiles []*csconfig.ProfileCfg) bool {

cmd/crowdsec/crowdsec.go

@@ -1,183 +1,146 @@
 package main
 
 import (
-	"context"
 	"fmt"
 	"os"
-	"path/filepath"
 	"sync"
 	"time"
 
-	log "github.com/sirupsen/logrus"
-	"gopkg.in/yaml.v3"
+	"path/filepath"
 
-	"github.com/crowdsecurity/go-cs-lib/trace"
+	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
 
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
-	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
-	"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
-	"github.com/crowdsecurity/crowdsec/pkg/appsec"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
 	"github.com/crowdsecurity/crowdsec/pkg/parser"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v2"
 )
 
-// initCrowdsec prepares the log processor service
-func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, []acquisition.DataSource, error) {
+func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
 	var err error
 
-	if err = alertcontext.LoadConsoleContext(cConfig, hub); err != nil {
-		return nil, nil, fmt.Errorf("while loading context: %w", err)
+	// Populate cwhub package tools
+	if err := cwhub.GetHubIdx(cConfig.Hub); err != nil {
+		return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err)
 	}
 
 	// Start loading configs
-	csParsers := parser.NewParsers(hub)
+	csParsers := parser.NewParsers()
 	if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
-		return nil, nil, fmt.Errorf("while loading parsers: %w", err)
+		return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err)
 	}
 
-	if err := LoadBuckets(cConfig, hub); err != nil {
-		return nil, nil, fmt.Errorf("while loading scenarios: %w", err)
+	if err := LoadBuckets(cConfig); err != nil {
+		return &parser.Parsers{}, fmt.Errorf("Failed to load scenarios: %s", err)
 	}
 
-	if err := appsec.LoadAppsecRules(hub); err != nil {
-		return nil, nil, fmt.Errorf("while loading appsec rules: %w", err)
+	if err := LoadAcquisition(cConfig); err != nil {
+		return &parser.Parsers{}, fmt.Errorf("Error while loading acquisition config : %s", err)
 	}
-
-	datasources, err := LoadAcquisition(cConfig)
-	if err != nil {
-		return nil, nil, fmt.Errorf("while loading acquisition config: %w", err)
-	}
-
-	return csParsers, datasources, nil
+	return csParsers, nil
 }
 
-// runCrowdsec starts the log processor service
-func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.Hub, datasources []acquisition.DataSource) error {
+func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
 	inputEventChan = make(chan types.Event)
 	inputLineChan = make(chan types.Event)
 
-	// start go-routines for parsing, buckets pour and outputs.
+	//start go-routines for parsing, buckets pour and outputs.
 	parserWg := &sync.WaitGroup{}
-
 	parsersTomb.Go(func() error {
 		parserWg.Add(1)
-
 		for i := 0; i < cConfig.Crowdsec.ParserRoutinesCount; i++ {
 			parsersTomb.Go(func() error {
 				defer trace.CatchPanic("crowdsec/runParse")
-
-				if err := runParse(inputLineChan, inputEventChan, *parsers.Ctx, parsers.Nodes); err != nil {
-					// this error will never happen as parser.Parse is not able to return errors
+				if err := runParse(inputLineChan, inputEventChan, *parsers.Ctx, parsers.Nodes); err != nil { //this error will never happen as parser.Parse is not able to return errors
 					log.Fatalf("starting parse error : %s", err)
 					return err
 				}
-
 				return nil
 			})
 		}
 		parserWg.Done()
-
 		return nil
 	})
 	parserWg.Wait()
 
 	bucketWg := &sync.WaitGroup{}
-
 	bucketsTomb.Go(func() error {
 		bucketWg.Add(1)
 		/*restore previous state as well if present*/
 		if cConfig.Crowdsec.BucketStateFile != "" {
 			log.Warningf("Restoring buckets state from %s", cConfig.Crowdsec.BucketStateFile)
-
 			if err := leaky.LoadBucketsState(cConfig.Crowdsec.BucketStateFile, buckets, holders); err != nil {
-				return fmt.Errorf("unable to restore buckets: %w", err)
+				return fmt.Errorf("unable to restore buckets : %s", err)
 			}
 		}
 
 		for i := 0; i < cConfig.Crowdsec.BucketsRoutinesCount; i++ {
 			bucketsTomb.Go(func() error {
 				defer trace.CatchPanic("crowdsec/runPour")
-
 				if err := runPour(inputEventChan, holders, buckets, cConfig); err != nil {
 					log.Fatalf("starting pour error : %s", err)
 					return err
 				}
-
 				return nil
 			})
 		}
 		bucketWg.Done()
-
 		return nil
 	})
 	bucketWg.Wait()
 
-	apiClient, err := AuthenticatedLAPIClient(*cConfig.API.Client.Credentials, hub)
-	if err != nil {
-		return err
-	}
-
-	log.Debugf("Starting HeartBeat service")
-	apiClient.HeartBeat.StartHeartBeat(context.Background(), &outputsTomb)
-
 	outputWg := &sync.WaitGroup{}
-
 	outputsTomb.Go(func() error {
 		outputWg.Add(1)
-
 		for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ {
 			outputsTomb.Go(func() error {
 				defer trace.CatchPanic("crowdsec/runOutput")
-
-				if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, apiClient); err != nil {
+				if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials); err != nil {
 					log.Fatalf("starting outputs error : %s", err)
 					return err
 				}
-
 				return nil
 			})
 		}
 		outputWg.Done()
-
 		return nil
 	})
 	outputWg.Wait()
 
 	if cConfig.Prometheus != nil && cConfig.Prometheus.Enabled {
 		aggregated := false
-		if cConfig.Prometheus.Level == configuration.CFG_METRICS_AGGREGATE {
+		if cConfig.Prometheus.Level == "aggregated" {
 			aggregated = true
 		}
-
 		if err := acquisition.GetMetrics(dataSources, aggregated); err != nil {
-			return fmt.Errorf("while fetching prometheus metrics for datasources: %w", err)
+			return errors.Wrap(err, "while fetching prometheus metrics for datasources.")
 		}
-	}
 
+	}
 	log.Info("Starting processing data")
 
 	if err := acquisition.StartAcquisition(dataSources, inputLineChan, &acquisTomb); err != nil {
-		return fmt.Errorf("starting acquisition error: %w", err)
+		log.Fatalf("starting acquisition error : %s", err)
+		return err
 	}
 
 	return nil
 }
 
-// serveCrowdsec wraps the log processor service
-func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub.Hub, datasources []acquisition.DataSource, agentReady chan bool) {
+func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady chan bool) {
 	crowdsecTomb.Go(func() error {
 		defer trace.CatchPanic("crowdsec/serveCrowdsec")
-
 		go func() {
 			defer trace.CatchPanic("crowdsec/runCrowdsec")
 			// this logs every time, even at config reload
 			log.Debugf("running agent after %s ms", time.Since(crowdsecT0))
 			agentReady <- true
-
-			if err := runCrowdsec(cConfig, parsers, hub, datasources); err != nil {
+			if err := runCrowdsec(cConfig, parsers); err != nil {
 				log.Fatalf("unable to start crowdsec routines: %s", err)
 			}
 		}()
@@ -188,88 +151,74 @@ func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub
 		*/
 		waitOnTomb()
 		log.Debugf("Shutting down crowdsec routines")
-
 		if err := ShutdownCrowdsecRoutines(); err != nil {
 			log.Fatalf("unable to shutdown crowdsec routines: %s", err)
 		}
-
 		log.Debugf("everything is dead, return crowdsecTomb")
-
 		if dumpStates {
 			dumpParserState()
 			dumpOverflowState()
 			dumpBucketsPour()
 			os.Exit(0)
 		}
-
 		return nil
 	})
 }
 
 func dumpBucketsPour() {
-	fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "bucketpour-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o666)
+	fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "bucketpour-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
 	if err != nil {
 		log.Fatalf("open: %s", err)
 	}
-
 	out, err := yaml.Marshal(leaky.BucketPourCache)
 	if err != nil {
 		log.Fatalf("marshal: %s", err)
 	}
-
 	b, err := fd.Write(out)
 	if err != nil {
 		log.Fatalf("write: %s", err)
 	}
-
 	log.Tracef("wrote %d bytes", b)
-
 	if err := fd.Close(); err != nil {
 		log.Fatalf(" close: %s", err)
 	}
 }
 
 func dumpParserState() {
-	fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "parser-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o666)
+
+	fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "parser-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
 	if err != nil {
 		log.Fatalf("open: %s", err)
 	}
-
 	out, err := yaml.Marshal(parser.StageParseCache)
 	if err != nil {
 		log.Fatalf("marshal: %s", err)
 	}
-
 	b, err := fd.Write(out)
 	if err != nil {
 		log.Fatalf("write: %s", err)
 	}
-
 	log.Tracef("wrote %d bytes", b)
-
 	if err := fd.Close(); err != nil {
 		log.Fatalf(" close: %s", err)
 	}
 }
 
 func dumpOverflowState() {
-	fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "bucket-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o666)
+
+	fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "bucket-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
 	if err != nil {
 		log.Fatalf("open: %s", err)
 	}
-
 	out, err := yaml.Marshal(bucketOverflows)
 	if err != nil {
 		log.Fatalf("marshal: %s", err)
 	}
-
 	b, err := fd.Write(out)
 	if err != nil {
 		log.Fatalf("write: %s", err)
 	}
-
 	log.Tracef("wrote %d bytes", b)
-
 	if err := fd.Close(); err != nil {
 		log.Fatalf(" close: %s", err)
 	}
@@ -281,7 +230,7 @@ func waitOnTomb() {
 		case <-acquisTomb.Dead():
 			/*if it's acquisition dying it means that we were in "cat" mode.
 			while shutting down, we need to give time for all buckets to process in flight data*/
-			log.Info("Acquisition is finished, shutting down")
+			log.Warning("Acquisition is finished, shutting down")
 			/*
 				While it might make sense to want to shut-down parser/buckets/etc. as soon as acquisition is finished,
 				we might have some pending buckets: buckets that overflowed, but whose LeakRoutine are still alive because they

cmd/crowdsec/fatalhook.go

@@ -1,28 +0,0 @@
-package main
-
-import (
-	"io"
-
-	log "github.com/sirupsen/logrus"
-)
-
-// FatalHook is used to log fatal messages to stderr when the rest goes to a file
-type FatalHook struct {
-	Writer    io.Writer
-	LogLevels []log.Level
-}
-
-func (hook *FatalHook) Fire(entry *log.Entry) error {
-	line, err := entry.String()
-	if err != nil {
-		return err
-	}
-
-	_, err = hook.Writer.Write([]byte(line))
-
-	return err
-}
-
-func (hook *FatalHook) Levels() []log.Level {
-	return hook.LogLevels
-}

cmd/crowdsec/lapiclient.go

@@ -1,92 +0,0 @@
-package main
-
-import (
-	"context"
-	"fmt"
-	"net/url"
-	"time"
-
-	"github.com/go-openapi/strfmt"
-
-	"github.com/crowdsecurity/go-cs-lib/version"
-
-	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
-	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
-	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
-	"github.com/crowdsecurity/crowdsec/pkg/models"
-)
-
-func AuthenticatedLAPIClient(credentials csconfig.ApiCredentialsCfg, hub *cwhub.Hub) (*apiclient.ApiClient, error) {
-	scenarios, err := hub.GetInstalledNamesByType(cwhub.SCENARIOS)
-	if err != nil {
-		return nil, fmt.Errorf("loading list of installed hub scenarios: %w", err)
-	}
-
-	appsecRules, err := hub.GetInstalledNamesByType(cwhub.APPSEC_RULES)
-	if err != nil {
-		return nil, fmt.Errorf("loading list of installed hub appsec rules: %w", err)
-	}
-
-	installedScenariosAndAppsecRules := make([]string, 0, len(scenarios)+len(appsecRules))
-	installedScenariosAndAppsecRules = append(installedScenariosAndAppsecRules, scenarios...)
-	installedScenariosAndAppsecRules = append(installedScenariosAndAppsecRules, appsecRules...)
-
-	apiURL, err := url.Parse(credentials.URL)
-	if err != nil {
-		return nil, fmt.Errorf("parsing api url ('%s'): %w", credentials.URL, err)
-	}
-
-	papiURL, err := url.Parse(credentials.PapiURL)
-	if err != nil {
-		return nil, fmt.Errorf("parsing polling api url ('%s'): %w", credentials.PapiURL, err)
-	}
-
-	password := strfmt.Password(credentials.Password)
-
-	client, err := apiclient.NewClient(&apiclient.Config{
-		MachineID:     credentials.Login,
-		Password:      password,
-		Scenarios:     installedScenariosAndAppsecRules,
-		UserAgent:     fmt.Sprintf("crowdsec/%s", version.String()),
-		URL:           apiURL,
-		PapiURL:       papiURL,
-		VersionPrefix: "v1",
-		UpdateScenario: func() ([]string, error) {
-			scenarios, err := hub.GetInstalledNamesByType(cwhub.SCENARIOS)
-			if err != nil {
-				return nil, err
-			}
-			appsecRules, err := hub.GetInstalledNamesByType(cwhub.APPSEC_RULES)
-			if err != nil {
-				return nil, err
-			}
-			ret := make([]string, 0, len(scenarios)+len(appsecRules))
-			ret = append(ret, scenarios...)
-			ret = append(ret, appsecRules...)
-
-			return ret, nil
-		},
-	})
-	if err != nil {
-		return nil, fmt.Errorf("new client api: %w", err)
-	}
-
-	authResp, _, err := client.Auth.AuthenticateWatcher(context.Background(), models.WatcherAuthRequest{
-		MachineID: &credentials.Login,
-		Password:  &password,
-		Scenarios: installedScenariosAndAppsecRules,
-	})
-	if err != nil {
-		return nil, fmt.Errorf("authenticate watcher (%s): %w", credentials.Login, err)
-	}
-
-	var expiration time.Time
-	if err := expiration.UnmarshalText([]byte(authResp.Expire)); err != nil {
-		return nil, fmt.Errorf("unable to parse jwt expiration: %w", err)
-	}
-
-	client.GetClient().Transport.(*apiclient.JWTTransport).Token = authResp.Token
-	client.GetClient().Transport.(*apiclient.JWTTransport).Expiration = expiration
-
-	return client, nil
-}

cmd/crowdsec/main.go

@@ -1,22 +1,18 @@
 package main
 
 import (
-	"errors"
 	"flag"
 	"fmt"
 	_ "net/http/pprof"
 	"os"
-	"path/filepath"
 	"runtime"
-	"runtime/pprof"
 	"strings"
 	"time"
 
+	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/tomb.v2"
 
-	"github.com/crowdsecurity/go-cs-lib/trace"
-
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/csplugin"
@@ -55,15 +51,12 @@ var (
 )
 
 type Flags struct {
-	ConfigFile string
-
-	LogLevelTrace bool
-	LogLevelDebug bool
-	LogLevelInfo  bool
-	LogLevelWarn  bool
-	LogLevelError bool
-	LogLevelFatal bool
-
+	ConfigFile     string
+	TraceLevel     bool
+	DebugLevel     bool
+	InfoLevel      bool
+	WarnLevel      bool
+	ErrorLevel     bool
 	PrintVersion   bool
 	SingleFileType string
 	Labels         map[string]string
@@ -74,35 +67,27 @@ type Flags struct {
 	WinSvc         string
 	DisableCAPI    bool
 	Transform      string
-	OrderEvent     bool
-	CPUProfile     string
-}
-
-func (f *Flags) haveTimeMachine() bool {
-	return f.OneShotDSN != ""
 }
 
 type labelsMap map[string]string
 
-func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error {
+func LoadBuckets(cConfig *csconfig.Config) error {
 	var (
 		err   error
 		files []string
 	)
-
-	for _, hubScenarioItem := range hub.GetItemMap(cwhub.SCENARIOS) {
-		if hubScenarioItem.State.Installed {
-			files = append(files, hubScenarioItem.State.LocalPath)
+	for _, hubScenarioItem := range cwhub.GetItemMap(cwhub.SCENARIOS) {
+		if hubScenarioItem.Installed {
+			files = append(files, hubScenarioItem.LocalPath)
 		}
 	}
-
 	buckets = leakybucket.NewBuckets()
 
 	log.Infof("Loading %d scenario files", len(files))
+	holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, files, &bucketsTomb, buckets)
 
-	holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, hub, files, &bucketsTomb, buckets, flags.OrderEvent)
 	if err != nil {
-		return fmt.Errorf("scenario loading failed: %w", err)
+		return fmt.Errorf("scenario loading failed: %v", err)
 	}
 
 	if cConfig.Prometheus != nil && cConfig.Prometheus.Enabled {
@@ -110,11 +95,10 @@ func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error {
 			holders[holderIndex].Profiling = true
 		}
 	}
-
 	return nil
 }
 
-func LoadAcquisition(cConfig *csconfig.Config) ([]acquisition.DataSource, error) {
+func LoadAcquisition(cConfig *csconfig.Config) error {
 	var err error
 
 	if flags.SingleFileType != "" && flags.OneShotDSN != "" {
@@ -123,20 +107,16 @@ func LoadAcquisition(cConfig *csconfig.Config) ([]acquisition.DataSource, error)
 
 		dataSources, err = acquisition.LoadAcquisitionFromDSN(flags.OneShotDSN, flags.Labels, flags.Transform)
 		if err != nil {
-			return nil, fmt.Errorf("failed to configure datasource for %s: %w", flags.OneShotDSN, err)
+			return errors.Wrapf(err, "failed to configure datasource for %s", flags.OneShotDSN)
 		}
 	} else {
-		dataSources, err = acquisition.LoadAcquisitionFromFile(cConfig.Crowdsec, cConfig.Prometheus)
+		dataSources, err = acquisition.LoadAcquisitionFromFile(cConfig.Crowdsec)
 		if err != nil {
-			return nil, err
+			return err
 		}
 	}
 
-	if len(dataSources) == 0 {
-		return nil, errors.New("no datasource enabled")
-	}
-
-	return dataSources, nil
+	return nil
 }
 
 var (
@@ -150,28 +130,21 @@ func (l *labelsMap) String() string {
 }
 
 func (l labelsMap) Set(label string) error {
-	for _, pair := range strings.Split(label, ",") {
-		split := strings.Split(pair, ":")
-		if len(split) != 2 {
-			return fmt.Errorf("invalid format for label '%s', must be key:value", pair)
-		}
-
-		l[split[0]] = split[1]
+	split := strings.Split(label, ":")
+	if len(split) != 2 {
+		return errors.Wrapf(errors.New("Bad Format"), "for Label '%s'", label)
 	}
-
+	l[split[0]] = split[1]
 	return nil
 }
 
 func (f *Flags) Parse() {
 	flag.StringVar(&f.ConfigFile, "c", csconfig.DefaultConfigPath("config.yaml"), "configuration file")
-
-	flag.BoolVar(&f.LogLevelTrace, "trace", false, "set log level to 'trace' (VERY verbose)")
-	flag.BoolVar(&f.LogLevelDebug, "debug", false, "set log level to 'debug'")
-	flag.BoolVar(&f.LogLevelInfo, "info", false, "set log level to 'info'")
-	flag.BoolVar(&f.LogLevelWarn, "warning", false, "set log level to 'warning'")
-	flag.BoolVar(&f.LogLevelError, "error", false, "set log level to 'error'")
-	flag.BoolVar(&f.LogLevelFatal, "fatal", false, "set log level to 'fatal'")
-
+	flag.BoolVar(&f.TraceLevel, "trace", false, "VERY verbose")
+	flag.BoolVar(&f.DebugLevel, "debug", false, "print debug-level on stderr")
+	flag.BoolVar(&f.InfoLevel, "info", false, "print info-level on stderr")
+	flag.BoolVar(&f.WarnLevel, "warning", false, "print warning-level on stderr")
+	flag.BoolVar(&f.ErrorLevel, "error", false, "print error-level on stderr")
 	flag.BoolVar(&f.PrintVersion, "version", false, "display version")
 	flag.StringVar(&f.OneShotDSN, "dsn", "", "Process a single data source in time-machine")
 	flag.StringVar(&f.Transform, "transform", "", "expr to apply on the event after acquisition")
@@ -181,14 +154,10 @@ func (f *Flags) Parse() {
 	flag.BoolVar(&f.DisableAgent, "no-cs", false, "disable crowdsec agent")
 	flag.BoolVar(&f.DisableAPI, "no-api", false, "disable local API")
 	flag.BoolVar(&f.DisableCAPI, "no-capi", false, "disable communication with Central API")
-	flag.BoolVar(&f.OrderEvent, "order-event", false, "enforce event ordering with significant performance cost")
-
 	if runtime.GOOS == "windows" {
 		flag.StringVar(&f.WinSvc, "winsvc", "", "Windows service Action: Install, Remove etc..")
 	}
-
 	flag.StringVar(&dumpFolder, "dump-data", "", "dump parsers/buckets raw outputs")
-	flag.StringVar(&f.CPUProfile, "cpu-profile", "", "write cpu profile to file")
 	flag.Parse()
 }
 
@@ -203,18 +172,16 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
 
 	// override from flags
 	switch {
-	case f.LogLevelTrace:
+	case f.TraceLevel:
 		ret = log.TraceLevel
-	case f.LogLevelDebug:
+	case f.DebugLevel:
 		ret = log.DebugLevel
-	case f.LogLevelInfo:
+	case f.InfoLevel:
 		ret = log.InfoLevel
-	case f.LogLevelWarn:
+	case f.WarnLevel:
 		ret = log.WarnLevel
-	case f.LogLevelError:
+	case f.ErrorLevel:
 		ret = log.ErrorLevel
-	case f.LogLevelFatal:
-		ret = log.FatalLevel
 	default:
 	}
 
@@ -222,7 +189,6 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
 		// avoid returning a new ptr to the same value
 		return curLevelPtr
 	}
-
 	return &ret
 }
 
@@ -230,11 +196,11 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
 func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*csconfig.Config, error) {
 	cConfig, _, err := csconfig.NewConfig(configFile, disableAgent, disableAPI, quiet)
 	if err != nil {
-		return nil, fmt.Errorf("while loading configuration file: %w", err)
+		return nil, err
 	}
 
-	if err := trace.Init(filepath.Join(cConfig.ConfigPaths.DataDir, "trace")); err != nil {
-		return nil, fmt.Errorf("while setting up trace directory: %w", err)
+	if (cConfig.Common == nil || *cConfig.Common == csconfig.CommonCfg{}) {
+		return nil, fmt.Errorf("unable to load configuration: common section is empty")
 	}
 
 	cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags)
@@ -246,6 +212,11 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 		dumpStates = true
 	}
 
+	// Configuration paths are dependency to load crowdsec configuration
+	if err := cConfig.LoadConfigurationPaths(); err != nil {
+		return nil, err
+	}
+
 	if flags.SingleFileType != "" && flags.OneShotDSN != "" {
 		// if we're in time-machine mode, we don't want to log to file
 		cConfig.Common.LogMedia = "stdout"
@@ -260,25 +231,18 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 		return nil, err
 	}
 
-	if cConfig.Common.LogMedia != "stdout" {
-		log.AddHook(&FatalHook{
-			Writer:    os.Stderr,
-			LogLevels: []log.Level{log.FatalLevel, log.PanicLevel},
-		})
-	}
-
 	if err := csconfig.LoadFeatureFlagsFile(configFile, log.StandardLogger()); err != nil {
 		return nil, err
 	}
 
-	if !cConfig.DisableAgent {
+	if !flags.DisableAgent {
 		if err := cConfig.LoadCrowdsec(); err != nil {
 			return nil, err
 		}
 	}
 
-	if !cConfig.DisableAPI {
-		if err := cConfig.LoadAPIServer(false); err != nil {
+	if !flags.DisableAPI {
+		if err := cConfig.LoadAPIServer(); err != nil {
 			return nil, err
 		}
 	}
@@ -288,7 +252,11 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 	}
 
 	if cConfig.DisableAPI && cConfig.DisableAgent {
-		return nil, errors.New("you must run at least the API Server or crowdsec")
+		return nil, errors.New("You must run at least the API Server or crowdsec")
+	}
+
+	if flags.TestMode && !cConfig.DisableAgent {
+		cConfig.Crowdsec.LintOnly = true
 	}
 
 	if flags.OneShotDSN != "" && flags.SingleFileType == "" {
@@ -308,10 +276,9 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 			cConfig.API.Server.OnlineClient = nil
 		}
 		/*if the api is disabled as well, just read file and exit, don't daemonize*/
-		if cConfig.DisableAPI {
+		if flags.DisableAPI {
 			cConfig.Common.Daemonize = false
 		}
-
 		log.Infof("single file mode : log_media=%s daemonize=%t", cConfig.Common.LogMedia, cConfig.Common.Daemonize)
 	}
 
@@ -321,7 +288,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 
 	if cConfig.Common.Daemonize && runtime.GOOS == "windows" {
 		log.Debug("Daemonization is not supported on Windows, disabling")
-
 		cConfig.Common.Daemonize = false
 	}
 
@@ -339,16 +305,12 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
 var crowdsecT0 time.Time
 
 func main() {
-	// The initial log level is INFO, even if the user provided an -error or -warning flag
-	// because we need feature flags before parsing cli flags
-	log.SetFormatter(&log.TextFormatter{TimestampFormat: time.RFC3339, FullTimestamp: true})
-
 	if err := fflag.RegisterAllFeatures(); err != nil {
 		log.Fatalf("failed to register features: %s", err)
 	}
 
 	// some features can require configuration or command-line options,
-	// so we need to parse them asap. we'll load from feature.yaml later.
+	// so wwe need to parse them asap. we'll load from feature.yaml later.
 	if err := csconfig.LoadFeatureFlagsEnv(log.StandardLogger()); err != nil {
 		log.Fatalf("failed to set feature flags from environment: %s", err)
 	}
@@ -373,28 +335,9 @@ func main() {
 		os.Exit(0)
 	}
 
-	if flags.CPUProfile != "" {
-		f, err := os.Create(flags.CPUProfile)
-		if err != nil {
-			log.Fatalf("could not create CPU profile: %s", err)
-		}
-
-		log.Infof("CPU profile will be written to %s", flags.CPUProfile)
-
-		if err := pprof.StartCPUProfile(f); err != nil {
-			f.Close()
-			log.Fatalf("could not start CPU profile: %s", err)
-		}
-
-		defer f.Close()
-		defer pprof.StopCPUProfile()
-	}
-
 	err := StartRunSvc()
 	if err != nil {
-		pprof.StopCPUProfile()
-		log.Fatal(err) //nolint:gocritic // Disable warning for the defer pprof.StopCPUProfile() call
+		log.Fatal(err)
 	}
-
 	os.Exit(0)
 }

cmd/crowdsec/metrics.go

@@ -3,15 +3,16 @@ package main
 import (
 	"fmt"
 	"net/http"
+	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/crowdsecurity/go-cs-lib/trace"
-	"github.com/crowdsecurity/go-cs-lib/version"
+	"github.com/crowdsecurity/go-cs-lib/pkg/trace"
+	"github.com/crowdsecurity/go-cs-lib/pkg/version"
 
-	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
+	waf "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/waf"
 	v1 "github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers/v1"
 	"github.com/crowdsecurity/crowdsec/pkg/cache"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
@@ -21,8 +22,7 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/parser"
 )
 
-// Prometheus
-
+/*prometheus*/
 var globalParserHits = prometheus.NewCounterVec(
 	prometheus.CounterOpts{
 		Name: "cs_parser_hits_total",
@@ -30,7 +30,6 @@ var globalParserHits = prometheus.NewCounterVec(
 	},
 	[]string{"source", "type"},
 )
-
 var globalParserHitsOk = prometheus.NewCounterVec(
 	prometheus.CounterOpts{
 		Name: "cs_parser_hits_ok_total",
@@ -38,7 +37,6 @@ var globalParserHitsOk = prometheus.NewCounterVec(
 	},
 	[]string{"source", "type"},
 )
-
 var globalParserHitsKo = prometheus.NewCounterVec(
 	prometheus.CounterOpts{
 		Name: "cs_parser_hits_ko_total",
@@ -105,29 +103,25 @@ var globalPourHistogram = prometheus.NewHistogramVec(
 
 func computeDynamicMetrics(next http.Handler, dbClient *database.Client) http.HandlerFunc {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		// catch panics here because they are not handled by servePrometheus
-		defer trace.CatchPanic("crowdsec/computeDynamicMetrics")
-		// update cache metrics (stash)
+		//update cache metrics (stash)
 		cache.UpdateCacheMetrics()
-		// update cache metrics (regexp)
+		//update cache metrics (regexp)
 		exprhelpers.UpdateRegexpCacheMetrics()
 
-		// decision metrics are only relevant for LAPI
+		//decision metrics are only relevant for LAPI
 		if dbClient == nil {
 			next.ServeHTTP(w, r)
 			return
 		}
 
-		decisions, err := dbClient.QueryDecisionCountByScenario()
+		decisionsFilters := make(map[string][]string, 0)
+		decisions, err := dbClient.QueryDecisionCountByScenario(decisionsFilters)
 		if err != nil {
 			log.Errorf("Error querying decisions for metrics: %v", err)
 			next.ServeHTTP(w, r)
-
 			return
 		}
-
 		globalActiveDecisions.Reset()
-
 		for _, d := range decisions {
 			globalActiveDecisions.With(prometheus.Labels{"reason": d.Scenario, "origin": d.Origin, "action": d.Type}).Set(float64(d.Count))
 		}
@@ -139,10 +133,10 @@ func computeDynamicMetrics(next http.Handler, dbClient *database.Client) http.Ha
 		}
 
 		alerts, err := dbClient.AlertsCountPerScenario(alertsFilter)
+
 		if err != nil {
 			log.Errorf("Error querying alerts for metrics: %v", err)
 			next.ServeHTTP(w, r)
-
 			return
 		}
 
@@ -158,18 +152,26 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
 	if !config.Enabled {
 		return
 	}
+	if config.ListenAddr == "" {
+		log.Warning("prometheus is enabled, but the listen address is empty, using '127.0.0.1'")
+		config.ListenAddr = "127.0.0.1"
+	}
+	if config.ListenPort == 0 {
+		log.Warning("prometheus is enabled, but the listen port is empty, using '6060'")
+		config.ListenPort = 6060
+	}
 
 	// Registering prometheus
 	// If in aggregated mode, do not register events associated with a source, to keep the cardinality low
-	if config.Level == configuration.CFG_METRICS_AGGREGATE {
+	if config.Level == "aggregated" {
 		log.Infof("Loading aggregated prometheus collectors")
 		prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo,
 			globalCsInfo, globalParsingHistogram, globalPourHistogram,
 			leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow,
 			v1.LapiRouteHits,
 			leaky.BucketsCurrentCount,
-			cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, parser.NodesWlHitsOk, parser.NodesWlHits,
-		)
+			cache.CacheMetrics, exprhelpers.RegexpCacheMetrics,
+			waf.WafParsingHistogram, waf.WafReqCounter, waf.WafRuleHits)
 	} else {
 		log.Infof("Loading prometheus collectors")
 		prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo,
@@ -177,15 +179,14 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
 			globalCsInfo, globalParsingHistogram, globalPourHistogram,
 			v1.LapiRouteHits, v1.LapiMachineHits, v1.LapiBouncerHits, v1.LapiNilDecisions, v1.LapiNonNilDecisions, v1.LapiResponseTime,
 			leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, leaky.BucketsCurrentCount,
-			globalActiveDecisions, globalAlerts, parser.NodesWlHitsOk, parser.NodesWlHits,
+			globalActiveDecisions, globalAlerts,
 			cache.CacheMetrics, exprhelpers.RegexpCacheMetrics,
-		)
+			waf.WafParsingHistogram, waf.WafReqCounter, waf.WafRuleHits)
+
 	}
 }
 
-func servePrometheus(config *csconfig.PrometheusCfg, dbClient *database.Client, agentReady chan bool) {
-	<-agentReady
-
+func servePrometheus(config *csconfig.PrometheusCfg, dbClient *database.Client, apiReady chan bool, agentReady chan bool) {
 	if !config.Enabled {
 		return
 	}
@@ -193,11 +194,10 @@ func servePrometheus(config *csconfig.PrometheusCfg, dbClient *database.Client,
 	defer trace.CatchPanic("crowdsec/servePrometheus")
 
 	http.Handle("/metrics", computeDynamicMetrics(promhttp.Handler(), dbClient))
-
+	<-apiReady
+	<-agentReady
+	log.Debugf("serving metrics after %s ms", time.Since(crowdsecT0))
 	if err := http.ListenAndServe(fmt.Sprintf("%s:%d", config.ListenAddr, config.ListenPort), nil); err != nil {
-		// in time machine, we most likely have the LAPI using the port
-		if !flags.haveTimeMachine() {
-			log.Warningf("prometheus: %s", err)
-		}
+		log.Warningf("prometheus: %s", err)
 	}
 }

cmd/crowdsec/output.go

@@ -3,19 +3,26 @@ package main
 import (
 	"context"
 	"fmt"
+	"net/url"
 	"sync"
 	"time"
 
-	log "github.com/sirupsen/logrus"
+	"github.com/crowdsecurity/go-cs-lib/pkg/version"
 
 	"github.com/crowdsecurity/crowdsec/pkg/apiclient"
+	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
+	"github.com/crowdsecurity/crowdsec/pkg/cwhub"
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
 	"github.com/crowdsecurity/crowdsec/pkg/models"
 	"github.com/crowdsecurity/crowdsec/pkg/parser"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/go-openapi/strfmt"
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
 )
 
 func dedupAlerts(alerts []types.RuntimeAlert) ([]*models.Alert, error) {
+
 	var dedupCache []*models.Alert
 
 	for idx, alert := range alerts {
@@ -25,21 +32,16 @@ func dedupAlerts(alerts []types.RuntimeAlert) ([]*models.Alert, error) {
 			dedupCache = append(dedupCache, alert.Alert)
 			continue
 		}
-
 		for k, src := range alert.Sources {
-			refsrc := *alert.Alert // copy
-
+			refsrc := *alert.Alert //copy
 			log.Tracef("source[%s]", k)
-
 			refsrc.Source = &src
 			dedupCache = append(dedupCache, &refsrc)
 		}
 	}
-
 	if len(dedupCache) != len(alerts) {
 		log.Tracef("went from %d to %d alerts", len(alerts), len(dedupCache))
 	}
-
 	return dedupCache, nil
 }
 
@@ -48,27 +50,72 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error
 	alertsToPush, err := dedupAlerts(alerts)
 
 	if err != nil {
-		return fmt.Errorf("failed to transform alerts for api: %w", err)
+		return errors.Wrap(err, "failed to transform alerts for api")
 	}
-
 	_, _, err = client.Alerts.Add(ctx, alertsToPush)
 	if err != nil {
-		return fmt.Errorf("failed sending alert to LAPI: %w", err)
+		return errors.Wrap(err, "failed sending alert to LAPI")
 	}
-
 	return nil
 }
 
 var bucketOverflows []types.Event
 
-func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets, postOverflowCTX parser.UnixParserCtx,
-	postOverflowNodes []parser.Node, client *apiclient.ApiClient) error {
-	var (
-		cache      []types.RuntimeAlert
-		cacheMutex sync.Mutex
-	)
+func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets,
+	postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node, apiConfig csconfig.ApiCredentialsCfg) error {
 
+	var err error
 	ticker := time.NewTicker(1 * time.Second)
+
+	var cache []types.RuntimeAlert
+	var cacheMutex sync.Mutex
+
+	scenarios, err := cwhub.GetInstalledScenariosAsString()
+	if err != nil {
+		return errors.Wrapf(err, "loading list of installed hub scenarios: %s", err)
+	}
+
+	apiURL, err := url.Parse(apiConfig.URL)
+	if err != nil {
+		return errors.Wrapf(err, "parsing api url ('%s'): %s", apiConfig.URL, err)
+	}
+	papiURL, err := url.Parse(apiConfig.PapiURL)
+	if err != nil {
+		return errors.Wrapf(err, "parsing polling api url ('%s'): %s", apiConfig.PapiURL, err)
+	}
+	password := strfmt.Password(apiConfig.Password)
+
+	Client, err := apiclient.NewClient(&apiclient.Config{
+		MachineID:      apiConfig.Login,
+		Password:       password,
+		Scenarios:      scenarios,
+		UserAgent:      fmt.Sprintf("crowdsec/%s", version.String()),
+		URL:            apiURL,
+		PapiURL:        papiURL,
+		VersionPrefix:  "v1",
+		UpdateScenario: cwhub.GetInstalledScenariosAsString,
+	})
+	if err != nil {
+		return errors.Wrapf(err, "new client api: %s", err)
+	}
+	authResp, _, err := Client.Auth.AuthenticateWatcher(context.Background(), models.WatcherAuthRequest{
+		MachineID: &apiConfig.Login,
+		Password:  &password,
+		Scenarios: scenarios,
+	})
+	if err != nil {
+		return errors.Wrapf(err, "authenticate watcher (%s)", apiConfig.Login)
+	}
+
+	if err := Client.GetClient().Transport.(*apiclient.JWTTransport).Expiration.UnmarshalText([]byte(authResp.Expire)); err != nil {
+		return errors.Wrap(err, "unable to parse jwt expiration")
+	}
+
+	Client.GetClient().Transport.(*apiclient.JWTTransport).Token = authResp.Token
+
+	//start the heartbeat service
+	log.Debugf("Starting HeartBeat service")
+	Client.HeartBeat.StartHeartBeat(context.Background(), &outputsTomb)
 LOOP:
 	for {
 		select {
@@ -79,9 +126,9 @@ LOOP:
 				newcache := make([]types.RuntimeAlert, 0)
 				cache = newcache
 				cacheMutex.Unlock()
-				if err := PushAlerts(cachecopy, client); err != nil {
+				if err := PushAlerts(cachecopy, Client); err != nil {
 					log.Errorf("while pushing to api : %s", err)
-					// just push back the events to the queue
+					//just push back the events to the queue
 					cacheMutex.Lock()
 					cache = append(cache, cachecopy...)
 					cacheMutex.Unlock()
@@ -92,13 +139,19 @@ LOOP:
 				cacheMutex.Lock()
 				cachecopy := cache
 				cacheMutex.Unlock()
-				if err := PushAlerts(cachecopy, client); err != nil {
+				if err := PushAlerts(cachecopy, Client); err != nil {
 					log.Errorf("while pushing leftovers to api : %s", err)
 				}
 			}
-
 			break LOOP
 		case event := <-overflow:
+			//if the Alert is nil, it's to signal bucket is ready for GC, don't track this
+			if dumpStates && event.Overflow.Alert != nil {
+				if bucketOverflows == nil {
+					bucketOverflows = make([]types.Event, 0)
+				}
+				bucketOverflows = append(bucketOverflows, event)
+			}
 			/*if alert is empty and mapKey is present, the overflow is just to cleanup bucket*/
 			if event.Overflow.Alert == nil && event.Overflow.Mapkey != "" {
 				buckets.Bucket_map.Delete(event.Overflow.Mapkey)
@@ -107,17 +160,9 @@ LOOP:
 			/* process post overflow parser nodes */
 			event, err := parser.Parse(postOverflowCTX, event, postOverflowNodes)
 			if err != nil {
-				return fmt.Errorf("postoverflow failed: %w", err)
+				return fmt.Errorf("postoverflow failed : %s", err)
 			}
 			log.Printf("%s", *event.Overflow.Alert.Message)
-			// if the Alert is nil, it's to signal bucket is ready for GC, don't track this
-			// dump after postoveflow processing to avoid missing whitelist info
-			if dumpStates && event.Overflow.Alert != nil {
-				if bucketOverflows == nil {
-					bucketOverflows = make([]types.Event, 0)
-				}
-				bucketOverflows = append(bucketOverflows, event)
-			}
 			if event.Overflow.Whitelisted {
 				log.Printf("[%s] is whitelisted, skip.", *event.Overflow.Alert.Message)
 				continue
@@ -137,6 +182,6 @@ LOOP:
 	}
 
 	ticker.Stop()
-
 	return nil
+
 }

cmd/crowdsec/parse.go

@@ -11,6 +11,7 @@ import (
 )
 
 func runParse(input chan types.Event, output chan types.Event, parserCTX parser.UnixParserCtx, nodes []parser.Node) error {
+
 LOOP:
 	for {
 		select {
@@ -21,13 +22,6 @@ LOOP:
 			if !event.Process {
 				continue
 			}
-			/*Application security engine is going to generate 2 events:
-			- one that is treated as a log and can go to scenarios
-			- another one that will go directly to LAPI*/
-			if event.Type == types.APPSEC {
-				outputEventChan <- event
-				continue
-			}
 			if event.Line.Module == "" {
 				log.Errorf("empty event.Line.Module field, the acquisition module must set it ! : %+v", event.Line)
 				continue
@@ -55,6 +49,5 @@ LOOP:
 			output <- parsed
 		}
 	}
-
 	return nil
 }

cmd/crowdsec/pour.go

@@ -4,30 +4,29 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/prometheus/client_golang/prometheus"
-	log "github.com/sirupsen/logrus"
-
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
 )
 
 func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *leaky.Buckets, cConfig *csconfig.Config) error {
-	count := 0
-
+	var (
+		count int
+	)
 	for {
-		// bucket is now ready
+		//bucket is now ready
 		select {
 		case <-bucketsTomb.Dying():
 			log.Infof("Bucket routine exiting")
 			return nil
 		case parsed := <-input:
 			startTime := time.Now()
-
 			count++
 			if count%5000 == 0 {
 				log.Infof("%d existing buckets", leaky.LeakyRoutineCount)
-				// when in forensics mode, garbage collect buckets
+				//when in forensics mode, garbage collect buckets
 				if cConfig.Crowdsec.BucketsGCEnabled {
 					if parsed.MarshaledTime != "" {
 						z := &time.Time{}
@@ -35,30 +34,26 @@ func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *lea
 							log.Warningf("Failed to unmarshal time from event '%s' : %s", parsed.MarshaledTime, err)
 						} else {
 							log.Warning("Starting buckets garbage collection ...")
-
 							if err = leaky.GarbageCollectBuckets(*z, buckets); err != nil {
-								return fmt.Errorf("failed to start bucket GC : %w", err)
+								return fmt.Errorf("failed to start bucket GC : %s", err)
 							}
 						}
 					}
 				}
 			}
-			// here we can bucketify with parsed
+			//here we can bucketify with parsed
 			poured, err := leaky.PourItemToHolders(parsed, holders, buckets)
 			if err != nil {
 				log.Errorf("bucketify failed for: %v", parsed)
 				continue
 			}
-
 			elapsed := time.Since(startTime)
 			globalPourHistogram.With(prometheus.Labels{"type": parsed.Line.Module, "source": parsed.Line.Src}).Observe(elapsed.Seconds())
-
 			if poured {
 				globalBucketPourOk.Inc()
 			} else {
 				globalBucketPourKo.Inc()
 			}
-
 			if len(parsed.MarshaledTime) != 0 {
 				if err := lastProcessedItem.UnmarshalText([]byte(parsed.MarshaledTime)); err != nil {
 					log.Warningf("failed to unmarshal time from event : %s", err)