functional tests with bats-core (#1266)

.github/workflows/ci_bats.yml

@@ -0,0 +1,67 @@
+name: BATS functional tests
+
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - master
+
+jobs:
+  build:
+    name: "Build the application"
+    runs-on: ubuntu-latest
+    timeout-minutes: 20
+    steps:
+
+    - name: "Set up Go 1.17"
+      uses: actions/setup-go@v1
+      with:
+        go-version: 1.17
+      id: go
+
+    - name: "Clone CrowdSec"
+      uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
+        submodules: true
+
+    - name: "Install bats dependencies"
+      run: |
+        sudo apt install daemonize netcat-openbsd
+        GO111MODULE=on go get github.com/mikefarah/yq/v4
+
+    - name: "BATS: build crowdsec"
+      run: make bats-clean bats-build
+
+    - name: "BATS: prepare fixture config+data"
+      run: make bats-instance-data
+
+    - name: "BATS: run tests"
+      run: make bats-test
+
+    - name: "BATS: collect hub coverage"
+      run: ./tests/collect-hub-coverage >> $GITHUB_ENV
+
+    - name: "Create Parsers badge"
+      uses: schneegans/dynamic-badges-action@v1.1.0
+      if: ${{ github.ref == 'refs/heads/master' }}
+      with:
+        auth: ${{ secrets.GIST_BADGES_SECRET }}
+        gistID: ${{ secrets.GIST_BADGES_ID }}
+        filename: crowdsec_parsers_badge.json
+        label: Hub Parsers
+        message: ${{ env.PARSERS_COV }}
+        color: ${{ env.SCENARIO_BADGE_COLOR }}
+
+    - name: "Create Scenarios badge"
+      uses: schneegans/dynamic-badges-action@v1.1.0
+      if: ${{ github.ref == 'refs/heads/master' }}
+      with:
+        auth: ${{ secrets.GIST_BADGES_SECRET }}
+        gistID: ${{ secrets.GIST_BADGES_ID }}
+        filename: crowdsec_scenarios_badge.json
+        label: Hub Scenarios
+        message: ${{ env.SCENARIOS_COV }}
+        color: ${{ env.SCENARIO_BADGE_COLOR }}

.github/workflows/ci_functests-install.yml

@@ -1,84 +0,0 @@
-name: Functional tests
-
-on:
-  push:
-    branches:
-      - master
-    paths-ignore:
-      - 'docs/**'
-      - 'mkdocs.yml'
-      - 'README.md'
-  pull_request:
-    branches:
-      - master
-    paths-ignore:
-      - 'docs/**'
-      - 'mkdocs.yml'
-      - 'README.md'
-
-jobs:
-  build:
-    name: Install generated release and perform functional tests
-    runs-on: ubuntu-latest
-    steps:
-    - name: Set up Go 1.17
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.17
-      id: go
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
-    - id: keydb
-      uses: pozetroninc/github-action-get-latest-release@master
-      with:
-        owner: crowdsecurity
-        repo: crowdsec
-        excludes: draft
-    - name: Build release
-      run: BUILD_VERSION=${{ steps.keydb.outputs.release }} make release
-    - name: "Force machineid"
-      run: |
-          sudo chmod +w /etc/machine-id
-          echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
-    - name: Install release
-      run: |
-        cd crowdsec-${{ steps.keydb.outputs.release }}
-        sudo ./wizard.sh --unattended
-    - name: "Test post-install base"
-      run: |
-          cd scripts/func_tests/
-          ./tests_post-install_0base.sh
-    - name: "Test post-install bouncer"
-      run: |
-          cd scripts/func_tests/
-          ./tests_post-install_1bouncers.sh
-    - name: "Test post-install bouncer"
-      run: |
-          cd scripts/func_tests/
-          ./tests_post-install_2collections.sh
-    - name: "Test post-install bouncer"
-      run: |
-          cd scripts/func_tests/
-          ./tests_post-install_3machines.sh   
-    - name: "Test post-install ip management"
-      run: |
-          cd scripts/func_tests/
-          ./tests_post-install_99ip_mgmt.sh
-    - name: "Test cold logs"
-      run: |
-          cd scripts/func_tests/
-          ./tests_post-install_4cold-logs.sh
-    - name: "Test simulation"
-      run: |
-          cd scripts/func_tests/
-          ./tests_post-install_5simulation.sh
-    - name: "Test post-install plugins"
-      run: |
-          cd scripts/func_tests/
-          sudo ./tests_post-install_7_plugin.sh
-    - name: "Uninstall"
-      run: sudo ./wizard.sh --uninstall
-    - name: "Test post remove"
-      run: |
-          cd scripts/func_tests/
-          bash -x ./tests_post-remove_0base.sh      

.github/workflows/ci_hubtest.yml

@@ -1,71 +0,0 @@
-name: Hub Tests
-
-on:
-  push:
-    branches:
-      - master
-  pull_request:
-    branches:
-      - master
-
-jobs:
-  hubtest:
-    name: Hub tests
-    runs-on: ubuntu-latest
-    steps:
-    - name: Set up Go 1.17
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.17
-      id: go
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
-    - id: keydb
-      uses: pozetroninc/github-action-get-latest-release@master
-      with:
-        owner: crowdsecurity
-        repo: crowdsec
-        excludes: draft
-    - name: Build release
-      run: BUILD_VERSION=${{ steps.keydb.outputs.release }} make release
-    - name: "Force machineid"
-      run: |
-          sudo chmod +w /etc/machine-id
-          echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
-    - name: Install release
-      run: |
-        cd crowdsec-${{ steps.keydb.outputs.release }}
-        sudo ./wizard.sh --unattended
-    - name: "Clone CrowdSec Hub"
-      run: |
-          git clone https://github.com/crowdsecurity/hub.git
-    - name: "Run tests"
-      run: |
-          cd hub/
-          cscli hubtest run --all --clean
-          echo "PARSERS_COV=$(cscli hubtest coverage --parsers --percent | cut -d '=' -f2)" >> $GITHUB_ENV
-          echo "SCENARIOS_COV=$(cscli hubtest coverage --scenarios --percent | cut -d '=' -f2)" >> $GITHUB_ENV
-          PARSERS_COV_NUMBER=$(cscli hubtest coverage --parsers --percent | cut -d '=' -f2 | tr -d '%' | tr -d '[[:space:]]')
-          SCENARIOS_COV_NUMBER=$(cscli hubtest coverage --scenarios --percent | cut -d '=' -f2 | tr -d '%' | tr -d '[[:space:]]')
-          echo "PARSER_BADGE_COLOR=$(if [ "$PARSERS_COV_NUMBER" -lt "70" ]; then echo 'red'; else echo 'green'; fi)" >> $GITHUB_ENV
-          echo "SCENARIO_BADGE_COLOR=$(if [ "$SCENARIOS_COV_NUMBER" -lt "70" ]; then echo 'red'; else echo 'green'; fi)" >> $GITHUB_ENV
-    - name: Create Parsers badge
-      uses: schneegans/dynamic-badges-action@v1.1.0
-      if: ${{ github.ref == 'refs/heads/master' }}
-      with:
-        auth: ${{ secrets.GIST_BADGES_SECRET }}
-        gistID: ${{ secrets.GIST_BADGES_ID }}
-        filename: crowdsec_parsers_badge.json
-        label: Hub Parsers
-        message: ${{ env.PARSERS_COV }}
-        color: ${{ env.SCENARIO_BADGE_COLOR }}
-    - name: Create Scenarios badge
-      uses: schneegans/dynamic-badges-action@v1.1.0
-      if: ${{ github.ref == 'refs/heads/master' }}
-      with:
-        auth: ${{ secrets.GIST_BADGES_SECRET }}
-        gistID: ${{ secrets.GIST_BADGES_ID }}
-        filename: crowdsec_scenarios_badge.json
-        label: Hub Scenarios
-        message: ${{ env.SCENARIOS_COV }}
-        color: ${{ env.SCENARIO_BADGE_COLOR }}

.gitignore

@@ -13,6 +13,10 @@
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
 
+# Development artifacts, backups, etc
+*.swp
+*.swo
+
 # Dependency directories (remove the comment below to include it)
 # vendor/
 

.gitmodules

@@ -0,0 +1,12 @@
+[submodule "tests/lib/bats-core"]
+	path = tests/lib/bats-core
+	url = https://github.com/crowdsecurity/bats-core.git
+[submodule "tests/lib/bats-file"]
+	path = tests/lib/bats-file
+	url = https://github.com/crowdsecurity/bats-file.git
+[submodule "tests/lib/bats-assert"]
+	path = tests/lib/bats-assert
+	url = https://github.com/crowdsecurity/bats-assert.git
+[submodule "tests/lib/bats-support"]
+	path = tests/lib/bats-support
+	url = https://github.com/crowdsecurity/bats-support.git

Makefile

@@ -142,7 +142,7 @@ email-plugin_static:goversion
 	@GOARCH=$(GOARCH) GOOS=$(GOOS) $(MAKE) -C $(EMAIL_PLUGIN_FOLDER) static --no-print-directory
 
 .PHONY: testclean
-testclean:
+testclean: bats-clean
 	@$(RM) pkg/apiserver/ent
 	@$(RM) -r pkg/cwhub/hubdir
 
@@ -214,3 +214,6 @@ release: check_release build package
 
 .PHONY: release_static
 release_static: check_release static package_static
+
+include tests/bats.mk
+

scripts/func_tests/README.md

@@ -1,53 +0,0 @@
-## Functional testing
-
-This directory contains scripts for functional testing of crowdsec, to unify testing across packages (ie. tgz, deb, rpm).
-
-Each package system tests the installation/removal, and the scripts here cover basic functional testing.
-
-### cscli
-
-| Feature       | Covered     | Note     |
-| :------------- | :----------: | -----------: |
-| `cscli alerts` |  🟢 | 99ip_mgmt.sh |
-| `cscli bouncers` |  🟢 | 1bouncers.sh |
-| `cscli capi` |  ❌  | 0base.sh : `status` only |
-| `cscli collections` |  🟢 | 2collections.sh |
-| `cscli config` | ❌  | 0base.sh : minimal testing (no crash) |
-| `cscli dashboard` | ❌   | docker inside docker 😞    |
-| `cscli decisions` |  🟢 | 99ip_mgmt.sh |
-| `cscli hub` |  ❌ | TBD |
-| `cscli lapi` |  🟢 | 3machines.sh  |
-| `cscli machines` |  🟢 | 3machines.sh |
-| `cscli metrics` |  ❌ | TBD |
-| `cscli parsers` |  ❌ | TBD |
-| `cscli postoverflows` |  ❌ | TBD |
-| `cscli scenarios` |  ❌ | TBD |
-| `cscli simulation` |  ❌ | TBD |
-| `cscli version` |  🟢 | 0base.sh |
-
-### crowdsec
-
-| Feature       | Covered     | Note     |
-| :------------- | :----------: | -----------: |
-| `systemctl` start/stop/restart | 🟢 | 0base.sh |
-| agent behaviour | 🟢 | 4cold-logs.sh : minimal testing  (simple ssh-bf detection) |
-| forensic mode  | 🟢  | 4cold-logs.sh : minimal testing (simple ssh-bf detection) |
-| starting only LAPI  | ❌  | TBD |
-| starting only agent  | ❌  | TBD |
-| prometheus testing  | ❌  | TBD |
-
-### API
-
-
-| Feature       | Covered     | Note     |
-| :------------- | :----------: | -----------: |
-| alerts GET/POST | 🟢 | 99ip_mgmt.sh |
-| decisions GET/POST | 🟢 | 99ip_mgmt.sh |
-
-
-## Automation
-
-https://github.com/crowdsecurity/crowdsec/ uses dispatch to triggers tests in the other packages build repositories.
-
-
-

scripts/func_tests/config/config.yaml

@@ -1,48 +0,0 @@
-common:
-  daemonize: true
-  pid_dir: /var/run/
-  log_media: file
-  log_level: info
-  log_dir: /var/log/
-  working_dir: .
-config_paths:
-  config_dir: /etc/crowdsec/
-  data_dir: /var/lib/crowdsec/data/
-  simulation_path: /etc/crowdsec/simulation.yaml
-  hub_dir: /etc/crowdsec/hub/
-  index_path: /etc/crowdsec/hub/.index.json
-  notification_dir: /etc/crowdsec/notifications/
-  plugin_dir: /usr/local/lib/crowdsec/plugins
-crowdsec_service:
-  acquisition_path: /etc/crowdsec/acquis.yaml
-  parser_routines: 1
-cscli:
-  output: human
-plugin_config:
-  user: nobody # plugin process would be ran on behalf of this user
-  group: nogroup # plugin process would be ran on behalf of this group
-db_config:
-  log_level: info
-  type: sqlite
-  db_path: /var/lib/crowdsec/data/crowdsec.db
-  flush:
-    max_items: 5000
-    max_age: 7d
-api:
-  client:
-    insecure_skip_verify: false
-    credentials_path: /etc/crowdsec/local_api_credentials.yaml
-  server:
-    log_level: info
-    listen_uri: 127.0.0.1:8080
-    profiles_path: /etc/crowdsec/profiles.yaml
-    online_client: # Central API credentials (to push signals and receive bad IPs)
-      credentials_path: /etc/crowdsec/online_api_credentials.yaml
-#    tls:
-#      cert_file: /etc/crowdsec/ssl/cert.pem
-#      key_file: /etc/crowdsec/ssl/key.pem
-prometheus:
-  enabled: true
-  level: full
-  listen_addr: 127.0.0.1
-  listen_port: 6060

scripts/func_tests/config/config_no_agent.yaml

@@ -1,46 +0,0 @@
-
-common:
-  daemonize: true
-  pid_dir: /var/run/
-  log_media: file
-  log_level: info
-  log_dir: ./
-  working_dir: .
-config_paths:
-  config_dir: /etc/crowdsec/
-  data_dir: /var/lib/crowdsec/data/
-  simulation_path: /etc/crowdsec/simulation.yaml
-  hub_dir: /etc/crowdsec/hub/
-  index_path: /etc/crowdsec/hub/.index.json
-  notification_dir: /etc/crowdsec/notifications/
-  plugin_dir: /usr/local/lib/crowdsec/plugins
-cscli:
-  output: human
-db_config:
-  log_level: info
-  type: sqlite
-  db_path: /var/lib/crowdsec/data/crowdsec.db
-  flush:
-    max_items: 5000
-    max_age: 7d
-plugin_config:
-  user: nobody # plugin process would be ran on behalf of this user
-  group: nogroup # plugin process would be ran on behalf of this group
-api:
-  client:
-    insecure_skip_verify: false
-    credentials_path: /etc/crowdsec/local_api_credentials.yaml
-  server:
-    log_level: info
-    listen_uri: 127.0.0.1:8080
-    profiles_path: /etc/crowdsec/profiles.yaml
-    online_client: # Central API credentials (to push signals and receive bad IPs)
-      credentials_path: /etc/crowdsec/online_api_credentials.yaml
-#    tls:
-#      cert_file: /etc/crowdsec/ssl/cert.pem
-#      key_file: /etc/crowdsec/ssl/key.pem
-prometheus:
-  enabled: true
-  level: full
-  listen_addr: 127.0.0.1
-  listen_port: 6060

scripts/func_tests/config/config_no_capi.yaml

@@ -1,43 +0,0 @@
-common:
-  daemonize: true
-  pid_dir: /var/run/
-  log_media: file
-  log_level: info
-  log_dir: ./
-  working_dir: .
-config_paths:
-  config_dir: /etc/crowdsec/
-  data_dir: /var/lib/crowdsec/data/
-  simulation_path: /etc/crowdsec/simulation.yaml
-  hub_dir: /etc/crowdsec/hub/
-  index_path: /etc/crowdsec/hub/.index.json
-  notification_dir: /etc/crowdsec/notifications/
-  plugin_dir: /usr/local/lib/crowdsec/plugins
-crowdsec_service:
-  acquisition_path: /etc/crowdsec/acquis.yaml
-  parser_routines: 1
-cscli:
-  output: human
-db_config:
-  log_level: info
-  type: sqlite
-  db_path: /var/lib/crowdsec/data/crowdsec.db
-  flush:
-    max_items: 5000
-    max_age: 7d
-plugin_config:
-  user: nobody # plugin process would be ran on behalf of this user
-  group: nogroup # plugin process would be ran on behalf of this group
-api:
-  client:
-    insecure_skip_verify: false
-    credentials_path: /etc/crowdsec/local_api_credentials.yaml
-  server:
-    log_level: info
-    listen_uri: 127.0.0.1:8080
-    profiles_path: /etc/crowdsec/profiles.yaml
-prometheus:
-  enabled: true
-  level: full
-  listen_addr: 127.0.0.1
-  listen_port: 6060

scripts/func_tests/config/config_no_lapi.yaml

@@ -1,39 +0,0 @@
-common:
-  daemonize: true
-  pid_dir: /var/run/
-  log_media: file
-  log_level: info
-  log_dir: ./
-  working_dir: .
-config_paths:
-  config_dir: /etc/crowdsec/
-  data_dir: /var/lib/crowdsec/data/
-  simulation_path: /etc/crowdsec/simulation.yaml
-  hub_dir: /etc/crowdsec/hub/
-  index_path: /etc/crowdsec/hub/.index.json
-  notification_dir: /etc/crowdsec/notifications/
-  plugin_dir: /usr/local/lib/crowdsec/plugins
-crowdsec_service:
-  acquisition_path: /etc/crowdsec/acquis.yaml
-  parser_routines: 1
-cscli:
-  output: human
-plugin_config:
-  user: nobody # plugin process would be ran on behalf of this user
-  group: nogroup # plugin process would be ran on behalf of this group
-db_config:
-  log_level: info
-  type: sqlite
-  db_path: /var/lib/crowdsec/data/crowdsec.db
-  flush:
-    max_items: 5000
-    max_age: 7d
-api:
-  client:
-    insecure_skip_verify: false
-    credentials_path: /etc/crowdsec/local_api_credentials.yaml
-prometheus:
-  enabled: true
-  level: full
-  listen_addr: 127.0.0.1
-  listen_port: 6060

scripts/func_tests/config/http.yaml

@@ -1,24 +0,0 @@
-# Don't change this
-type: http
-
-name: http_default # this must match with the registered plugin in the profile
-log_level: info # Options include: trace, debug, info, warn, error, off
-
-format: |  # This template receives list of models.Alert objects. The request body would contain this. 
-  {{.|toJson}}
-
-url: http://localhost:9999 # plugin will make requests to this url. Eg value https://www.example.com/
-
-method: POST # eg either of "POST", "GET", "PUT" and other http verbs is valid value. 
-
-# headers:
-#   Authorization: token 0x64312313
-
-# skip_tls_verification:  # either true or false. Default is false
-
-group_wait: 5s # duration to wait collecting alerts before sending to this plugin, eg "30s"
-group_threshold: 2 # if alerts exceed this, then the plugin will be sent the message. eg "10"
-
-# max_retry: # number of tries to attempt to send message to plugins in case of error.
-
-# timeout: # duration to wait for response from plugin before considering this attempt a failure. eg "10s"

scripts/func_tests/systemd/crowdsec.service

@@ -1,15 +0,0 @@
-[Unit]
-Description=Crowdsec agent
-After=syslog.target network.target remote-fs.target nss-lookup.target
-
-[Service]
-Type=notify
-Environment=LC_ALL=C LANG=C
-PIDFile=/var/run/crowdsec.pid
-ExecStartPre=crowdsec -c /etc/crowdsec/config.yaml -t
-ExecStart=crowdsec -c /etc/crowdsec/config.yaml
-#ExecStartPost=/bin/sleep 0.1
-ExecReload=/bin/kill -HUP $MAINPID
-
-[Install]
-WantedBy=multi-user.target

scripts/func_tests/systemd/crowdsec_no_agent.service

@@ -1,15 +0,0 @@
-[Unit]
-Description=Crowdsec agent
-After=syslog.target network.target remote-fs.target nss-lookup.target
-
-[Service]
-Type=notify
-Environment=LC_ALL=C LANG=C
-PIDFile=/var/run/crowdsec.pid
-ExecStartPre=crowdsec -c /etc/crowdsec/config.yaml -t
-ExecStart=crowdsec -c /etc/crowdsec/config.yaml -no-cs
-#ExecStartPost=/bin/sleep 0.1
-ExecReload=/bin/kill -HUP $MAINPID
-
-[Install]
-WantedBy=multi-user.target

scripts/func_tests/systemd/crowdsec_no_lapi.service

@@ -1,15 +0,0 @@
-[Unit]
-Description=Crowdsec agent
-After=syslog.target network.target remote-fs.target nss-lookup.target
-
-[Service]
-Type=notify
-Environment=LC_ALL=C LANG=C
-PIDFile=/var/run/crowdsec.pid
-ExecStartPre=crowdsec -c /etc/crowdsec/config.yaml -t
-ExecStart=crowdsec -c /etc/crowdsec/config.yaml -no-api
-#ExecStartPost=/bin/sleep 0.1
-ExecReload=/bin/kill -HUP $MAINPID
-
-[Install]
-WantedBy=multi-user.target

scripts/func_tests/tests_base.sh

@@ -1,51 +0,0 @@
-#! /usr/bin/env bash
-# -*- coding: utf-8 -*-
-
-
-# sourced by other functionnal tests
-
-PACKAGE_PATH="${PACKAGE_PATH:-./crowdsec.deb}"
-
-CSCLI_BIN="cscli"
-CSCLI="sudo ${CSCLI_BIN}"
-JQ="jq -e"
-
-LC_ALL=C
-SYSTEMCTL="sudo systemctl --no-pager"
-
-CROWDSEC="sudo crowdsec"
-CROWDSEC_PROCESS="crowdsec"
-
-# helpers
-function fail {
-    echo "ACTION FAILED, STOP : $@"
-    caller
-    exit 1
-}
-
-function pathadd {
-    if [ -d "$1" ] && [[ ":$PATH:" != *":$1:"* ]]; then
-        PATH="${PATH:+"$PATH:"}$1"
-    fi
-}
-
-function wait_for_service {
-    count=0
-    while ! nc -z localhost 6060; do   
-        sleep 0.5
-        ((count ++))
-        if [[ $count == 21 ]]; then
-            fail "$@"
-        fi
-    done
-}
-
-pathadd /usr/sbin
-
-if [ -f /etc/systemd/system/crowdsec.service ]; then
-    SYSTEMD_SERVICE_FILE=/etc/systemd/system/crowdsec.service
-elif  [ -f /usr/lib/systemd/system/crowdsec.service ]; then
-    SYSTEMD_SERVICE_FILE=/usr/lib/systemd/system/crowdsec.service
-elif  [ -f /lib/systemd/system/crowdsec.service ]; then
-    SYSTEMD_SERVICE_FILE=/lib/systemd/system/crowdsec.service
-fi

scripts/func_tests/tests_post-install_0base.sh

@@ -1,159 +0,0 @@
-#! /usr/bin/env bash
-# -*- coding: utf-8 -*-
-
-source tests_base.sh
-
-echo $PATH
-
-sudo cp /etc/crowdsec/config.yaml ./config.yaml.backup
-
-CROWDSEC_PATH=$(which crowdsec)
-
-##########################
-## TEST AGENT/LAPI/CAPI ##
-echo "CROWDSEC (AGENT+LAPI+CAPI)"
-
-## status / start / stop
-# service should be up
-pidof crowdsec || fail "crowdsec process should be running"
-${SYSTEMCTL} status crowdsec || fail "systemctl status crowdsec failed"
-
-#shut it down
-${SYSTEMCTL} stop crowdsec || fail "failed to stop service"
-${SYSTEMCTL} status crowdsec && fail "crowdsec should be down"
-pidof crowdsec && fail "crowdsec process shouldn't be running"
-
-#start it again
-${SYSTEMCTL} start crowdsec || fail "failed to stop service"
-${SYSTEMCTL} status crowdsec || fail "crowdsec should be up"
-wait_for_service "crowdsec process should be running"
-
-#restart it
-${SYSTEMCTL} restart crowdsec || fail "failed to stop service"
-${SYSTEMCTL} status crowdsec || fail "crowdsec should be up"
-wait_for_service "crowdsec process should be running"
-
-## version
-${CSCLI} version || fail "cannot run cscli version"
-
-## alerts
-# alerts list at startup should just return one entry : community pull
-sleep 40
-${CSCLI} alerts list -ojson  | ${JQ} '. | length >= 1' || fail "expected at least one entry from cscli alerts list"
-## capi
-${CSCLI} capi status || fail "capi status should be ok"
-## config
-${CSCLI} config show || fail "failed to show config"
-${CSCLI} config backup ./test || fail "failed to backup config"
-sudo rm -rf ./test
-## lapi
-${CSCLI} lapi status || fail "lapi status failed"
-## metrics
-${CSCLI} metrics || fail "failed to get metrics"
-
-${SYSTEMCTL} stop crowdsec || fail "crowdsec should be down"
-
-sudo mkdir -p /etc/systemd/system/crowdsec.service.d/
-
-#######################
-## TEST WITHOUT LAPI ##
-
-echo "CROWDSEC (AGENT)"
-
-# test with -no-api flag
-echo -ne "[Service]\nExecStart=\nExecStart=${CROWDSEC_PATH} -c /etc/crowdsec/config.yaml -no-api\n" | sudo tee /etc/systemd/system/crowdsec.service.d/override.conf
-
-${SYSTEMCTL} daemon-reload
-${SYSTEMCTL} start crowdsec
-sleep 1
-pidof crowdsec && fail "crowdsec shouldn't run without LAPI (in flag)"
-${SYSTEMCTL} stop crowdsec
-
-${SYSTEMCTL} daemon-reload
-
-# test with no api server in configuration file
-sudo cp ./config/config_no_lapi.yaml /etc/crowdsec/config.yaml
-${SYSTEMCTL} start crowdsec
-sleep 1
-pidof crowdsec && fail "crowdsec agent should not run without lapi (in configuration file)"
-
-##### cscli test ####
-## capi
-${CSCLI} -c ./config/config_no_lapi.yaml capi status && fail "capi status shouldn't be ok"
-## config
-${CSCLI_BIN} -c ./config/config_no_lapi.yaml config show || fail "failed to show config"
-${CSCLI} -c ./config/config_no_lapi.yaml config backup ./test || fail "failed to backup config"
-sudo rm -rf ./test
-## lapi
-${CSCLI} -c ./config/config_no_lapi.yaml lapi status && fail "lapi status should not be ok" ## if lapi status success, it means that the test fail
-## metrics
-${CSCLI_BIN} -c ./config/config_no_lapi.yaml metrics
-
-${SYSTEMCTL} stop crowdsec
-sudo cp ./config/config.yaml /etc/crowdsec/config.yaml
-
-########################
-## TEST WITHOUT AGENT ##
-
-echo "CROWDSEC (LAPI+CAPI)"
-
-# test with -no-cs flag
-echo -ne "[Service]\nExecStart=\nExecStart=${CROWDSEC_PATH} -c /etc/crowdsec/config.yaml -no-cs" | sudo tee /etc/systemd/system/crowdsec.service.d/override.conf
-
-${SYSTEMCTL} daemon-reload
-sudo rm -f /var/log/crowdsec.log
-${SYSTEMCTL} start crowdsec
-wait_for_service "crowdsec LAPI should run without agent (in flag)"
-${SYSTEMCTL} stop crowdsec
-
-echo -ne "[service]\nExecStart=\nExecStart=${CROWDSEC_PATH} -c /etc/crowdsec/config.yaml" | sudo tee /etc/systemd/system/crowdsec.service.d/override.conf
-
-${SYSTEMCTL} daemon-reload
-
-# test with no crowdsec agent in configuration file
-sudo cp ./config/config_no_agent.yaml /etc/crowdsec/config.yaml
-${SYSTEMCTL} start crowdsec 
-wait_for_service "crowdsec LAPI should run without agent (in configuration file)"
-
-
-## capi
-${CSCLI} -c ./config/config_no_agent.yaml capi status || fail "capi status should be ok"
-## config
-${CSCLI_BIN} -c ./config/config_no_agent.yaml config show || fail "failed to show config"
-${CSCLI} -c ./config/config_no_agent.yaml config backup ./test || fail "failed to backup config"
-sudo rm -rf ./test
-## lapi
-${CSCLI} -c ./config/config_no_agent.yaml lapi status || fail "lapi status failed"
-## metrics
-${CSCLI_BIN} -c ./config/config_no_agent.yaml metrics || fail "failed to get metrics"
-
-${SYSTEMCTL} stop crowdsec
-sudo cp ./config/config.yaml /etc/crowdsec/config.yaml
-rm -f /etc/systemd/system/crowdsec.service.d/override.conf
-${SYSTEMCTL} daemon-reload
-
-#######################
-## TEST WITHOUT CAPI ##
-echo "CROWDSEC (AGENT+LAPI)"
-
-# test with no online client in configuration file
-sudo cp ./config/config_no_capi.yaml /etc/crowdsec/config.yaml
-${SYSTEMCTL} start crowdsec 
-wait_for_service "crowdsec LAPI should run without CAPI (in configuration file)"
-
-## capi
-${CSCLI} -c ./config/config_no_capi.yaml capi status && fail "capi status should not be ok" ## if capi status success, it means that the test fail
-## config
-${CSCLI_BIN} -c ./config/config_no_capi.yaml config show || fail "failed to show config"
-${CSCLI} -c ./config/config_no_capi.yaml config backup ./test || fail "failed to backup config"
-sudo rm -rf ./test
-## lapi
-${CSCLI} -c ./config/config_no_capi.yaml lapi status || fail "lapi status failed"
-## metrics
-${CSCLI_BIN} -c ./config/config_no_capi.yaml metrics || fail "failed to get metrics"
-
-sudo cp ./config.yaml.backup /etc/crowdsec/config.yaml 
-
-${SYSTEMCTL} daemon-reload
-${SYSTEMCTL} restart crowdsec
-wait_for_service "crowdsec should be restarted)"

scripts/func_tests/tests_post-install_1bouncers.sh

@@ -1,27 +0,0 @@
-#! /usr/bin/env bash
-# -*- coding: utf-8 -*-
-
-source tests_base.sh
-
-
-## bouncers
-
-# we should have 0 bouncers
-${CSCLI} bouncers list -ojson  | ${JQ} '. | length == 0' || fail "expected 0 bouncers"
-
-# we can add one bouncer - should we save token for later ?
-${CSCLI} bouncers add ciTestBouncer || fail "failed to add bouncer"
-
-# but we can't add it twice - we would get a fatal error
-${CSCLI} bouncers add ciTestBouncer -ojson  2>&1 | ${JQ} '.level == "fatal"' || fail "didn't receive the expected error"
-
-# we should have 1 bouncer
-${CSCLI} bouncers list -ojson  | ${JQ} '. | length == 1' || fail "expected 1 bouncers"
-
-# delete the bouncer :)
-${CSCLI} bouncers delete ciTestBouncer || fail "failed to delete bouncer"
-
-# we should have 0 bouncers
-${CSCLI} bouncers list -ojson  | ${JQ} '. | length == 0' || fail "expected 0 bouncers"
-
-

scripts/func_tests/tests_post-install_2collections.sh

@@ -1,30 +0,0 @@
-#! /usr/bin/env bash
-# -*- coding: utf-8 -*-
-
-source tests_base.sh
-
-## collections
-
-${CSCLI_BIN} collections list || fail "failed to list collections"
-
-BASE_COLLECTION_COUNT=2
-
-# we expect 1 collections : linux 
-${CSCLI_BIN} collections list -ojson | ${JQ} ".collections | length == ${BASE_COLLECTION_COUNT}" || fail "(first) expected exactly ${BASE_COLLECTION_COUNT} collection"
-
-# install an extra collection
-${CSCLI} collections install crowdsecurity/mysql || fail "failed to install collection"
-
-BASE_COLLECTION_COUNT=$(($BASE_COLLECTION_COUNT+1))
-
-# we should now have 2 collections :)
-${CSCLI_BIN} collections list -ojson | ${JQ} ".collections | length == ${BASE_COLLECTION_COUNT}" || fail "(post install) expected exactly ${BASE_COLLECTION_COUNT} collection"
-
-# remove the collection
-${CSCLI} collections remove crowdsecurity/mysql || fail "failed to remove collection"
-
-BASE_COLLECTION_COUNT=$(($BASE_COLLECTION_COUNT-1))
-
-# we expect 1 collections : linux 
-${CSCLI_BIN} collections list -ojson | ${JQ} ".collections | length == ${BASE_COLLECTION_COUNT}" || fail "(post remove) expected exactly ${BASE_COLLECTION_COUNT} collection"
-

scripts/func_tests/tests_post-install_3machines.sh

@@ -1,22 +0,0 @@
-#! /usr/bin/env bash
-# -*- coding: utf-8 -*-
-
-source tests_base.sh
-
-## machines
-
-${CSCLI} machines list -ojson | ${JQ} '. | length == 1' || fail "expected exactly one machine"
-
-# add a new machine
-${CSCLI} machines add -a -f ./test_machine.yaml CiTestMachine -ojson || fail "expected exactly one machine"
-${CSCLI} machines list -ojson | ${JQ} '. | length == 2' || fail "expected exactly one machine"
-${CSCLI} machines delete CiTestMachine -ojson || fail "expected exactly one machine"
-${CSCLI} machines list -ojson | ${JQ} '. | length == 1' || fail "expected exactly one machine"
-
-#try register/validate
-${CSCLI} lapi register  --machine CiTestMachineRegister -f new_machine.yaml
-#the newly added machine isn't validated yet
-${CSCLI} machines list -ojson | ${JQ} '.[1].isValidated == null' || fail "machine shouldn't be validated"
-${CSCLI} machines validate CiTestMachineRegister  || fail "failed to validate machine"
-${CSCLI} machines list -ojson | ${JQ} '.[1].isValidated == true' || fail "machine should be validated"
-

scripts/func_tests/tests_post-install_4cold-logs.sh

@@ -1,61 +0,0 @@
-#! /usr/bin/env bash
-# -*- coding: utf-8 -*-
-
-source tests_base.sh
-
-
-# install sshd collection
-
-${CSCLI} collections install crowdsecurity/sshd
-${CSCLI} decisions delete --all
-${SYSTEMCTL} reload crowdsec
-
-
-# generate a fake bf log -> cold logs processing
-rm  -f ssh-bf.log
-
-sync
-
-for i in `seq 1 6` ; do 
-    echo `LC_ALL=C date '+%b %d %H:%M:%S '`'sd-126005 sshd[12422]: Invalid user netflix from 1.1.1.172 port 35424' >> ssh-bf.log
-done;
-
-sync
-
-${CROWDSEC} -dsn "file://./ssh-bf.log" -type syslog -no-api
-
-${CSCLI} decisions list -o=json | ${JQ} '. | length == 1' || fail "expected exactly one decision"
-${CSCLI} decisions list -o=json | ${JQ} '.[].decisions[0].value == "1.1.1.172"'  || fail "(exact) expected ban on 1.1.1.172"
-${CSCLI} decisions list -r 1.1.1.0/24 -o=json --contained | ${JQ} '.[].decisions[0].value == "1.1.1.172"' || fail "(range/contained) expected ban on 1.1.1.172"
-${CSCLI} decisions list -r 1.1.2.0/24 -o=json | ${JQ} '. == null'  || fail "(range/NOT-contained) expected no ban on 1.1.1.172"
-${CSCLI} decisions list -i 1.1.1.172 -o=json | ${JQ} '.[].decisions[0].value == "1.1.1.172"'  || fail "(range/NOT-contained) expected ban on 1.1.1.172"
-${CSCLI} decisions list -i 1.1.1.173 -o=json | ${JQ} '. == null' || fail "(exact) expected no ban on 1.1.1.173"
-
-# generate a live ssh bf
-
-${CSCLI} decisions delete --all
-
-sudo cp /etc/crowdsec/acquis.yaml ./acquis.yaml.backup
-echo "" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
-echo "filename: /tmp/test.log" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
-echo "labels:" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
-echo "  type: syslog" | sudo tee -a /etc/crowdsec/acquis.yaml > /dev/null
-touch /tmp/test.log
-
-${SYSTEMCTL} restart crowdsec
-wait_for_service "crowdsec should run (cold logs)"
-${SYSTEMCTL} status crowdsec
-
-sleep 2s
-
-cat ssh-bf.log >> /tmp/test.log
-
-sleep 5s
-${CSCLI} decisions list -o=json | ${JQ} '.[].decisions[0].value == "1.1.1.172"' || fail "(live) expected ban on 1.1.1.172"
-
-sudo cp ./acquis.yaml.backup /etc/crowdsec/acquis.yaml
-
-sync
-
-${SYSTEMCTL} restart crowdsec
-wait_for_service "crowdsec should run"

scripts/func_tests/tests_post-install_5simulation.sh

@@ -1,57 +0,0 @@
-#! /usr/bin/env bash
-# -*- coding: utf-8 -*-
-
-source tests_base.sh
-
-COLLECTION=crowdsecurity/sshd
-SCENARIO=crowdsecurity/ssh-bf
-
-# install sshd collection
-
-${CSCLI} collections install $COLLECTION
-${CSCLI} decisions delete --all
-${SYSTEMCTL} reload crowdsec
-
-
-# generate a fake bf log -> cold logs processing
-rm  -f ssh-bf.log
-
-sync
-
-for i in `seq 1 10` ; do 
-    echo `LC_ALL=C date '+%b %d %H:%M:%S '`'sd-126005 sshd[12422]: Invalid user netflix from 1.1.1.174 port 35424' >> ssh-bf.log
-done;
-
-sync
-
-${CROWDSEC} -dsn file://./ssh-bf.log -type syslog -no-api
-
-sleep 1s
-
-${CSCLI} decisions list -o=json | ${JQ} '. | length == 1' || fail "expected exactly one decision"
-${CSCLI} decisions list -o=json | ${JQ} '.[].decisions[0].value == "1.1.1.174"'  || fail "(exact) expected ban on 1.1.1.174"
-${CSCLI} decisions list -o=json | ${JQ} '.[].decisions[0].simulated == false'  || fail "(exact) expected simulated on false"
-
-
-sleep 1s
-
-# enable simulation on specific scenario and try with same logs
-
-${CSCLI} decisions delete --all
-${CSCLI} simulation enable $SCENARIO
-
-${CROWDSEC} -dsn file://./ssh-bf.log -type syslog -no-api
-
-${CSCLI} decisions list --no-simu -o=json | ${JQ} '. == null' || fail "expected no decision (listing only non-simulated decisions)"
-
-sleep 1s
-# enable global simulation and try with same logs
-
-${CSCLI} decisions delete --all
-${CSCLI} simulation disable $SCENARIO
-${CSCLI} simulation enable --global
-
-${CROWDSEC} -dsn file://./ssh-bf.log -type syslog -no-api
-
-sleep 1s
-${CSCLI} decisions list --no-simu -o=json | ${JQ} '. == null' || fail "expected no decision (listing only non-simulated decisions)"

scripts/func_tests/tests_post-install_6hubtests.sh

@@ -1,12 +0,0 @@
-#! /usr/bin/env bash
-# -*- coding: utf-8 -*-
-
-source tests_base.sh
-
-CURRENT_DIR=$(pwd)
-
-git clone https://github.com/crowdsecurity/hub.git
-cd hub/
-${CSCLI} hubtest run --all --clean
-
-cd "${CURRENT_DIR}"

scripts/func_tests/tests_post-install_7_plugin.sh

@@ -1,100 +0,0 @@
-#! /usr/bin/env bash
-# -*- coding: utf-8 -*-
-
-source tests_base.sh
-
-MOCK_SERVER_PID=""
-
-function backup () {
-    cat /etc/crowdsec/profiles.yaml > ./backup_profiles.yaml
-    cat /etc/crowdsec/notifications/http.yaml > ./backup_http.yaml
-}
-
-function restore_backup () {
-    cat ./backup_profiles.yaml | sudo tee /etc/crowdsec/profiles.yaml > /dev/null
-    cat ./backup_http.yaml | sudo tee /etc/crowdsec/notifications/http.yaml > /dev/null
-}
-
-function clear_backup() {
-    rm ./backup_profiles.yaml
-    rm ./backup_http.yaml
-}
-
-function modify_config() {
-    PLUGINS_DIR=$(sudo find /usr -type d -wholename "*"crowdsec/plugins)
-    sed -i "s#/usr/local/lib/crowdsec/plugins#${PLUGINS_DIR}#g" ./config/config.yaml
-    cat ./config/config.yaml | sed 's/group: nogroup/group: '$(groups nobody | cut -d ':' -f2 | tr -d ' ')'/' | sudo tee /etc/crowdsec/config.yaml > /dev/null
-    cat ./config/http.yaml | sudo tee /etc/crowdsec/notifications/http.yaml > /dev/null
-    cat ./config/profiles.yaml | sudo tee /etc/crowdsec/profiles.yaml > /dev/null
-    
-    ${SYSTEMCTL} restart crowdsec
-    sleep 5s
-}
-
-function setup_tests() {
-    backup
-    cscli decisions delete --all
-    modify_config
-    python3 -u mock_http_server.py > mock_http_server_logs.log &
-    count=0
-    while ! nc -z localhost 9999; do   
-        sleep 0.5
-        ((count ++))
-        if [[ $count == 41 ]]; then
-            fail "mock server not up after 20s"
-        fi
-    done
-
-    MOCK_SERVER_PID=$!
-}
-
-function cleanup_tests() {
-    restore_backup
-    clear_backup
-    kill -9 $MOCK_SERVER_PID
-    rm mock_http_server_logs.log
-    ${SYSTEMCTL} restart crowdsec
-    sleep 5s
-}
-
-function run_tests() {
-    log_line_count=$(cat mock_http_server_logs.log | wc -l)
-
-    if [[ $log_line_count -ne "0" ]] ; then
-        cleanup_tests
-        fail "expected 0 log lines fom mock http server before adding decisions"
-    fi
-    sleep 5s
-    ${CSCLI} decisions add --ip 1.2.3.4 --duration 30s
-    ${CSCLI} decisions add --ip 1.2.3.5 --duration 30s
-    sleep 5s
-    cat mock_http_server_logs.log
-    log_line_count=$(cat mock_http_server_logs.log | wc -l)
-    if [[ $log_line_count -ne "1" ]] ; then
-        cleanup_tests
-        fail "expected 1 log line from http server"
-    fi
-
-    total_alerts=$(cat mock_http_server_logs.log   | jq  .request_body | jq length)
-    if [[ $total_alerts -ne "2" ]] ; then
-        cleanup_tests
-        fail "expected to receive 2 alerts in the request body from plugin"
-    fi
-
-    first_received_ip=$(cat mock_http_server_logs.log  | jq -r .request_body[0].decisions[0].value)
-    if [[ $first_received_ip != "1.2.3.4" ]] ; then
-        cleanup_tests
-        fail "expected to receive IP 1.2.3.4 as value of first decision"
-    fi 
-
-    second_received_ip=$(cat mock_http_server_logs.log  | jq -r .request_body[1].decisions[0].value)
-    if [[ $second_received_ip != "1.2.3.5" ]] ; then
-        cleanup_tests
-        fail "expected to receive IP 1.2.3.5 as value of second decision"
-    fi 
-}
-
-setup_tests
-run_tests
-cleanup_tests
-

scripts/func_tests/tests_post-install_99ip_mgmt.sh

@@ -1,408 +0,0 @@
-#! /usr/bin/env bash
-# -*- coding: utf-8 -*-
-
-
-source tests_base.sh
-
-
-# Codes
-RED='\033[0;31m'
-GREEN='\033[0;32m'
-NC='\033[0m'
-OK_STR="${GREEN}OK${NC}"
-FAIL_STR="${RED}FAIL${NC}"
-
-
-CROWDSEC_API_URL="http://localhost:8081"
-CROWDSEC_VERSION=""
-API_KEY=""
-
-RELEASE_FOLDER_FULL=""
-FAILED="false"
-MUST_FAIL="false"
-
-### Helpers
-function docurl
-{
-    URI=$1
-    curl -s -H "X-Api-Key: ${API_KEY}" "${CROWDSEC_API_URL}${URI}"
-} 
-
-function bouncer_echo {
-    if [[ ${FAILED} == "false" ]];
-    then
-        echo -e "[bouncer] $1: ${OK_STR}"
-    else
-        echo -e "[bouncer] $1: ${FAIL_STR}"
-    fi
-    FAILED="false"
-}
-
-function cscli_echo {
-    if [[ ${FAILED} == "false" ]];
-    then
-        echo -e "[cscli]   $1: ${OK_STR}"
-    else
-        echo -e "[cscli]   $1: ${FAIL_STR}"
-    fi
-    FAILED="false"
-}
-
-function test_ipv4_ip
-{
-    echo ""
-    echo "##########################################"
-    echo "$FUNCNAME"
-    echo "##########################################"
-    echo ""
-    
-    ${CSCLI} decisions list -o json | ${JQ} '. == null' > /dev/null || fail
-    cscli_echo "first decisions list"
-    
-    docurl /v1/decisions | ${JQ} '. == null' > /dev/null || fail
-    bouncer_echo "first bouncer decisions request (must be empty)"
-
-    #add ip decision
-    echo "adding decision for 1.2.3.4"
-    ${CSCLI} decisions add -i 1.2.3.4  > /dev/null 2>&1 || fail
-    
-    ${CSCLI} decisions list -o json | ${JQ} '.[].decisions[0].value == "1.2.3.4"' > /dev/null || fail
-    cscli_echo "getting all decision"
-    
-    docurl /v1/decisions | ${JQ} '.[0].value == "1.2.3.4"' > /dev/null || fail
-    bouncer_echo "getting all decision"
-
-    #check ip match
-    ${CSCLI} decisions list -i 1.2.3.4 -o json | ${JQ} '.[].decisions[0].value == "1.2.3.4"'  > /dev/null || fail
-    cscli_echo "getting decision for 1.2.3.4"
-    
-    docurl /v1/decisions?ip=1.2.3.4 | ${JQ} '.[0].value == "1.2.3.4"' > /dev/null || fail
-    bouncer_echo "getting decision for 1.2.3.4"
-
-    ${CSCLI} decisions list -i 1.2.3.5 -o json | ${JQ} '. == null'  > /dev/null || fail
-    cscli_echo "getting decision for 1.2.3.5"
-
-    docurl /v1/decisions?ip=1.2.3.5 | ${JQ} '. == null' > /dev/null || fail
-    bouncer_echo "getting decision for 1.2.3.5"
-
-    #check outer range match
-    ${CSCLI} decisions list -r 1.2.3.0/24 -o json | ${JQ} '. == null'  > /dev/null || fail
-    cscli_echo "getting decision for 1.2.3.0/24"
-
-    docurl "/v1/decisions?range=1.2.3.0/24" | ${JQ} '. == null' > /dev/null || fail
-    bouncer_echo "getting decision for 1.2.3.0/24"
-
-    ${CSCLI} decisions list -r 1.2.3.0/24 --contained -o json |${JQ} '.[].decisions[0].value == "1.2.3.4"'  > /dev/null || fail
-    cscli_echo "getting decisions where IP in 1.2.3.0/24"
-
-    docurl "/v1/decisions?range=1.2.3.0/24&contains=false" | ${JQ} '.[0].value == "1.2.3.4"' > /dev/null || fail
-    bouncer_echo "getting decisions where IP in 1.2.3.0/24"
-
-}
-
-function test_ipv4_range
-{
-    echo ""
-    echo "##########################################"
-    echo "$FUNCNAME"
-    echo "##########################################"
-    echo ""
-
-
-    cscli_echo "adding decision for range 4.4.4.0/24"
-    ${CSCLI} decisions add -r 4.4.4.0/24 > /dev/null 2>&1 || fail
-
-    ${CSCLI} decisions list -o json | ${JQ} '.[0].decisions[0].value == "4.4.4.0/24", .[1].decisions[0].value == "1.2.3.4"'> /dev/null || fail
-    cscli_echo "getting all decision"
-    
-    docurl ${APIK} "/v1/decisions" | ${JQ} '.[0].value == "1.2.3.4", .[1].value == "4.4.4.0/24"'> /dev/null || fail
-    bouncer_echo "getting all decision"
-
-    #check ip within/outside of range
-    ${CSCLI} decisions list -i 4.4.4.3 -o json | ${JQ} '.[].decisions[0].value == "4.4.4.0/24"' > /dev/null || fail
-    cscli_echo "getting decisions for ip 4.4.4."
-
-    docurl ${APIK} "/v1/decisions?ip=4.4.4.3" | ${JQ} '.[0].value == "4.4.4.0/24"' > /dev/null || fail
-    bouncer_echo "getting decisions for ip 4.4.4."
-
-    ${CSCLI} decisions list -i 4.4.4.4 -o json --contained | ${JQ} '. == null'> /dev/null || fail
-    cscli_echo "getting decisions for ip contained in 4.4.4."
-
-    docurl ${APIK} "/v1/decisions?ip=4.4.4.4&contains=false" | ${JQ} '. == null' > /dev/null || fail
-    bouncer_echo "getting decisions for ip contained in 4.4.4."
-
-    ${CSCLI} decisions list -i 5.4.4.3 -o json | ${JQ} '. == null' > /dev/null || fail
-    cscli_echo "getting decisions for ip 5.4.4."
-
-    docurl ${APIK} "/v1/decisions?ip=5.4.4.3" | ${JQ} '. == null' > /dev/null || fail
-    bouncer_echo "getting decisions for ip 5.4.4."
-
-    ${CSCLI} decisions list -r 4.4.0.0/16 -o json | ${JQ} '. == null' > /dev/null || fail
-    cscli_echo "getting decisions for range 4.4.0.0/1"
-
-    docurl ${APIK} "/v1/decisions?range=4.4.0.0/16" | ${JQ} '. == null' > /dev/null || fail
-    bouncer_echo "getting decisions for range 4.4.0.0/1"
-
-    ${CSCLI} decisions list -r 4.4.0.0/16 -o json --contained | ${JQ} '.[].decisions[0].value == "4.4.4.0/24"' > /dev/null || fail
-    cscli_echo "getting decisions for ip/range in 4.4.0.0/1"
-
-    docurl ${APIK} "/v1/decisions?range=4.4.0.0/16&contains=false" | ${JQ} '.[0].value == "4.4.4.0/24"' > /dev/null || fail
-    bouncer_echo "getting decisions for ip/range in 4.4.0.0/1"
-
-    #check subrange
-    ${CSCLI} decisions list -r 4.4.4.2/28 -o json | ${JQ} '.[].decisions[0].value == "4.4.4.0/24"' > /dev/null || fail
-    cscli_echo "getting decisions for range 4.4.4.2/2"
-
-    docurl ${APIK} "/v1/decisions?range=4.4.4.2/28" | ${JQ} '.[].value == "4.4.4.0/24"' > /dev/null || fail
-    bouncer_echo "getting decisions for range 4.4.4.2/2"
-
-    ${CSCLI} decisions list -r 4.4.3.2/28 -o json | ${JQ} '. == null' > /dev/null || fail
-    cscli_echo "getting decisions for range 4.4.3.2/2"
-
-    docurl ${APIK} "/v1/decisions?range=4.4.3.2/28" | ${JQ} '. == null' > /dev/null || fail
-    bouncer_echo "getting decisions for range 4.4.3.2/2"
-
-}
-
-function test_ipv6_ip
-{
-
-    echo ""
-    echo "##########################################"
-    echo "$FUNCNAME"
-    echo "##########################################"
-    echo ""
-
-    cscli_echo "adding decision for ip 1111:2222:3333:4444:5555:6666:7777:8888"
-    ${CSCLI} decisions add -i 1111:2222:3333:4444:5555:6666:7777:8888 > /dev/null 2>&1
-
-    ${CSCLI} decisions list -o json | ${JQ} '.[].decisions[0].value == "1111:2222:3333:4444:5555:6666:7777:8888"'  > /dev/null || fail
-    cscli_echo "getting all decision"
-
-    docurl ${APIK} "/v1/decisions" | ${JQ} '.[].value == "1111:2222:3333:4444:5555:6666:7777:8888"' > /dev/null || fail
-    bouncer_echo "getting all decision"
-
-    ${CSCLI} decisions list -i 1111:2222:3333:4444:5555:6666:7777:8888 -o json | ${JQ} '.[].decisions[0].value == "1111:2222:3333:4444:5555:6666:7777:8888"'  > /dev/null || fail
-    cscli_echo "getting decisions for ip 1111:2222:3333:4444:5555:6666:7777:8888"
-    
-    docurl ${APIK} "/v1/decisions?ip=1111:2222:3333:4444:5555:6666:7777:8888" | ${JQ} '.[].value == "1111:2222:3333:4444:5555:6666:7777:8888"' > /dev/null || fail
-    bouncer_echo "getting decisions for ip 1111:2222:3333:4444:5555:6666:7777:888"
-
-    ${CSCLI} decisions list -i 1211:2222:3333:4444:5555:6666:7777:8888 -o json | ${JQ} '. == null' > /dev/null || fail
-    cscli_echo "getting decisions for ip 1211:2222:3333:4444:5555:6666:7777:8888"
-
-    docurl ${APIK} "/v1/decisions?ip=1211:2222:3333:4444:5555:6666:7777:8888" | ${JQ} '. == null' > /dev/null || fail
-    bouncer_echo "getting decisions for ip 1211:2222:3333:4444:5555:6666:7777:888"
-
-    ${CSCLI} decisions list -i 1111:2222:3333:4444:5555:6666:7777:8887 -o json | ${JQ} '. == null' > /dev/null || fail
-    cscli_echo "getting decisions for ip 1111:2222:3333:4444:5555:6666:7777:8887"
-
-    docurl ${APIK} "/v1/decisions?ip=1111:2222:3333:4444:5555:6666:7777:8887" | ${JQ} '. == null' > /dev/null || fail
-    bouncer_echo "getting decisions for ip 1111:2222:3333:4444:5555:6666:7777:888"
-
-    ${CSCLI} decisions list -r 1111:2222:3333:4444:5555:6666:7777:8888/48 -o json | ${JQ} '. == null' > /dev/null || fail
-    cscli_echo "getting decisions for range 1111:2222:3333:4444:5555:6666:7777:8888/48"
-
-    docurl ${APIK} "/v1/decisions?range=1111:2222:3333:4444:5555:6666:7777:8888/48" | ${JQ} '. == null' > /dev/null || fail
-    bouncer_echo "getting decisions for range 1111:2222:3333:4444:5555:6666:7777:8888/48"
-
-    ${CSCLI} decisions list -r 1111:2222:3333:4444:5555:6666:7777:8888/48 --contained -o json | ${JQ} '.[].decisions[0].value == "1111:2222:3333:4444:5555:6666:7777:8888"' > /dev/null || fail
-    cscli_echo "getting decisions for ip/range in range 1111:2222:3333:4444:5555:6666:7777:8888/48"
-
-    docurl ${APIK} "/v1/decisions?range=1111:2222:3333:4444:5555:6666:7777:8888/48&&contains=false" | ${JQ} '.[].value == "1111:2222:3333:4444:5555:6666:7777:8888"' > /dev/null || fail
-    bouncer_echo "getting decisions for ip/range in 1111:2222:3333:4444:5555:6666:7777:8888/48"
-
-    ${CSCLI} decisions list -r 1111:2222:3333:4444:5555:6666:7777:8888/64 -o json | ${JQ} '. == null' > /dev/null || fail
-    cscli_echo "getting decisions for range 1111:2222:3333:4444:5555:6666:7777:8888/64"
-
-    docurl ${APIK} "/v1/decisions?range=1111:2222:3333:4444:5555:6666:7777:8888/64" | ${JQ} '. == null' > /dev/null || fail
-    bouncer_echo "getting decisions for range 1111:2222:3333:4444:5555:6666:7777:8888/64"
-
-    ${CSCLI} decisions list -r 1111:2222:3333:4444:5555:6666:7777:8888/64 -o json --contained | ${JQ} '.[].decisions[0].value == "1111:2222:3333:4444:5555:6666:7777:8888"'  > /dev/null || fail
-    cscli_echo "getting decisions for ip/range in 1111:2222:3333:4444:5555:6666:7777:8888/64"
-
-    docurl ${APIK} "/v1/decisions?range=1111:2222:3333:4444:5555:6666:7777:8888/64&&contains=false" | ${JQ} '.[].value == "1111:2222:3333:4444:5555:6666:7777:8888"' > /dev/null || fail
-    bouncer_echo "getting decisions for ip/range in 1111:2222:3333:4444:5555:6666:7777:8888/64"
-
-    cscli_echo "adding decision for ip 1111:2222:3333:4444:5555:6666:7777:8889"
-    ${CSCLI} decisions add -i 1111:2222:3333:4444:5555:6666:7777:8889 > /dev/null 2>&1
-
-    cscli_echo "deleting decision for ip 1111:2222:3333:4444:5555:6666:7777:8889"
-    ${CSCLI} decisions delete -i 1111:2222:3333:4444:5555:6666:7777:8889
-
-    ${CSCLI} decisions list -i 1111:2222:3333:4444:5555:6666:7777:8889 -o json | ${JQ} '. == null' > /dev/null || fail
-    cscli_echo "getting decisions for ip 1111:2222:3333:4444:5555:6666:7777:8889 after delete"
-
-    cscli_echo "deleting decision for range 1111:2222:3333:4444:5555:6666:7777:8888/64"
-    ${CSCLI} decisions delete -r 1111:2222:3333:4444:5555:6666:7777:8888/64 --contained
-
-    ${CSCLI} decisions list -r 1111:2222:3333:4444:5555:6666:7777:8888/64 -o json --contained | ${JQ} '. == null'  > /dev/null || fail
-    cscli_echo "getting decisions for ip/range in 1111:2222:3333:4444:5555:6666:7777:8888/64 after delete"
-}
-
-function test_ipv6_range
-{
-    echo ""
-    echo "##########################################"
-    echo "$FUNCNAME"
-    echo "##########################################"
-    echo ""
-
-    cscli_echo "adding decision for range aaaa:2222:3333:4444::/64"
-    ${CSCLI} decisions add -r aaaa:2222:3333:4444::/64 > /dev/null 2>&1 || fail
-     
-    ${CSCLI} decisions list -o json | ${JQ} '.[0].decisions[0].value == "aaaa:2222:3333:4444::/64"' > /dev/null || fail
-    cscli_echo "getting all decision"
-
-    docurl ${APIK} "/v1/decisions" | ${JQ} '.[0].value == "aaaa:2222:3333:4444::/64"' > /dev/null || fail
-    bouncer_echo "getting all decision"
-
-    #check ip within/out of range
-    ${CSCLI} decisions list -i aaaa:2222:3333:4444:5555:6666:7777:8888 -o json | ${JQ} '.[].decisions[0].value == "aaaa:2222:3333:4444::/64"'  > /dev/null || fail
-    cscli_echo "getting decisions for ip aaaa:2222:3333:4444:5555:6666:7777:8888"
-
-    docurl ${APIK} "/v1/decisions?ip=aaaa:2222:3333:4444:5555:6666:7777:8888" | ${JQ} '.[].value == "aaaa:2222:3333:4444::/64"' > /dev/null || fail
-    bouncer_echo "getting decisions for ip aaaa:2222:3333:4444:5555:6666:7777:8888"
-
-    ${CSCLI} decisions list -i aaaa:2222:3333:4445:5555:6666:7777:8888 -o json | ${JQ} '. == null' > /dev/null || fail
-    cscli_echo "getting decisions for ip aaaa:2222:3333:4445:5555:6666:7777:8888"
-
-    docurl ${APIK} "/v1/decisions?ip=aaaa:2222:3333:4445:5555:6666:7777:8888" | ${JQ} '. == null' > /dev/null || fail
-    bouncer_echo "getting decisions for ip aaaa:2222:3333:4445:5555:6666:7777:8888"
-
-    ${CSCLI} decisions list -i aaa1:2222:3333:4444:5555:6666:7777:8887 -o json | ${JQ} '. == null' > /dev/null || fail
-    cscli_echo "getting decisions for ip aaa1:2222:3333:4444:5555:6666:7777:8887"
-
-    docurl ${APIK} "/v1/decisions?ip=aaa1:2222:3333:4444:5555:6666:7777:8887" | ${JQ} '. == null' > /dev/null || fail
-    bouncer_echo "getting decisions for ip aaa1:2222:3333:4444:5555:6666:7777:8887"
-
-    #check subrange within/out of range
-    ${CSCLI} decisions list -r aaaa:2222:3333:4444:5555::/80 -o json | ${JQ} '.[].decisions[0].value == "aaaa:2222:3333:4444::/64"'  > /dev/null || fail
-    cscli_echo "getting decisions for range aaaa:2222:3333:4444:5555::/80"
-    
-    docurl ${APIK} "/v1/decisions?range=aaaa:2222:3333:4444:5555::/80" | ${JQ} '.[].value == "aaaa:2222:3333:4444::/64"' > /dev/null || fail
-    bouncer_echo "getting decisions for range aaaa:2222:3333:4444:5555::/80"
-
-    ${CSCLI} decisions list -r aaaa:2222:3333:4441:5555::/80 -o json | ${JQ} '. == null'  > /dev/null || fail
-    cscli_echo "getting decisions for range aaaa:2222:3333:4441:5555::/80"
-    
-    docurl ${APIK} "/v1/decisions?range=aaaa:2222:3333:4441:5555::/80" | ${JQ} '. == null' > /dev/null || fail
-    bouncer_echo "getting decisions for range aaaa:2222:3333:4441:5555::/80"
-
-    ${CSCLI} decisions list -r aaa1:2222:3333:4444:5555::/80 -o json | ${JQ} '. == null'  > /dev/null || fail
-    cscli_echo "getting decisions for range aaa1:2222:3333:4444:5555::/80"
-
-    docurl ${APIK} "/v1/decisions?range=aaa1:2222:3333:4444:5555::/80" | ${JQ} '. == null' > /dev/null || fail
-    bouncer_echo "getting decisions for range aaa1:2222:3333:4444:5555::/80"
-
-    #check outer range
-    ${CSCLI} decisions list -r aaaa:2222:3333:4444:5555:6666:7777:8888/48 -o json | ${JQ} '. == null' > /dev/null || fail
-    cscli_echo "getting decisions for range aaaa:2222:3333:4444:5555:6666:7777:8888/48"
-
-    docurl ${APIK} "/v1/decisions?range=aaaa:2222:3333:4444:5555:6666:7777:8888/48" | ${JQ} '. == null' > /dev/null || fail
-    bouncer_echo "getting decisions for range aaaa:2222:3333:4444:5555:6666:7777:8888/48"
-
-    ${CSCLI} decisions list -r aaaa:2222:3333:4444:5555:6666:7777:8888/48 -o json --contained | ${JQ} '.[].decisions[0].value == "aaaa:2222:3333:4444::/64"' > /dev/null || fail
-    cscli_echo "getting decisions for ip/range in aaaa:2222:3333:4444:5555:6666:7777:8888/48"
-
-    docurl ${APIK} "/v1/decisions?range=aaaa:2222:3333:4444:5555:6666:7777:8888/48&contains=false" | ${JQ} '.[].value == "aaaa:2222:3333:4444::/64"' > /dev/null || fail
-    bouncer_echo "getting decisions for ip/range in aaaa:2222:3333:4444:5555:6666:7777:8888/48"
-
-    ${CSCLI} decisions list -r aaaa:2222:3333:4445:5555:6666:7777:8888/48 -o json | ${JQ} '. == null' > /dev/null || fail
-    cscli_echo "getting decisions for ip/range aaaa:2222:3333:4445:5555:6666:7777:8888/48"
-
-    docurl ${APIK} "/v1/decisions?range=aaaa:2222:3333:4445:5555:6666:7777:8888/48" | ${JQ} '. == null' > /dev/null || fail
-    bouncer_echo "getting decisions for ip/range in aaaa:2222:3333:4445:5555:6666:7777:8888/48"
-
-    #bbbb:db8:: -> bbbb:db8:0000:0000:0000:7fff:ffff:ffff
-    ${CSCLI} decisions add -r bbbb:db8::/81 > /dev/null 2>&1
-    cscli_echo "adding decision for range bbbb:db8::/81" > /dev/null || fail
-
-    ${CSCLI} decisions list -o json -i bbbb:db8:0000:0000:0000:6fff:ffff:ffff | ${JQ} '.[].decisions[0].value == "bbbb:db8::/81"'  > /dev/null || fail
-    cscli_echo "getting decisions for ip bbbb:db8:0000:0000:0000:6fff:ffff:ffff"
-    
-    docurl ${APIK} "/v1/decisions?ip=bbbb:db8:0000:0000:0000:6fff:ffff:ffff" | ${JQ} '.[].value == "bbbb:db8::/81"' > /dev/null || fail
-    bouncer_echo "getting decisions for ip in bbbb:db8:0000:0000:0000:6fff:ffff:ffff"
-
-    ${CSCLI} decisions list -o json -i bbbb:db8:0000:0000:0000:8fff:ffff:ffff | ${JQ} '. == null' > /dev/null || fail
-    cscli_echo "getting decisions for ip bbbb:db8:0000:0000:0000:8fff:ffff:ffff"
-
-    docurl ${APIK} "/v1/decisions?ip=bbbb:db8:0000:0000:0000:8fff:ffff:ffff" | ${JQ} '. == null' > /dev/null || fail
-    bouncer_echo "getting decisions for ip in bbbb:db8:0000:0000:0000:8fff:ffff:ffff"
-
-    cscli_echo "deleting decision for range aaaa:2222:3333:4444:5555:6666:7777:8888/48"
-    ${CSCLI} decisions delete -r aaaa:2222:3333:4444:5555:6666:7777:8888/48 --contained > /dev/null 2>&1 || fail
-    
-    ${CSCLI} decisions list -o json -r aaaa:2222:3333:4444::/64 | ${JQ} '. == null' > /dev/null || fail
-    cscli_echo "getting decisions for range aaaa:2222:3333:4444::/64 after delete"
-
-    cscli_echo "adding decision for ip bbbb:db8:0000:0000:0000:8fff:ffff:ffff"
-    ${CSCLI} decisions add -i bbbb:db8:0000:0000:0000:8fff:ffff:ffff > /dev/null 2>&1 || fail
-    cscli_echo "adding decision for ip bbbb:db8:0000:0000:0000:6fff:ffff:ffff"
-    ${CSCLI} decisions add -i bbbb:db8:0000:0000:0000:6fff:ffff:ffff > /dev/null 2>&1 || fail
-
-    cscli_echo "deleting decision for range bbbb:db8::/81"
-    ${CSCLI} decisions delete -r bbbb:db8::/81 --contained > /dev/null 2>&1 || fail
-    
-    ${CSCLI} decisions list -o json | ${JQ} '.[].decisions[0].value == "bbbb:db8:0000:0000:0000:8fff:ffff:ffff"' > /dev/null || fail
-    cscli_echo "getting all decisions"
-
-}
-
-
-function start_test
-{
-
-    ## ipv4 testing
-    ${CSCLI} decisions delete --all
-
-    test_ipv4_ip
-    test_ipv4_range
-
-    ## ipv6 testing
-    ${CSCLI} decisions delete --all
-    test_ipv6_ip
-    test_ipv6_range
-}
-
-
-usage() {
-      echo "Usage:"
-      echo ""
-      echo "    ./ip_mgmt_tests.sh -h                                   Display this help message."
-      echo "    ./ip_mgmt_tests.sh                                      Run all the testsuite. Go must be available to make the release"
-      echo "    ./ip_mgmt_tests.sh --release <path_to_release_folder>   If go is not installed, please provide a path to the crowdsec-vX.Y.Z release folder"
-      echo ""
-      exit 0  
-}
-
-while [[ $# -gt 0 ]]
-do
-    key="${1}"
-    case ${key} in
-    --release|-r)
-        RELEASE_FOLDER="${2}"
-        shift #past argument
-        shift
-        ;;   
-    -h|--help)
-        usage
-        exit 0
-        ;;
-    *)    # unknown option
-        echo "Unknown argument ${key}."
-        usage
-        exit 1
-        ;;
-    esac
-done
-
-
-start_test
-
-if [[ "${MUST_FAIL}" == "true" ]];
-then
-    echo ""
-    echo "One or more tests have failed !"
-    exit 1
-fi

scripts/func_tests/tests_post-remove_0base.sh

@@ -1,7 +0,0 @@
-#! /usr/bin/env bash
-# -*- coding: utf-8 -*-
-
-source tests_base.sh
-
-pidof crowdsec && fail "crowdsec shouldn't run anymore" || true
-

scripts/test_env.sh

@@ -1,122 +0,0 @@
-#!/bin/bash
-
-BASE="./tests"
-
-usage() {
-	  echo "Usage:"
-	  echo "    ./wizard.sh -h                               Display this help message."
-	  echo "    ./test_env.sh -d ./tests                     Create test environment in './tests' folder"
-	  exit 0
-}
-
-
-while [[ $# -gt 0 ]]
-do
-	key="${1}"
-	case ${key} in
-	-d|--directory)
-		BASE=${2}
-		shift #past argument
-		shift
-		;;  
-	-h|--help)
-		usage
-		exit 0
-		;;
-	*)    # unknown option
-		log_err "Unknown argument ${key}."
-		usage
-		exit 1
-		;;
-	esac
-done
-
-BASE=$(realpath $BASE)
-
-DATA_DIR="$BASE/data"
-
-LOG_DIR="$BASE/logs/"
-
-CONFIG_DIR="$BASE/config"
-CONFIG_FILE="$BASE/dev.yaml"
-CSCLI_DIR="$CONFIG_DIR/crowdsec-cli"
-PARSER_DIR="$CONFIG_DIR/parsers"
-PARSER_S00="$PARSER_DIR/s00-raw"
-PARSER_S01="$PARSER_DIR/s01-parse"
-PARSER_S02="$PARSER_DIR/s02-enrich"
-SCENARIOS_DIR="$CONFIG_DIR/scenarios"
-POSTOVERFLOWS_DIR="$CONFIG_DIR/postoverflows"
-HUB_DIR="$CONFIG_DIR/hub"
-PLUGINS="http slack splunk"
-PLUGINS_DIR="plugins"
-NOTIF_DIR="notifications"
-
-log_info() {
-	msg=$1
-	date=$(date +%x:%X)
-	echo -e "[$date][INFO] $msg"
-}
-
-create_arbo() {
-	mkdir -p "$BASE"
-	mkdir -p "$DATA_DIR"
-	mkdir -p "$LOG_DIR"
-	mkdir -p "$CONFIG_DIR"
-	mkdir -p "$PARSER_DIR"
-	mkdir -p "$PARSER_S00"
-	mkdir -p "$PARSER_S01"
-	mkdir -p "$PARSER_S02"
-	mkdir -p "$SCENARIOS_DIR"
-	mkdir -p "$POSTOVERFLOWS_DIR"
-	mkdir -p "$CSCLI_DIR"
-	mkdir -p "$HUB_DIR"
-	mkdir -p $CONFIG_DIR/$NOTIF_DIR/$plugin
-	mkdir -p $BASE/$PLUGINS_DIR
-}
-
-copy_files() {
-	cp "./config/profiles.yaml" "$CONFIG_DIR"
-	cp  "./config/simulation.yaml" "$CONFIG_DIR"
-	cp "./cmd/crowdsec/crowdsec" "$BASE"
-	cp "./cmd/crowdsec-cli/cscli" "$BASE"
-	cp -r "./config/patterns" "$CONFIG_DIR"
-	cp "./config/acquis.yaml" "$CONFIG_DIR"
-	touch "$CONFIG_DIR"/local_api_credentials.yaml
-	touch "$CONFIG_DIR"/online_api_credentials.yaml
-	envsubst < "./config/dev.yaml" > $BASE/dev.yaml
-	for plugin in $PLUGINS
-	do
-		cp $PLUGINS_DIR/$NOTIF_DIR/$plugin/notification-$plugin $BASE/$PLUGINS_DIR/notification-$plugin
-		cp $PLUGINS_DIR/$NOTIF_DIR/$plugin/$plugin.yaml $CONFIG_DIR/$NOTIF_DIR/$plugin.yaml
-	done
-}
-
-
-setup() {
-	$BASE/cscli -c "$CONFIG_FILE" hub update
-	$BASE/cscli -c "$CONFIG_FILE" collections install crowdsecurity/linux
-}
-
-setup_api() {
-	$BASE/cscli -c "$CONFIG_FILE" machines add test -p testpassword -f $CONFIG_DIR/local_api_credentials.yaml --force
-}
-
-
-main() {
-	log_info "Creating test arboresence in $BASE"
-	create_arbo
-	log_info "Arboresence created"
-	log_info "Copying needed files for tests environment"
-	copy_files
-	log_info "Files copied"
-	log_info "Setting up configurations"
-	CURRENT_PWD=$(pwd)
-	cd $BASE
-	setup_api
-	setup
-	cd $CURRENT_PWD
-	log_info "Environment is ready in $BASE"
-}
-
-
-main

tests/.gitignore

@@ -0,0 +1,4 @@
+/local/
+/local-init/
+/.environment.sh
+/dyn-bats/*.bats

tests/README.md

@@ -0,0 +1,297 @@
+
+# What is this?
+
+This directory contains scripts for functional testing. The tests are run with
+the [bats-core](https://github.com/bats-core/bats-core) framework, which is an
+active fork of the older BATS (Bash Automated Testing System).
+
+The goal is to be cross-platform but not explicitly test the packaging system
+or service management. Those parts are specific to each distribution and are
+tested separately (triggered by crowdsec releases, but they run in other
+repositories).
+
+### cscli
+
+| Feature               | Covered            | Notes                      |
+| :-------------------- | :----------------- | :------------------------- |
+| `cscli alerts`        | -                  |                            |
+| `cscli bouncers`      | `10_bouncers`      |                            |
+| `cscli capi`          | `01_base`          | `status` only              |
+| `cscli collections`   | `20_collections`   |                            |
+| `cscli config`        | `01_base`          | minimal testing (no crash) |
+| `cscli dashboard`     | -                  | docker inside docker 😞    |
+| `cscli decisions`     | `9[78]_ipv[46]*`   |                            |
+| `cscli hub`           | `dyn_bats/99_hub`  |                            |
+| `cscli lapi`          | `01_base`          |                            |
+| `cscli machines`      | `30_machines`      |                            |
+| `cscli metrics`       | -                  |                            |
+| `cscli parsers`       | -                  |                            |
+| `cscli postoverflows` | -                  |                            |
+| `cscli scenarios`     | -                  |                            |
+| `cscli simulation`    | `50_simulation`    |                            |
+| `cscli version`       | `01_base`          |                            |
+
+### crowdsec
+
+| Feature                        | Covered        | Notes                                      |
+| :----------------------------- | :------------- | :----------------------------------------- |
+| `systemctl` start/stop/restart | -              |                                            |
+| agent behaviour                | `40_live-ban`  | minimal testing  (simple ssh-bf detection) |
+| forensic mode                  | `40_cold-logs` | minimal testing (simple ssh-bf detection)  |
+| starting withou LAPI           | `02_nolapi`    |                                            |
+| starting without agent         | `03_noagent`   |                                            |
+| starting without CAPI          | `04_nocapi`    |                                            |
+| prometheus testing             | -              |                                            |
+
+### API
+
+| Feature            | Covered          | Notes        |
+| :----------------- | :--------------- | :----------- |
+| alerts GET/POST    | `9[78]_ipv[46]*` |              |
+| decisions GET/POST | `9[78]_ipv[46]*` |              |
+
+
+# How to use it
+
+Run `make clean bats-all` to perform a test build + run.
+
+To repeat test runs without rebuilding crowdsec, use `make bats-test`.
+
+
+# How does it work?
+
+In BATS, you write tests in the form of Bash functions that have unique
+descriptions (the name of the test). You can do most things that you can
+normally do in a shell function. If there is any error condition, the test
+fails. A set of functions is provided to implement assertions, and a mechanism
+of `setup`/`teardown` is provided a the level of individual tests (functions)
+or group of tests (files).
+
+The stdout/stderr of the commands within the test function are captured by
+bats-core and will only be shown if the test fails. If you want to always print
+something to debug your test case, you can redirect the output to the file
+descriptor 3:
+
+```sh
+@test "mytest" {
+   echo "hello world!" >&3
+   run some-command
+   assert_success
+   echo "goodbye." >&3
+}
+```
+
+If you do that, please remove it once the test is finished, because this practice breaks the test protocol.
+
+You can find here the documentation for the main framework and the plugins we use in this test suite:
+
+ - [bats-core tutorial](https://bats-core.readthedocs.io/en/stable/tutorial.html)
+ - [Writing tests](https://bats-core.readthedocs.io/en/stable/writing-tests.html)
+ - [bats-assert](https://github.com/bats-core/bats-assert)
+ - [bats-support](https://github.com/bats-core/bats-support)
+ - [bats-file](https://github.com/bats-core/bats-file)
+
+> As it often happens with open source, the first results from search engines refer to the old, unmaintained forks.
+> Be sure to use the links above to find the good versions.
+
+Since bats-core is [TAP (Test Anything Protocol)](https://testanything.org/)
+compliant, its output is in a standardized format. It can be integrated with a
+separate [tap reporter](https://www.npmjs.com/package/tape#pretty-reporters) or
+included in a larger test suite. The TAP specification is pretty minimalist and
+some glue may be needed.
+
+
+Other tools that you can find useful:
+
+ - [mikefarah/yq](https://github.com/mikefarah/yq) - to parse and update YAML files on the fly
+ - [aliou/bats.vim](https://github.com/aliou/bats.vim) - for syntax highlighting (use bash otherwise)
+
+# setup and teardown
+
+If you have read the bats-core tutorial linked above, you are aware of the
+`setup` and `teardown` functions.
+
+What you may have overlooked is that the script body outside the functions is
+executed multiple times, so we have to be careful of what we put there.
+
+Here we have a look at the execution flow with two tests:
+
+```sh
+echo "begin" >&3
+
+setup_file() {
+        echo "setup_file" >&3
+}
+
+teardown_file() {
+        echo "teardown_file" >&3
+}
+
+setup() {
+        echo "setup" >&3
+}
+
+teardown() {
+        echo "teardown" >&3
+}
+
+@test "test 1" {
+        echo "test #1" >&3
+}
+
+@test "test 2" {
+        echo "test #2" >&3
+}
+
+echo "end" >&3
+```
+
+The above test suite produces the following output:
+
+```
+begin
+end
+setup_file
+begin
+end
+ ✓ test 1
+setup
+test #1
+teardown
+begin
+end
+ ✓ test 2
+setup
+test #2
+teardown
+teardown_file
+```
+
+See how "begin" and "end" are repeated three times each? The code outside
+setup/teardown/test functions is really executed three times (more as you add
+more tests). You can put there variables or function definitions, but keep it
+to a minimum and [don't write anything to the standard
+output](https://bats-core.readthedocs.io/en/stable/writing-tests.html#code-outside-of-test-cases).
+For most things you want to use `setup_file()` instead.
+
+But.. there is a but. Quoting from [the FAQ](https://bats-core.readthedocs.io/en/stable/faq.html):
+
+> You can simply source <your>.sh files. However, be aware that source`ing
+> files with errors outside of any function (or inside `setup_file) will trip
+> up bats and lead to hard to diagnose errors. Therefore, it is safest to only
+> source inside setup or the test functions themselves.
+
+This doesn't mean you can't do that, just that you're on your own if the is an error.
+
+
+# Testing crowdsec
+
+## Fixtures
+
+For the purpose of functional tests, crowdsec and its companions (cscli, plugin
+notifiers, bouncers) are installed in a local environment, which means tests
+should not install or touch anything outside a `./tests/local` directory. This
+includes binaries, configuration files, databases, data downloaded from
+internet, logs... The use of `/tmp` is tolerated, but BATS also provides [three
+useful
+variables](https://bats-core.readthedocs.io/en/stable/writing-tests.html#special-variables):
+`$BATS_SUITE_TMPDIR`, `$BATS_FILE_TMPDIR` and `$BATS_TEST_TMPDIR` that let you
+ensure your desired level of isolation of temporary files across the tests.
+
+When built with `make bats-build`, the binaries will look there by default for
+their configuration and data needs. So you can run `./local/bin/cscli` from
+a shell with no need for further parameters.
+
+To set up the installation described above we provide a couple of scripts,
+`instance-data` and `instance-crowdsec`. They manage fixture and background
+processes; they are meant to be used in setup/teardown in several ways,
+according to the specific needs of the group of tests in the file.
+
+ - `instance-data make`
+
+   Creates a tar file in `./local-init/init-config-data.tar`.
+   The file contains all the configuration, hub and database files needed
+   to restore crowdsec to a known initial state.
+   Things like `machines add ...`, `capi register`, `hub update`, `collections
+   install crowdsecurity/linux` are executed here so they don't need to be
+   repeated for each test or group of tests.
+
+ - `instance-data load`
+
+   Extracts the files created by `instance-data make` for use by the local
+   crowdsec instance. Crowdsec must not be running while this operation is
+   performed.
+
+ - `instance-crowdsec [ start | stop ]`
+
+   Runs (or stops) crowdsec as a background process. PID and lockfiles are
+   written in `./local/var/run/`.
+
+
+Here are some ways to use these two scripts.
+
+ - case 1: load a fresh crowsec instance + data for each test (01_base, 10_bouncers, 20_collections...)
+
+    This offers the best isolation, but the tests run slower. More importantly,
+    since there is no concept of "grouping" tests in bats-core with the exception
+    of files, if you need to perform some setup that is common to two or more
+    tests, you will have to repeat the code.
+
+ - case 2: load a fresh set of data for each test, but run crowdsec only for
+   the tests that need it, possibly after altering the configuration
+   (02_nolapi, 03_noagent, 04_nocapi, 40_live-ban)
+
+    This is useful because: 1) you sometimes don't want crowdsec to run at all,
+    for example when testing `cscli` in isolation, or you may want to tweak the
+    configuration inside the test function before running the lapi/agent. See
+    how we use `yq` to change the YAML files to that effect.
+
+ - case 3: start crowdsec with the inital set of configuration+data once, and keep it
+   running for all the tests (50_simulation, 98_ipv4, 98_ipv6)
+
+     This offers no isolation across tests, which over time could break more
+     often as result, but you can rely on the test order to test more complex
+     scenarios with a reasonable performance and the least amount of code.
+
+
+## status, stdout and stderr
+
+As we said, if any error occurs inside a test function, the test
+fails immediately. You call `mycommand`, it exits with $? != 0, the test fails.
+
+But how to test the output, then? If we call `run mycommand`, then $? will be 0
+allowing the test to keep running. The real error status is stored in the
+`$status` variable, and the command output and standard error content are put
+together in the `$output` variable. By specifying `run --separate-stderr`, you
+can have separated `$output` and `$stderr` variables.
+
+The above is better explained in the bats-core tutorial. If you have not read it
+yet, now is a good time.
+
+The `$output` variable gets special treatment with the
+[bats-support](https://github.com/bats-core/bats-support) and
+[bats-assert][https://github.com/bats-core/bats-assert) plugins and can be
+checked with `assert_*` commands. The `$stderr` variable does not have these,
+but we can use `run echo "$stderr"` and then check `$output` with asserts.
+
+Remember that `run` always overwrites the `$output` variable, so if you consume
+it with `run jq <(output)` you can only do it once, because the second time it
+will read the output of the `jq` command. But you can construct a list of all
+the values you want and check them all in a single step.
+
+See `lib/setup_file.sh` for other tricks we employ.
+
+## file operations
+
+We included the [bats-file](https://github.com/bats-core/bats-file) plugin to
+check the result of file system operations: existence, type/size/ownership checks
+on files, symlinks, directories, sockets.
+
+## gotchas
+
+ - pay attention to tests that are not run - for example "bats warning: Executed 143
+   instead of expected 144 tests". They are especially tricky to debug.
+
+ - using the `load` command in `teardown()` causes tests to be silently skipped or break in "funny"
+   ways. The other functions seem safe.
+

tests/assert-crowdsec-not-running

@@ -0,0 +1,19 @@
+#!/usr/bin/env bash
+
+pgrep crowdsec >/dev/null || exit 0
+
+# removing this second test causes CI to fail sometimes
+sleep 2
+pgrep crowdsec >/dev/null || exit 0
+
+msg="A CrowdSec process is already running. Please terminate it and run the tests again."
+
+# Are we inside a setup() or @test? Is file descriptor 3 open?
+if { true >&3; } 2>/dev/null; then
+    echo "$msg" >&3
+else
+    echo "$msg" >&2
+fi
+
+# cause the calling setup() or @test to fail
+exit 1

tests/bats.mk

@@ -0,0 +1,66 @@
+TEST_DIR = $(ROOT)/tests
+LOCAL_DIR = $(TEST_DIR)/local
+
+BIN_DIR = $(LOCAL_DIR)/bin
+CONFIG_DIR = $(LOCAL_DIR)/etc/crowdsec
+DATA_DIR = $(LOCAL_DIR)/var/lib/crowdsec/data
+LOCAL_INIT_DIR = $(TEST_DIR)/local-init
+LOG_DIR = $(LOCAL_DIR)/var/log
+PID_DIR = $(LOCAL_DIR)/var/run
+PLUGIN_DIR = $(LOCAL_DIR)/lib/crowdsec/plugins
+
+define ENV :=
+export TEST_DIR="$(TEST_DIR)"
+export LOCAL_DIR="$(LOCAL_DIR)"
+export BIN_DIR="$(BIN_DIR)"
+export CONFIG_DIR="$(CONFIG_DIR)"
+export DATA_DIR="$(DATA_DIR)"
+export LOCAL_INIT_DIR="$(LOCAL_INIT_DIR)"
+export LOG_DIR="$(LOG_DIR)"
+export PID_DIR="$(PID_DIR)"
+export PLUGIN_DIR="$(PLUGIN_DIR)"
+endef
+
+bats-all: bats-clean bats-build bats-instance-data bats-test
+
+# Source this to run the scripts outside of the Makefile
+bats-environment:
+	$(file >$(TEST_DIR)/.environment.sh,$(ENV))
+
+# See if bats-core has been cloned from the repo
+check-bats-libs:
+	@$(TEST_DIR)/lib/bats-core/bin/bats --version >/dev/null 2>&1 || (echo "ERROR: bats-core submodule is required. Please run 'git submodule init; git submodule update' and retry."; exit 1)
+
+# Builds and installs crowdsec in a local directory
+bats-build: bats-environment
+	@DEFAULT_CONFIGDIR=$(CONFIG_DIR) DEFAULT_DATADIR=$(DATA_DIR) $(MAKE) build
+	@mkdir -p $(BIN_DIR) $(CONFIG_DIR) $(DATA_DIR) $(LOG_DIR) $(PID_DIR) $(LOCAL_INIT_DIR) $(PLUGIN_DIR)
+	@install -m 0755 cmd/crowdsec/crowdsec $(BIN_DIR)/
+	@install -m 0755 cmd/crowdsec-cli/cscli $(BIN_DIR)/
+	@install -m 0755 plugins/notifications/email/notification-email $(PLUGIN_DIR)/
+	@install -m 0755 plugins/notifications/http/notification-http $(PLUGIN_DIR)/
+	@install -m 0755 plugins/notifications/slack/notification-slack $(PLUGIN_DIR)/
+	@install -m 0755 plugins/notifications/splunk/notification-splunk $(PLUGIN_DIR)/
+
+# Create a reusable package with initial configuration + data
+bats-instance-data: bats-environment
+	$(TEST_DIR)/instance-data make
+
+# Removes the local crowdsec installation and the fixture config + data
+bats-clean:
+	@$(RM) -r $(LOCAL_DIR) $(LOCAL_INIT_DIR) $(TEST_DIR)/dyn-bats/*.bats
+
+# Creates the hub tests
+bats-generate-hubtests: bats-environment check-bats-libs
+	${TEST_DIR}/generate-hub-tests
+
+# Run the test suite
+bats-test: bats-environment check-bats-libs bats-generate-hubtests
+	$(TEST_DIR)/run-tests
+
+# Static checks for the test scripts.
+# Not failproof but they can catch bugs and improve learning of sh/bash
+bats-lint:
+	@shellcheck --version >/dev/null 2>&1 || (echo "ERROR: shellcheck is required."; exit 1)
+	@shellcheck -x ${TEST_DIR}/bats/*.bats
+

tests/bats/01_base.bats

@@ -0,0 +1,129 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh" >&3 2>&1
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh" >&3 2>&1
+}
+
+setup() {
+    load "../lib/setup.sh"
+    ./instance-data load
+    ./instance-crowdsec start
+}
+
+teardown() {
+    ./instance-crowdsec stop
+}
+
+# to silence shellcheck
+declare stderr
+
+#----------
+
+@test "$FILE cscli version" {
+    run -0 cscli version
+    assert_output --partial "version:"
+    assert_output --partial "Codename:"
+    assert_output --partial "BuildDate:"
+    assert_output --partial "GoVersion:"
+    assert_output --partial "Platform:"
+    assert_output --partial "Constraint_parser:"
+    assert_output --partial "Constraint_scenario:"
+    assert_output --partial "Constraint_api:"
+    assert_output --partial "Constraint_acquis:"
+}
+
+@test "$FILE cscli alerts list: at startup returns one entry: community pull" {
+    loop_max=15
+    for ((i=0; i<=loop_max; i++)); do
+        sleep 2
+        run -0 cscli alerts list -o json
+        [[ "$output" != "null" ]] && break
+    done
+    run -0 jq -r '. | length' <(output)
+    assert_output 1
+}
+
+@test "$FILE cscli capi status" {
+    run -0 cscli capi status
+    assert_output --partial "Loaded credentials from"
+    assert_output --partial "Trying to authenticate with username"
+    assert_output --partial " on https://api.crowdsec.net/"
+    assert_output --partial "You can successfully interact with Central API (CAPI)"
+}
+
+@test "$FILE cscli config show -o human" {
+    run -0 cscli config show -o human
+    assert_output --partial "Global:"
+    assert_output --partial "Crowdsec:"
+    assert_output --partial "cscli:"
+    assert_output --partial "Local API Server:"
+}
+
+@test "$FILE cscli config show -o json" {
+    run -0 cscli config show -o json
+    assert_output --partial '"API":'
+    assert_output --partial '"Common":'
+    assert_output --partial '"ConfigPaths":'
+    assert_output --partial '"Crowdsec":'
+    assert_output --partial '"Cscli":'
+    assert_output --partial '"DbConfig":'
+    assert_output --partial '"Hub":'
+    assert_output --partial '"PluginConfig":'
+    assert_output --partial '"Prometheus":'
+}
+
+@test "$FILE cscli config show -o raw" {
+    run -0 cscli config show -o raw
+    assert_line "api:"
+    assert_line "common:"
+    assert_line "config_paths:"
+    assert_line "crowdsec_service:"
+    assert_line "cscli:"
+    assert_line "db_config:"
+    assert_line "plugin_config:"
+    assert_line "prometheus:"
+}
+
+@test "$FILE cscli config show --key" {
+    run -0 cscli config show --key Config.API.Server.ListenURI
+    assert_output "127.0.0.1:8080"
+}
+
+@test "$FILE cscli config backup" {
+    tempdir=$(mktemp -u -p "${BATS_TEST_TMPDIR}")
+    run -0 cscli config backup "${tempdir}"
+    assert_output --partial "Starting configuration backup"
+    run -1 --separate-stderr cscli config backup "${tempdir}"
+
+    run -0 echo "$stderr"
+    assert_output --partial "Failed to backup configurations"
+    assert_output --partial "file exists"
+    rm -rf -- "${tempdir:?}"
+}
+
+@test "$FILE cscli lapi status" {
+    run -0 --separate-stderr cscli lapi status
+
+    run -0 echo "$stderr"
+    assert_output --partial "Loaded credentials from"
+    assert_output --partial "Trying to authenticate with username"
+    assert_output --partial " on http://127.0.0.1:8080/"
+    assert_output --partial "You can successfully interact with Local API (LAPI)"
+}
+
+@test "$FILE cscli metrics" {
+    run -0 cscli lapi status
+    run -0 --separate-stderr cscli metrics
+    assert_output --partial "ROUTE"
+    assert_output --partial '/v1/watchers/login'
+
+    run -0 echo "$stderr"
+    assert_output --partial "Local Api Metrics:"
+}

tests/bats/02_nolapi.bats

@@ -0,0 +1,94 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh" >&3 2>&1
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh" >&3 2>&1
+}
+
+setup() {
+    load "../lib/setup.sh"
+    # always reset config and data, but run the daemon only if one test requires it
+    ./instance-data load
+}
+
+teardown() {
+    ./instance-crowdsec stop
+}
+
+declare stderr
+
+#----------
+
+@test "$FILE test without -no-api flag" {
+    run -124 --separate-stderr timeout 1s "${CROWDSEC}"
+    # from `man timeout`: If  the  command  times  out,  and --preserve-status is not set, then exit with status 124.
+}
+
+@test "$FILE crowdsec should not run without LAPI (-no-api flag)" {
+    run -1 --separate-stderr timeout 1s "${CROWDSEC}" -no-api
+}
+
+@test "$FILE crowdsec should not run without LAPI (no api.server in configuration file)" {
+    yq 'del(.api.server)' -i "${CONFIG_DIR}/config.yaml"
+    run -1 --separate-stderr timeout 1s "${CROWDSEC}"
+
+    run -0 echo "$stderr"
+    assert_output --partial "crowdsec local API is disabled"
+}
+
+@test "$FILE capi status shouldn't be ok without api.server" {
+    yq 'del(.api.server)' -i "${CONFIG_DIR}/config.yaml"
+    run -1 --separate-stderr cscli capi status
+
+    run -0 echo "$stderr"
+    assert_output --partial "Local API is disabled, please run this command on the local API machine"
+}
+
+@test "$FILE cscli config show -o human" {
+    yq 'del(.api.server)' -i "${CONFIG_DIR}/config.yaml"
+    run -0 cscli config show -o human
+    assert_output --partial "Global:"
+    assert_output --partial "Crowdsec:"
+    assert_output --partial "cscli:"
+    refute_output --partial "Local API Server:"
+}
+
+@test "$FILE cscli config backup" {
+    yq 'del(.api.server)' -i "${CONFIG_DIR}/config.yaml"
+    tempdir=$(mktemp -u -p "${BATS_TEST_TMPDIR}")
+    run -0 cscli config backup "${tempdir}"
+    assert_output --partial "Starting configuration backup"
+    run -1 --separate-stderr cscli config backup "${tempdir}"
+    rm -rf -- "${tempdir:?}"
+
+    run -0 echo "$stderr"
+    assert_output --partial "Failed to backup configurations"
+    assert_output --partial "file exists"
+}
+
+@test "$FILE lapi status shouldn't be ok without api.server" {
+    yq 'del(.api.server)' -i "${CONFIG_DIR}/config.yaml"
+    ./instance-crowdsec start
+    run -1 --separate-stderr cscli machines list
+    run -0 echo "$stderr"
+    assert_output --partial "Local API is disabled, please run this command on the local API machine"
+}
+
+@test "$FILE cscli metrics" {
+    skip 'need to trigger metrics with a live parse'
+    yq 'del(.api.server)' -i "${CONFIG_DIR}/config.yaml"
+    ./instance-crowdsec start
+    run -0 --separate-stderr cscli metrics
+    assert_output --partial "ROUTE"
+    assert_output --partial "/v1/watchers/login"
+
+    run -0 echo "$stderr"
+    assert_output --partial "crowdsec local API is disabled"
+    assert_output --partial "Local API is disabled, please run this command on the local API machine"
+}

tests/bats/03_noagent.bats

@@ -0,0 +1,94 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh" >&3 2>&1
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh" >&3 2>&1
+}
+
+setup() {
+    load "../lib/setup.sh"
+    ./instance-data load
+}
+
+teardown() {
+    ./instance-crowdsec stop
+}
+
+declare stderr
+
+#----------
+
+config_disable_agent() {
+    yq 'del(.crowdsec_service)' -i "${CONFIG_DIR}/config.yaml"
+}
+
+@test "$FILE with agent: test without -no-cs flag" {
+    run -124 timeout 1s "${CROWDSEC}"
+    # from `man timeout`: If  the  command  times  out,  and --preserve-status is not set, then exit with status 124.
+}
+
+@test "$FILE no agent: crowdsec LAPI should run (-no-cs flag)" {
+    run -124 timeout 1s "${CROWDSEC}" -no-cs
+}
+
+@test "$FILE no agent: crowdsec LAPI should run (no crowdsec_service in configuration file)" {
+    config_disable_agent
+    run -124 --separate-stderr timeout 1s "${CROWDSEC}"
+
+    run -0 echo "$stderr"
+    assert_output --partial "crowdsec agent is disabled"
+}
+
+@test "$FILE no agent: capi status should be ok" {
+    config_disable_agent
+    ./instance-crowdsec start
+    run -0 --separate-stderr cscli capi status
+
+    run -0 echo "$stderr"
+    assert_output --partial "You can successfully interact with Central API (CAPI)"
+}
+
+@test "$FILE no agent: cscli config show" {
+    config_disable_agent
+    run -0 --separate-stderr cscli config show -o human
+    assert_output --partial "Global:"
+    assert_output --partial "cscli:"
+    assert_output --partial "Local API Server:"
+
+    refute_output --partial "Crowdsec:"
+}
+
+@test "$FILE no agent: cscli config backup" {
+    config_disable_agent
+    tempdir=$(mktemp -u -p "${BATS_TEST_TMPDIR}")
+    run -0 cscli config backup "${tempdir}"
+    assert_output --partial "Starting configuration backup"
+    run -1 --separate-stderr cscli config backup "${tempdir}"
+
+    run -0 echo "$stderr"
+    assert_output --partial "Failed to backup configurations"
+    assert_output --partial "file exists"
+    rm -rf -- "${tempdir:?}"
+}
+
+@test "$FILE no agent: lapi status should be ok" {
+    config_disable_agent
+    ./instance-crowdsec start
+    run -0 --separate-stderr cscli lapi status
+
+    run -0 echo "$stderr"
+    assert_output --partial "You can successfully interact with Local API (LAPI)"
+}
+
+@test "$FILE cscli metrics" {
+    config_disable_agent
+    ./instance-crowdsec start
+    run -0 cscli lapi status
+    run -0 cscli metrics
+}

tests/bats/04_nocapi.bats

@@ -0,0 +1,90 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh" >&3 2>&1
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh" >&3 2>&1
+}
+
+setup() {
+    load "../lib/setup.sh"
+    ./instance-data load
+}
+
+teardown() {
+    ./instance-crowdsec stop
+}
+
+declare stderr
+
+#----------
+
+config_disable_capi() {
+    yq 'del(.api.server.online_client)' -i "${CONFIG_DIR}/config.yaml"
+}
+
+@test "$FILE without capi: crowdsec LAPI should still work" {
+    config_disable_capi
+    run -124 --separate-stderr timeout 1s "${CROWDSEC}"
+    # from `man timeout`: If  the  command  times  out,  and --preserve-status is not set, then exit with status 124.
+
+    run -0 echo "$stderr"
+    assert_output --partial "push and pull to Central API disabled"
+}
+
+@test "$FILE without capi: cscli capi status -> fail" {
+    config_disable_capi
+    ./instance-crowdsec start
+    run -1 --separate-stderr cscli capi status
+
+    run -0 echo "$stderr"
+    assert_output --partial "no configuration for Central API in "
+}
+
+@test "$FILE no capi: cscli config show" {
+    config_disable_capi
+    run -0 --separate-stderr cscli config show -o human
+    assert_output --partial "Global:"
+    assert_output --partial "cscli:"
+    assert_output --partial "Crowdsec:"
+    assert_output --partial "Local API Server:"
+}
+
+@test "$FILE no agent: cscli config backup" {
+    config_disable_capi
+    tempdir=$(mktemp -u -p "${BATS_TEST_TMPDIR}")
+    run -0 cscli config backup "${tempdir}"
+    assert_output --partial "Starting configuration backup"
+    run -1 --separate-stderr cscli config backup "${tempdir}"
+
+    run -0 echo "$stderr"
+    assert_output --partial "Failed to backup configurations"
+    assert_output --partial "file exists"
+    rm -rf -- "${tempdir:?}"
+}
+
+@test "$FILE without capi: cscli lapi status -> success" {
+    config_disable_capi
+    ./instance-crowdsec start
+    run -0 --separate-stderr cscli lapi status
+
+    run -0 echo "$stderr"
+    assert_output --partial "You can successfully interact with Local API (LAPI)"
+}
+
+@test "$FILE cscli metrics" {
+    config_disable_capi
+    ./instance-crowdsec start
+    run -0 cscli lapi status
+    run -0 --separate-stderr cscli metrics
+    assert_output --partial "ROUTE"
+    assert_output --partial '/v1/watchers/login'
+
+    run -0 echo "$stderr"
+    assert_output --partial "Local Api Metrics:"
+}

tests/bats/10_bouncers.bats

@@ -0,0 +1,58 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh" >&3 2>&1
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh" >&3 2>&1
+}
+
+setup() {
+    load "../lib/setup.sh"
+    ./instance-data load
+    ./instance-crowdsec start
+}
+
+teardown() {
+    ./instance-crowdsec stop
+}
+
+#----------
+
+@test "$FILE there are 0 bouncers" {
+    run -0 cscli bouncers list -o json
+    assert_output "[]"
+}
+
+@test "$FILE we can add one bouncer, and delete it" {
+    run -0 cscli bouncers add ciTestBouncer
+    assert_output --partial "Api key for 'ciTestBouncer':"
+    run -0 cscli bouncers delete ciTestBouncer
+    run -0 cscli bouncers list -o json
+    assert_output '[]'
+}
+
+@test "$FILE we can't add the same bouncer twice" {
+    run -0 cscli bouncers add ciTestBouncer
+    run -1 --separate-stderr cscli bouncers add ciTestBouncer -o json
+
+    run -0 jq -r '.level' <(stderr)
+    assert_output 'fatal'
+    run -0 jq -r '.msg' <(stderr)
+    assert_output "unable to create bouncer: bouncer ciTestBouncer already exists"
+
+    run -0 cscli bouncers list -o json
+    run -0 jq '. | length' <(output)
+    assert_output 1
+}
+
+@test "$FILE delete the bouncer multiple times, even if it does not exist" {
+    run -0 cscli bouncers add ciTestBouncer
+    run -0 cscli bouncers delete ciTestBouncer
+    run -0 cscli bouncers delete ciTestBouncer
+    run -0 cscli bouncers delete foobarbaz
+}

tests/bats/20_collections.bats

@@ -0,0 +1,55 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh" >&3 2>&1
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh" >&3 2>&1
+}
+
+setup() {
+    load "../lib/setup.sh"
+    ./instance-data load
+    ./instance-crowdsec start
+}
+
+teardown() {
+    ./instance-crowdsec stop
+}
+
+#----------
+
+@test "$FILE we can list collections" {
+    run -0 cscli collections list
+}
+
+@test "$FILE there are 2 collections (linux and sshd)" {
+    run -0 cscli collections list -o json
+    run -0 jq '.collections | length' <(output)
+    assert_output 2
+}
+
+@test "$FILE can install a collection (as a regular user) and remove it" {
+    run -0 cscli collections install crowdsecurity/mysql -o human
+    assert_output --partial "Enabled crowdsecurity/mysql"
+    run -0 cscli collections list -o json
+    run -0 jq '.collections | length' <(output)
+    assert_output 3
+    run -0 cscli collections remove crowdsecurity/mysql -o human
+    assert_output --partial "Removed symlink [crowdsecurity/mysql]"
+}
+
+@test "$FILE cannot remove a collection twice" {
+    run -0 cscli collections install crowdsecurity/mysql -o human
+    run -0 --separate-stderr cscli collections remove crowdsecurity/mysql
+    run -1 --separate-stderr cscli collections remove crowdsecurity/mysql -o json
+    run -0 jq -r '.level' <(stderr)
+    assert_output 'fatal'
+    run -0 jq -r '.msg' <(stderr)
+    assert_output --partial "unable to disable crowdsecurity/mysql"
+    assert_output --partial "doesn't exist"
+}

tests/bats/30_machines.bats

@@ -0,0 +1,89 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh" >&3 2>&1
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh" >&3 2>&1
+}
+
+setup() {
+    load "../lib/setup.sh"
+    ./instance-data load
+    ./instance-crowdsec start
+}
+
+teardown() {
+    ./instance-crowdsec stop
+}
+
+#----------
+
+@test "$FILE can list machines as regular user" {
+    run -0 cscli machines list
+}
+
+@test "$FILE we have exactly one machine, localhost" {
+    run -0 cscli machines list -o json
+    run -0 jq -c '[. | length, .[0].machineId, .[0].isValidated, .[0].ipAddress]' <(output)
+    assert_output '[1,"githubciXXXXXXXXXXXXXXXXXXXXXXXX",true,"127.0.0.1"]'
+    # the machine gets an IP address when it talks to the LAPI
+    # XXX already done in instance-data make
+    #run -0 cscli lapi status
+    #run -0 cscli machines list -o json
+    #run -0 jq -r '.[0].ipAddress' <(output)
+    #assert_output '127.0.0.1'
+}
+
+@test "$FILE add a new machine and delete it" {
+    run -0 cscli machines add -a -f /dev/null CiTestMachine -o human
+    assert_output --partial "Machine 'CiTestMachine' successfully added to the local API"
+    assert_output --partial "API credentials dumped to '/dev/null'"
+
+    # we now have two machines
+    run -0 cscli machines list -o json
+    run -0 jq -c '[. | length, .[-1].machineId, .[0].isValidated]' <(output)
+    assert_output '[2,"CiTestMachine",true]'
+
+    # delete the test machine
+    run -0 cscli machines delete CiTestMachine -o human
+    assert_output --partial "machine 'CiTestMachine' deleted successfully"
+
+    # we now have one machine again
+    run -0 cscli machines list -o json
+    run -0 jq '. | length' <(output)
+    assert_output 1
+}
+
+@test "$FILE register, validate and then remove a machine" {
+    run -0 cscli lapi register --machine CiTestMachineRegister -f /dev/null -o human
+    assert_output --partial "Successfully registered to Local API (LAPI)"
+    assert_output --partial "Local API credentials dumped to '/dev/null'"
+
+    # "the machine is not validated yet" {
+    run -0 cscli machines list -o json
+    run -0 jq '.[-1].isValidated' <(output)
+    assert_output 'null'
+
+    # "validate the machine" {
+    run -0 cscli machines validate CiTestMachineRegister -o human
+    assert_output --partial "machine 'CiTestMachineRegister' validated successfully"
+
+    # the machine is now validated
+    run -0 cscli machines list -o json
+    run -0 jq '.[-1].isValidated' <(output)
+    assert_output 'true'
+
+    # delete the test machine again
+    run -0 cscli machines delete CiTestMachineRegister -o human
+    assert_output --partial "machine 'CiTestMachineRegister' deleted successfully"
+
+    # we now have one machine, again
+    run -0 cscli machines list -o json
+    run -0 jq '. | length' <(output)
+    assert_output 1
+}

tests/bats/40_cold-logs.bats

@@ -0,0 +1,63 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+fake_log() {
+    for _ in $(seq 1 6); do
+        echo "$(LC_ALL=C date '+%b %d %H:%M:%S ')"'sd-126005 sshd[12422]: Invalid user netflix from 1.1.1.172 port 35424'
+    done
+}
+
+setup_file() {
+    load "../lib/setup_file.sh" >&3 2>&1
+
+    # we reset config and data, and only run the daemon once for all the tests in this file
+    ./instance-data load
+    ./instance-crowdsec start
+    fake_log | "${CROWDSEC}" -dsn file:///dev/fd/0 -type syslog -no-api
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh" >&3 2>&1
+}
+
+setup() {
+    load "../lib/setup.sh"
+}
+
+#----------
+
+@test "$FILE we have one decision" {
+    run -0 cscli decisions list -o json
+    run -0 jq '. | length' <(output)
+    assert_output 1
+}
+
+@test "$FILE 1.1.1.172 has been banned" {
+    run -0 cscli decisions list -o json
+    run -0 jq -r '.[].decisions[0].value' <(output)
+    assert_output '1.1.1.172'
+}
+
+@test "$FILE 1.1.1.172 has been banned (range/contained: -r 1.1.1.0/24 --contained)" {
+    run -0 cscli decisions list -r 1.1.1.0/24 --contained -o json
+    run -0 jq -r '.[].decisions[0].value' <(output)
+    assert_output '1.1.1.172'
+}
+
+@test "$FILE 1.1.1.172 has not been banned (range/NOT-contained: -r 1.1.2.0/24)" {
+    run -0 cscli decisions list -r 1.1.2.0/24 -o json
+    assert_output 'null'
+}
+
+@test "$FILE 1.1.1.172 has been banned (exact: -i 1.1.1.172)" {
+    run -0 cscli decisions list -i 1.1.1.172 -o json
+    run -0 jq -r '.[].decisions[0].value' <(output)
+    assert_output '1.1.1.172'
+}
+
+@test "$FILE 1.1.1.173 has not been banned (exact: -i 1.1.1.173)" {
+    run -0 cscli decisions list -i 1.1.1.173 -o json
+    assert_output 'null'
+}

tests/bats/40_live-ban.bats

@@ -0,0 +1,45 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+fake_log() {
+    for _ in $(seq 1 6); do
+        echo "$(LC_ALL=C date '+%b %d %H:%M:%S ')"'sd-126005 sshd[12422]: Invalid user netflix from 1.1.1.172 port 35424'
+    done
+}
+
+setup_file() {
+    load "../lib/setup_file.sh" >&3 2>&1
+    # we reset config and data, but run the daemon only in the tests that need it
+    ./instance-data load
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh" >&3 2>&1
+}
+
+setup() {
+    load "../lib/setup.sh"
+}
+
+teardown() {
+    ./instance-crowdsec stop
+}
+
+#----------
+
+@test "$FILE 1.1.1.172 has been banned" {
+    tmpfile=$(mktemp -p "${BATS_TEST_TMPDIR}")
+    touch "${tmpfile}"
+    echo -e "---\nfilename: $tmpfile\nlabels:\n  type: syslog\n" >>"${CONFIG_DIR}/acquis.yaml"
+
+    ./instance-crowdsec start
+    sleep 2
+    fake_log >>"${tmpfile}"
+    sleep 2
+    rm -f -- "${tmpfile}"
+    run -0 cscli decisions list -o json
+    run -0 jq -r '.[].decisions[0].value' <(output)
+    assert_output '1.1.1.172'
+}

tests/bats/50_simulation.bats

@@ -0,0 +1,66 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+fake_log() {
+    for _ in $(seq 1 10); do
+        echo "$(LC_ALL=C date '+%b %d %H:%M:%S ')"'sd-126005 sshd[12422]: Invalid user netflix from 1.1.1.174 port 35424'
+    done
+}
+
+setup_file() {
+    load "../lib/setup_file.sh" >&3 2>&1
+    ./instance-data load
+    ./instance-crowdsec start
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh" >&3 2>&1
+}
+
+setup() {
+    load "../lib/setup.sh"
+    cscli decisions delete --all
+}
+
+#----------
+
+@test "$FILE we have one decision" {
+    run -0 cscli simulation disable --global
+    fake_log | "${CROWDSEC}" -dsn file:///dev/fd/0 -type syslog -no-api
+    run -0 cscli decisions list -o json
+    run -0 jq '. | length' <(output)
+    assert_output 1
+}
+
+@test "$FILE 1.1.1.174 has been banned (exact)" {
+    run -0 cscli simulation disable --global
+    fake_log | "${CROWDSEC}" -dsn file:///dev/fd/0 -type syslog -no-api
+    run -0 cscli decisions list -o json
+    run -0 jq -r '.[].decisions[0].value' <(output)
+    assert_output '1.1.1.174'
+}
+
+@test "$FILE decision has simulated == false (exact)" {
+    run -0 cscli simulation disable --global
+    fake_log | "${CROWDSEC}" -dsn file:///dev/fd/0 -type syslog -no-api
+    run -0 cscli decisions list -o json
+    run -0 jq '.[].decisions[0].simulated' <(output)
+    assert_output 'false'
+}
+
+@test "$FILE simulated scenario, listing non-simulated: expect no decision" {
+    run -0 cscli simulation enable crowdsecurity/ssh-bf
+    fake_log | "${CROWDSEC}" -dsn file:///dev/fd/0 -type syslog -no-api
+    run -0 cscli decisions list --no-simu -o json
+    assert_output 'null'
+}
+
+@test "$FILE global simulation, listing non-simulated: expect no decision" {
+    run -0 cscli simulation disable crowdsecurity/ssh-bf
+    run -0 cscli simulation enable --global
+    fake_log | "${CROWDSEC}" -dsn file:///dev/fd/0 -type syslog -no-api
+    run -0 cscli decisions list --no-simu -o json
+    assert_output 'null'
+}

tests/bats/70_plugins.bats

@@ -0,0 +1,74 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh" >&3 2>&1
+    ./instance-data load
+
+    MOCK_OUT="${LOG_DIR}/mock-http.out"
+    export MOCK_OUT
+
+    yq '
+        .url="http://localhost:9999" |
+        .group_wait="5s" |
+        .group_threshold=2
+    ' -i "${CONFIG_DIR}/notifications/http.yaml"
+
+    yq '
+        .notifications=["http_default"] |
+        .filters=["Alert.GetScope() == \"Ip\""]
+    ' -i "${CONFIG_DIR}/profiles.yaml"
+
+    yq '
+        .plugin_config.user="" |
+        .plugin_config.group=""
+    ' -i "${CONFIG_DIR}/config.yaml"
+
+    rm -f -- "${MOCK_OUT}"
+
+    ./instance-crowdsec start
+    ./instance-mock-http start 9999
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh" >&3 2>&1
+    ./instance-mock-http stop
+}
+
+setup() {
+    load "../lib/setup.sh"
+}
+
+#----------
+
+@test "$FILE add two bans" {
+    run -0 cscli decisions add --ip 1.2.3.4 --duration 30s
+    assert_output --partial 'Decision successfully added'
+
+    run -0 cscli decisions add --ip 1.2.3.5 --duration 30s
+    assert_output --partial 'Decision successfully added'
+    sleep 2
+}
+
+@test "$FILE expected 1 log line from http server" {
+    run -0 wc -l <"${MOCK_OUT}"
+    assert_output 1
+}
+
+@test "$FILE expected to receive 2 alerts in the request body from plugin" {
+    run -0 jq -r '.request_body' <"${MOCK_OUT}"
+    run -0 jq -r 'length' <(output)
+    assert_output 2
+}
+
+@test "$FILE expected to receive IP 1.2.3.4 as value of first decision" {
+    run -0 jq -r '.request_body[0].decisions[0].value' <"${MOCK_OUT}"
+    assert_output 1.2.3.4
+}
+
+@test "$FILE expected to receive IP 1.2.3.5 as value of second decision" {
+    run -0 jq -r '.request_body[1].decisions[0].value' <"${MOCK_OUT}"
+    assert_output 1.2.3.5
+}

tests/bats/97_ipv4_single.bats

@@ -0,0 +1,104 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh" >&3 2>&1
+    ./instance-data load
+    ./instance-crowdsec start
+    API_KEY=$(cscli bouncers add testbouncer -o raw)
+    export API_KEY
+    CROWDSEC_API_URL="http://localhost:8080"
+    export CROWDSEC_API_URL
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh" >&3 2>&1
+}
+
+setup() {
+    load "../lib/setup.sh"
+}
+
+api() {
+    URI="$1"
+    curl -s -H "X-Api-Key: ${API_KEY}" "${CROWDSEC_API_URL}${URI}"
+}
+
+#----------
+
+@test "$FILE cli - first decisions list: must be empty" {
+    run -0 cscli decisions list -o json
+    assert_output 'null'
+}
+
+@test "$FILE API - first decisions list: must be empty" {
+    run -0 api '/v1/decisions'
+    assert_output 'null'
+}
+
+@test "$FILE adding decision for 1.2.3.4" {
+    run -0 cscli decisions add -i '1.2.3.4'
+    assert_output --partial 'Decision successfully added'
+}
+
+@test "$FILE CLI - all decisions" {
+    run -0 cscli decisions list -o json
+    run -0 jq -r '.[0].decisions[0].value' <(output)
+    assert_output '1.2.3.4'
+}
+
+@test "$FILE API - all decisions" {
+    run -0 api '/v1/decisions'
+    run -0 jq -c '[ . | length, .[0].value ]' <(output)
+    assert_output '[1,"1.2.3.4"]'
+}
+
+# check ip match
+
+@test "$FILE CLI - decision for 1.2.3.4" {
+    run -0 cscli decisions list -i '1.2.3.4' -o json
+    run -0 jq -r '.[0].decisions[0].value' <(output)
+    assert_output '1.2.3.4'
+}
+
+@test "$FILE API - decision for 1.2.3.4" {
+    run -0 api '/v1/decisions?ip=1.2.3.4'
+    run -0 jq -r '.[0].value' <(output)
+    assert_output '1.2.3.4'
+}
+
+@test "$FILE CLI - decision for 1.2.3.5" {
+    run -0 cscli decisions list -i '1.2.3.5' -o json
+    assert_output 'null'
+}
+
+@test "$FILE API - decision for 1.2.3.5" {
+    run -0 api '/v1/decisions?ip=1.2.3.5'
+    assert_output 'null'
+}
+
+## check outer range match
+
+@test "$FILE CLI - decision for 1.2.3.0/24" {
+    run -0 cscli decisions list -r '1.2.3.0/24' -o json
+    assert_output 'null'
+}
+
+@test "$FILE API - decision for 1.2.3.0/24" {
+    run -0 api '/v1/decisions?range=1.2.3.0/24'
+    assert_output 'null'
+}
+
+@test "$FILE CLI - decisions where IP in 1.2.3.0/24" {
+    run -0 cscli decisions list -r '1.2.3.0/24' --contained -o json
+    run -0 jq -r '.[0].decisions[0].value' <(output)
+    assert_output '1.2.3.4'
+}
+
+@test "$FILE API - decisions where IP in 1.2.3.0/24" {
+    run -0 api '/v1/decisions?range=1.2.3.0/24&contains=false'
+    run -0 jq -r '.[0].value' <(output)
+    assert_output '1.2.3.4'
+}

tests/bats/97_ipv6_single.bats

@@ -0,0 +1,147 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh" >&3 2>&1
+    ./instance-data load
+    ./instance-crowdsec start
+    API_KEY=$(cscli bouncers add testbouncer -o raw)
+    export API_KEY
+    CROWDSEC_API_URL="http://localhost:8080"
+    export CROWDSEC_API_URL
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh" >&3 2>&1
+}
+
+setup() {
+    load "../lib/setup.sh"
+}
+
+#----------
+
+api() {
+    URI="$1"
+    curl -s -H "X-Api-Key: ${API_KEY}" "${CROWDSEC_API_URL}${URI}"
+}
+
+@test "$FILE adding decision for ip 1111:2222:3333:4444:5555:6666:7777:8888" {
+    run -0 cscli decisions add -i '1111:2222:3333:4444:5555:6666:7777:8888'
+    assert_output --partial 'Decision successfully added'
+}
+
+@test "$FILE CLI - all decisions" {
+    run -0 cscli decisions list -o json
+    run -0 jq -r '.[].decisions[0].value' <(output)
+    assert_output '1111:2222:3333:4444:5555:6666:7777:8888'
+}
+
+@test "$FILE API - all decisions" {
+    run -0 api "/v1/decisions"
+    run -0 jq -r '.[].value' <(output)
+    assert_output '1111:2222:3333:4444:5555:6666:7777:8888'
+}
+
+@test "$FILE CLI - decisions for ip 1111:2222:3333:4444:5555:6666:7777:8888" {
+    run -0 cscli decisions list -i '1111:2222:3333:4444:5555:6666:7777:8888' -o json
+    run -0 jq -r '.[].decisions[0].value' <(output)
+    assert_output '1111:2222:3333:4444:5555:6666:7777:8888'
+}
+
+@test "$FILE API - decisions for ip 1111:2222:3333:4444:5555:6666:7777:888" {
+    run -0 api '/v1/decisions?ip=1111:2222:3333:4444:5555:6666:7777:8888'
+    run -0 jq -r '.[].value' <(output)
+    assert_output '1111:2222:3333:4444:5555:6666:7777:8888'
+}
+
+@test "$FILE CLI - decisions for ip 1211:2222:3333:4444:5555:6666:7777:8888" {
+    run -0 cscli decisions list -i '1211:2222:3333:4444:5555:6666:7777:8888' -o json
+    assert_output 'null'
+}
+
+@test "$FILE API - decisions for ip 1211:2222:3333:4444:5555:6666:7777:888" {
+    run -0 api '/v1/decisions?ip=1211:2222:3333:4444:5555:6666:7777:8888'
+    assert_output 'null'
+}
+
+@test "$FILE CLI - decisions for ip 1111:2222:3333:4444:5555:6666:7777:8887" {
+    run -0 cscli decisions list -i '1111:2222:3333:4444:5555:6666:7777:8887' -o json
+    assert_output 'null'
+}
+
+@test "$FILE API - decisions for ip 1111:2222:3333:4444:5555:6666:7777:8887" {
+    run -0 api '/v1/decisions?ip=1111:2222:3333:4444:5555:6666:7777:8887'
+    assert_output 'null'
+}
+
+@test "$FILE CLI - decisions for range 1111:2222:3333:4444:5555:6666:7777:8888/48" {
+    run -0 cscli decisions list -r '1111:2222:3333:4444:5555:6666:7777:8888/48' -o json
+    assert_output 'null'
+}
+
+@test "$FILE API - decisions for range 1111:2222:3333:4444:5555:6666:7777:8888/48" {
+    run -0 api '/v1/decisions?range=1111:2222:3333:4444:5555:6666:7777:8888/48'
+    assert_output 'null'
+}
+
+@test "$FILE CLI - decisions for ip/range in 1111:2222:3333:4444:5555:6666:7777:8888/48" {
+    run -0 cscli decisions list -r '1111:2222:3333:4444:5555:6666:7777:8888/48' --contained -o json
+    run -0 jq -r '.[].decisions[0].value' <(output)
+    assert_output '1111:2222:3333:4444:5555:6666:7777:8888'
+}
+
+@test "$FILE API - decisions for ip/range in 1111:2222:3333:4444:5555:6666:7777:8888/48" {
+    run -0 api '/v1/decisions?range=1111:2222:3333:4444:5555:6666:7777:8888/48&&contains=false'
+    run -0 jq -r '.[].value' <(output)
+    assert_output '1111:2222:3333:4444:5555:6666:7777:8888'
+}
+
+@test "$FILE CLI - decisions for range 1111:2222:3333:4444:5555:6666:7777:8888/64" {
+    run -0 cscli decisions list -r '1111:2222:3333:4444:5555:6666:7777:8888/64' -o json
+    assert_output 'null'
+}
+
+@test "$FILE API - decisions for range 1111:2222:3333:4444:5555:6666:7777:8888/64" {
+    run -0 api '/v1/decisions?range=1111:2222:3333:4444:5555:6666:7777:8888/64'
+    assert_output 'null'
+}
+
+@test "$FILE CLI - decisions for ip/range in 1111:2222:3333:4444:5555:6666:7777:8888/64" {
+    run -0 cscli decisions list -r '1111:2222:3333:4444:5555:6666:7777:8888/64' -o json --contained
+    run -0 jq -r '.[].decisions[0].value' <(output)
+    assert_output '1111:2222:3333:4444:5555:6666:7777:8888'
+}
+
+@test "$FILE API - decisions for ip/range in 1111:2222:3333:4444:5555:6666:7777:8888/64" {
+    run -0 api '/v1/decisions?range=1111:2222:3333:4444:5555:6666:7777:8888/64&&contains=false'
+    run -0 jq -r '.[].value' <(output)
+    assert_output '1111:2222:3333:4444:5555:6666:7777:8888'
+}
+
+@test "$FILE adding decision for ip 1111:2222:3333:4444:5555:6666:7777:8889" {
+    run -0 cscli decisions add -i '1111:2222:3333:4444:5555:6666:7777:8889'
+    assert_output --partial 'Decision successfully added'
+}
+
+@test "$FILE deleting decision for ip 1111:2222:3333:4444:5555:6666:7777:8889" {
+    run -0 cscli decisions delete -i '1111:2222:3333:4444:5555:6666:7777:8889'
+    assert_output --partial '1 decision(s) deleted'
+}
+
+@test "$FILE CLI - decisions for ip 1111:2222:3333:4444:5555:6666:7777:8889 after delete" {
+    run -0 cscli decisions list -i '1111:2222:3333:4444:5555:6666:7777:8889' -o json
+    assert_output 'null'
+}
+
+@test "$FILE deleting decision for range 1111:2222:3333:4444:5555:6666:7777:8888/64" {
+    run -0 cscli decisions delete -r '1111:2222:3333:4444:5555:6666:7777:8888/64' --contained
+    assert_output --partial '1 decision(s) deleted'
+}
+
+@test "$FILE CLI - decisions for ip/range in 1111:2222:3333:4444:5555:6666:7777:8888/64 after delete" {
+    run -0 cscli decisions list -r '1111:2222:3333:4444:5555:6666:7777:8888/64' -o json --contained
+    assert_output 'null'
+}

tests/bats/98_ipv4_range.bats

@@ -0,0 +1,128 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh" >&3 2>&1
+    ./instance-data load
+    ./instance-crowdsec start
+    API_KEY=$(cscli bouncers add testbouncer -o raw)
+    export API_KEY
+    CROWDSEC_API_URL="http://localhost:8080"
+    export CROWDSEC_API_URL
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh" >&3 2>&1
+}
+
+setup() {
+    load "../lib/setup.sh"
+}
+
+api() {
+    URI="$1"
+    curl -s -H "X-Api-Key: ${API_KEY}" "${CROWDSEC_API_URL}${URI}"
+}
+
+#----------
+
+@test "$FILE adding decision for range 4.4.4.0/24" {
+    run -0 cscli decisions add -r '4.4.4.0/24'
+    assert_output --partial 'Decision successfully added'
+}
+
+@test "$FILE CLI - all decisions" {
+    run -0 cscli decisions list -o json
+    run -0 jq -r '.[0].decisions[0].value' <(output)
+    assert_output '4.4.4.0/24'
+#    run -0 jq -c '[ .[0].decisions[0].value, .[1].decisions[0].value ]' <(output)
+#    assert_output '["4.4.4.0/24","1.2.3.4"]'
+}
+
+@test "$FILE API - all decisions" {
+    run -0 api '/v1/decisions'
+    run -0 jq -r '.[0].value' <(output)
+    assert_output '4.4.4.0/24'
+}
+
+# check ip within/outside of range
+
+@test "$FILE CLI - decisions for ip 4.4.4." {
+    run -0 cscli decisions list -i '4.4.4.3' -o json
+    run -0 jq -r '.[0].decisions[0].value' <(output)
+    assert_output '4.4.4.0/24'
+}
+
+@test "$FILE API - decisions for ip 4.4.4." {
+    run -0 api '/v1/decisions?ip=4.4.4.3'
+    run -0 jq -r '.[0].value' <(output)
+    assert_output '4.4.4.0/24'
+}
+
+@test "$FILE CLI - decisions for ip contained in 4.4.4." {
+    run -0 cscli decisions list -i '4.4.4.4' -o json --contained
+    assert_output 'null'
+}
+
+@test "$FILE API - decisions for ip contained in 4.4.4." {
+    run -0 api '/v1/decisions?ip=4.4.4.4&contains=false'
+    assert_output 'null'
+}
+
+@test "$FILE CLI - decisions for ip 5.4.4." {
+    run -0 cscli decisions list -i '5.4.4.3' -o json
+    assert_output 'null'
+}
+
+@test "$FILE API - decisions for ip 5.4.4." {
+    run -0 api '/v1/decisions?ip=5.4.4.3'
+    assert_output 'null'
+}
+
+@test "$FILE CLI - decisions for range 4.4.0.0/1" {
+    run -0 cscli decisions list -r '4.4.0.0/16' -o json
+    assert_output 'null'
+}
+
+@test "$FILE API - decisions for range 4.4.0.0/1" {
+    run -0 api '/v1/decisions?range=4.4.0.0/16'
+    assert_output 'null'
+}
+
+@test "$FILE CLI - decisions for ip/range in 4.4.0.0/1" {
+    run -0 cscli decisions list -r '4.4.0.0/16' -o json --contained
+    run -0 jq -r '.[0].decisions[0].value' <(output)
+    assert_output '4.4.4.0/24'
+}
+
+@test "$FILE API - decisions for ip/range in 4.4.0.0/1" {
+    run -0 api '/v1/decisions?range=4.4.0.0/16&contains=false'
+    run -0 jq -r '.[0].value' <(output)
+    assert_output '4.4.4.0/24'
+}
+
+# check subrange
+
+@test "$FILE CLI - decisions for range 4.4.4.2/2" {
+    run -0 cscli decisions list -r '4.4.4.2/28' -o json
+    run -0 jq -r '.[].decisions[0].value' <(output)
+    assert_output '4.4.4.0/24'
+}
+
+@test "$FILE API - decisions for range 4.4.4.2/2" {
+    run -0 api '/v1/decisions?range=4.4.4.2/28'
+    run -0 jq -r '.[].value' <(output)
+    assert_output '4.4.4.0/24'
+}
+
+@test "$FILE CLI - decisions for range 4.4.3.2/2" {
+    run -0 cscli decisions list -r '4.4.3.2/28' -o json
+    assert_output 'null'
+}
+
+@test "$FILE API - decisions for range 4.4.3.2/2" {
+    run -0 api '/v1/decisions?range=4.4.3.2/28'
+    assert_output 'null'
+}

tests/bats/98_ipv6_range.bats

@@ -0,0 +1,209 @@
+#!/usr/bin/env bats
+# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
+
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh" >&3 2>&1
+    ./instance-data load
+    ./instance-crowdsec start
+    API_KEY=$(cscli bouncers add testbouncer -o raw)
+    export API_KEY
+    CROWDSEC_API_URL="http://localhost:8080"
+    export CROWDSEC_API_URL
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh" >&3 2>&1
+}
+
+setup() {
+    load "../lib/setup.sh"
+}
+
+#----------
+
+api() {
+    URI="$1"
+    curl -s -H "X-Api-Key: ${API_KEY}" "${CROWDSEC_API_URL}${URI}"
+}
+
+@test "$FILE adding decision for range aaaa:2222:3333:4444::/64" {
+    run -0 cscli decisions add -r 'aaaa:2222:3333:4444::/64'
+    assert_output --partial 'Decision successfully added'
+}
+
+@test "$FILE CLI - all decisions (2)" {
+    run -0 cscli decisions list -o json
+    run -0 jq -r '.[].decisions[0].value' <(output)
+    assert_output 'aaaa:2222:3333:4444::/64'
+}
+
+@test "$FILE API - all decisions (2)" {
+    run -0 api '/v1/decisions'
+    run -0 jq -r '.[].value' <(output)
+    assert_output 'aaaa:2222:3333:4444::/64'
+}
+
+# check ip within/out of range
+
+@test "$FILE CLI - decisions for ip aaaa:2222:3333:4444:5555:6666:7777:8888" {
+    run -0 cscli decisions list -i 'aaaa:2222:3333:4444:5555:6666:7777:8888' -o json
+    run -0 jq -r '.[].decisions[0].value' <(output)
+    assert_output 'aaaa:2222:3333:4444::/64'
+}
+
+@test "$FILE API - decisions for ip aaaa:2222:3333:4444:5555:6666:7777:8888" {
+    run -0 api '/v1/decisions?ip=aaaa:2222:3333:4444:5555:6666:7777:8888'
+    run -0 jq -r '.[].value' <(output)
+    assert_output 'aaaa:2222:3333:4444::/64'
+}
+
+@test "$FILE CLI - decisions for ip aaaa:2222:3333:4445:5555:6666:7777:8888" {
+    run -0 cscli decisions list -i 'aaaa:2222:3333:4445:5555:6666:7777:8888' -o json
+    assert_output 'null'
+}
+
+@test "$FILE API - decisions for ip aaaa:2222:3333:4445:5555:6666:7777:8888" {
+    run -0 api '/v1/decisions?ip=aaaa:2222:3333:4445:5555:6666:7777:8888'
+    assert_output 'null'
+}
+
+@test "$FILE CLI - decisions for ip aaa1:2222:3333:4444:5555:6666:7777:8887" {
+    run -0 cscli decisions list -i 'aaa1:2222:3333:4444:5555:6666:7777:8887' -o json
+    assert_output 'null'
+}
+
+@test "$FILE API - decisions for ip aaa1:2222:3333:4444:5555:6666:7777:8887" {
+    run -0 api '/v1/decisions?ip=aaa1:2222:3333:4444:5555:6666:7777:8887'
+    assert_output 'null'
+}
+
+# check subrange within/out of range
+
+@test "$FILE CLI - decisions for range aaaa:2222:3333:4444:5555::/80" {
+    run -0 cscli decisions list -r 'aaaa:2222:3333:4444:5555::/80' -o json
+    run -0 jq -r '.[].decisions[0].value' <(output)
+    assert_output 'aaaa:2222:3333:4444::/64'
+}
+
+@test "$FILE API - decisions for range aaaa:2222:3333:4444:5555::/80" {
+    run -0 api '/v1/decisions?range=aaaa:2222:3333:4444:5555::/80'
+    run -0 jq -r '.[].value' <(output)
+    assert_output 'aaaa:2222:3333:4444::/64'
+}
+
+@test "$FILE CLI - decisions for range aaaa:2222:3333:4441:5555::/80" {
+    run -0 cscli decisions list -r 'aaaa:2222:3333:4441:5555::/80' -o json
+    assert_output 'null'
+
+}
+
+@test "$FILE API - decisions for range aaaa:2222:3333:4441:5555::/80" {
+    run -0 api '/v1/decisions?range=aaaa:2222:3333:4441:5555::/80'
+    assert_output 'null'
+}
+
+@test "$FILE CLI - decisions for range aaa1:2222:3333:4444:5555::/80" {
+    run -0 cscli decisions list -r 'aaa1:2222:3333:4444:5555::/80' -o json
+    assert_output 'null'
+}
+
+@test "$FILE API - decisions for range aaa1:2222:3333:4444:5555::/80" {
+    run -0 api '/v1/decisions?range=aaa1:2222:3333:4444:5555::/80'
+    assert_output 'null'
+}
+
+# check outer range
+
+@test "$FILE CLI - decisions for range aaaa:2222:3333:4444:5555:6666:7777:8888/48" {
+    run -0 cscli decisions list -r 'aaaa:2222:3333:4444:5555:6666:7777:8888/48' -o json
+    assert_output 'null'
+}
+
+@test "$FILE API - decisions for range aaaa:2222:3333:4444:5555:6666:7777:8888/48" {
+    run -0 api '/v1/decisions?range=aaaa:2222:3333:4444:5555:6666:7777:8888/48'
+    assert_output 'null'
+}
+
+@test "$FILE CLI - decisions for ip/range in aaaa:2222:3333:4444:5555:6666:7777:8888/48" {
+    run -0 cscli decisions list -r 'aaaa:2222:3333:4444:5555:6666:7777:8888/48' -o json --contained
+    run -0 jq -r '.[].decisions[0].value' <(output)
+    assert_output 'aaaa:2222:3333:4444::/64'
+}
+
+@test "$FILE API - decisions for ip/range in aaaa:2222:3333:4444:5555:6666:7777:8888/48" {
+    run -0 api '/v1/decisions?range=aaaa:2222:3333:4444:5555:6666:7777:8888/48&contains=false'
+    run -0 jq -r '.[].value' <(output)
+    assert_output 'aaaa:2222:3333:4444::/64'
+}
+
+@test "$FILE CLI - decisions for ip/range in aaaa:2222:3333:4445:5555:6666:7777:8888/48" {
+    run -0 cscli decisions list -r 'aaaa:2222:3333:4445:5555:6666:7777:8888/48' -o json
+    assert_output 'null'
+}
+
+@test "$FILE API - decisions for ip/range in aaaa:2222:3333:4445:5555:6666:7777:8888/48" {
+    run -0 api '/v1/decisions?range=aaaa:2222:3333:4445:5555:6666:7777:8888/48'
+    assert_output 'null'
+}
+
+# bbbb:db8:: -> bbbb:db8:0000:0000:0000:7fff:ffff:ffff
+
+@test "$FILE adding decision for range bbbb:db8::/81" {
+    run -0 cscli decisions add -r 'bbbb:db8::/81'
+    assert_output --partial 'Decision successfully added'
+}
+
+@test "$FILE CLI - decisions for ip bbbb:db8:0000:0000:0000:6fff:ffff:ffff" {
+    run -0 cscli decisions list -o json -i 'bbbb:db8:0000:0000:0000:6fff:ffff:ffff'
+    run -0 jq -r '.[].decisions[0].value' <(output)
+    assert_output 'bbbb:db8::/81'
+}
+
+@test "$FILE API - decisions for ip in bbbb:db8:0000:0000:0000:6fff:ffff:ffff" {
+    run -0 api '/v1/decisions?ip=bbbb:db8:0000:0000:0000:6fff:ffff:ffff'
+    run -0 jq -r '.[].value' <(output)
+    assert_output 'bbbb:db8::/81'
+}
+
+@test "$FILE CLI - decisions for ip bbbb:db8:0000:0000:0000:8fff:ffff:ffff" {
+    run -0 cscli decisions list -o json -i 'bbbb:db8:0000:0000:0000:8fff:ffff:ffff'
+    assert_output 'null'
+}
+
+@test "$FILE API - decisions for ip in bbbb:db8:0000:0000:0000:8fff:ffff:ffff" {
+    run -0 api '/v1/decisions?ip=bbbb:db8:0000:0000:0000:8fff:ffff:ffff'
+    assert_output 'null'
+}
+
+@test "$FILE deleting decision for range aaaa:2222:3333:4444:5555:6666:7777:8888/48" {
+    run -0 cscli decisions delete -r 'aaaa:2222:3333:4444:5555:6666:7777:8888/48' --contained
+    assert_output --partial '1 decision(s) deleted'
+}
+
+@test "$FILE CLI - decisions for range aaaa:2222:3333:4444::/64 after delete" {
+    run -0 cscli decisions list -o json -r 'aaaa:2222:3333:4444::/64'
+    assert_output 'null'
+}
+
+@test "$FILE adding decision for ip bbbb:db8:0000:0000:0000:8fff:ffff:ffff" {
+    run -0 cscli decisions add -i 'bbbb:db8:0000:0000:0000:8fff:ffff:ffff'
+    assert_output --partial 'Decision successfully added'
+}
+
+@test "$FILE adding decision for ip bbbb:db8:0000:0000:0000:6fff:ffff:ffff" {
+    run -0 cscli decisions add -i 'bbbb:db8:0000:0000:0000:6fff:ffff:ffff'
+    assert_output --partial 'Decision successfully added'
+}
+
+@test "$FILE deleting decisions for range bbbb:db8::/81" {
+    run -0 cscli decisions delete -r 'bbbb:db8::/81' --contained
+    assert_output --partial '2 decision(s) deleted'
+}
+
+@test "$FILE CLI - all decisions (3)" {
+    run -0 cscli decisions list -o json
+    run -0 jq -r '.[].decisions[0].value' <(output)
+    assert_output 'bbbb:db8:0000:0000:0000:8fff:ffff:ffff'
+}

tests/collect-hub-coverage

@@ -0,0 +1,30 @@
+#!/usr/bin/env bash
+
+set -eu
+
+die() {
+    echo >&2 "$@"
+    exit 1
+}
+
+# shellcheck disable=SC1007
+TEST_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+# shellcheck source=./.environment.sh
+. "${TEST_DIR}/.environment.sh"
+
+hubdir="${LOCAL_DIR}/hub-tests"
+
+coverage() {
+    "${BIN_DIR}/cscli" --crowdsec "${BIN_DIR}/crowdsec" --cscli "${BIN_DIR}/cscli" hubtest coverage --"$1" --percent
+}
+
+cd "$hubdir" || die "Could not find hub test results"
+
+echo "PARSERS_COV=$(coverage parsers | cut -d = -f2)"
+echo "SCENARIOS_COV=$(coverage scenarios | cut -d = -f2)"
+
+PARSERS_COV_NUMBER=$(coverage parsers | tr -d '%[[:space:]]')
+SCENARIOS_COV_NUMBER=$(coverage scenarios | tr -d '%[[:space:]]')
+
+echo "PARSERS_BADGE_COLOR=$(if [[ PARSERS_COV_NUMBER -lt 70 ]]; then echo 'red'; else echo 'green'; fi)"
+echo "SCENARIOS_BADGE_COLOR=$(if [[ SCENARIOS_COV_NUMBER -lt 70 ]]; then echo 'red'; else echo 'green'; fi)"

tests/config-templates/acquis.yaml

@@ -0,0 +1,16 @@
+filenames:
+  - /var/log/nginx/*.log
+  - ./tests/nginx/nginx.log
+#this is not a syslog log, indicate which kind of logs it is
+labels:
+  type: nginx
+---
+filenames:
+ - /var/log/auth.log
+ - /var/log/syslog
+labels:
+  type: syslog
+---
+filename: /var/log/apache2/*.log
+labels:
+  type: apache2

tests/config-templates/config.yaml

@@ -0,0 +1,54 @@
+common:
+  daemonize: false
+  # pid_dir: /var/run/
+  log_media: file
+  log_level: info
+  log_dir: ${LOG_DIR}
+  working_dir: .
+config_paths:
+  config_dir: ${CONFIG_DIR}
+  data_dir: ${DATA_DIR}
+  simulation_path: ${CONFIG_DIR}/simulation.yaml
+  hub_dir: ${CONFIG_DIR}/hub/
+  index_path: ${CONFIG_DIR}/hub/.index.json
+  notification_dir: ${CONFIG_DIR}/notifications/
+  plugin_dir: ${PLUGIN_DIR}
+crowdsec_service:
+  acquisition_path: ${CONFIG_DIR}/acquis.yaml
+  parser_routines: 1
+cscli:
+  output: human
+db_config:
+  log_level: info
+  type: sqlite
+  db_path: ${DATA_DIR}/crowdsec.db
+  #user: 
+  #password:
+  #db_name:
+  #host:
+  #port:
+  flush:
+    max_items: 5000
+    max_age: 7d
+plugin_config:
+  user: nobody # plugin process would be ran on behalf of this user
+  group: nogroup # plugin process would be ran on behalf of this group
+api:
+  client:
+    insecure_skip_verify: false
+    credentials_path: ${CONFIG_DIR}/local_api_credentials.yaml
+  server:
+    log_level: info
+    listen_uri: 127.0.0.1:8080
+    profiles_path: ${CONFIG_DIR}/profiles.yaml
+    console_path: ${CONFIG_DIR}/console.yaml
+    online_client: # Central API credentials (to push signals and receive bad IPs)
+      credentials_path: ${CONFIG_DIR}/online_api_credentials.yaml
+#    tls:
+#      cert_file: ${CONFIG_DIR}/ssl/cert.pem
+#      key_file: ${CONFIG_DIR}/ssl/key.pem
+prometheus:
+  enabled: true
+  level: full
+  listen_addr: 127.0.0.1
+  listen_port: 6060

tests/config-templates/local_api_credentials.yaml

@@ -0,0 +1 @@
+url: http://127.0.0.1:8080

tests/config-templates/notifications/email.yaml

@@ -0,0 +1,38 @@
+type: email           # Don't change
+name: email_default   # Must match the registered plugin in the profile
+
+# One of "trace", "debug", "info", "warn", "error", "off"
+log_level: info
+
+# group_wait:         # Time to wait collecting alerts before relaying a message to this plugin, eg "30s"
+# group_threshold:    # Amount of alerts that triggers a message before <group_wait> has expired, eg "10"
+# max_retry:          # Number of attempts to relay messages to plugins in case of error
+timeout: 20s          # Time to wait for response from the plugin before considering the attempt a failure, eg "10s"
+
+#-------------------------
+# plugin-specific options
+
+# The following template receives a list of models.Alert objects
+# The output goes in the email message body
+format: |
+  {{range . -}}
+    {{$alert := . -}}
+    {{range .Decisions -}}
+      <a href=https://www.whois.com/whois/{{.Value}}>{{.Value}}</a> will get <b>{{.Type}}</b> for next <b>{{.Duration}}</b> for triggering <b>{{.Scenario}}</b> on machine <b>{{$alert.MachineID}}</b>. <a href=https://www.shodan.io/host/{{.Value}}>Shodan</a>
+    {{end -}}
+  {{end -}}
+
+smtp_host:            # example: smtp.gmail.com
+smtp_username:        # Replace with your actual username
+smtp_password:        # Replace with your actual password
+smtp_port:            # Common values are any of [25, 465, 587, 2525]
+auth_type:            # Valid choices are "none", "crammd5", "login", "plain"
+sender_email:         # example: foo@gmail.com
+email_subject: "CrowdSec Notification"
+receiver_emails:
+# - email1@gmail.com
+# - email2@gmail.com
+
+# One of "ssltls", "none"
+encryption_type: ssltls
+

tests/config-templates/notifications/http.yaml

@@ -0,0 +1,30 @@
+type: http          # Don't change
+name: http_default  # Must match the registered plugin in the profile
+
+# One of "trace", "debug", "info", "warn", "error", "off"
+log_level: info
+
+# group_wait:         # Time to wait collecting alerts before relaying a message to this plugin, eg "30s"
+# group_threshold:    # Amount of alerts that triggers a message before <group_wait> has expired, eg "10"
+# max_retry:          # Number of attempts to relay messages to plugins in case of error
+# timeout:            # Time to wait for response from the plugin before considering the attempt a failure, eg "10s"
+
+#-------------------------
+# plugin-specific options
+
+# The following template receives a list of models.Alert objects
+# The output goes in the http request body
+format: |
+  {{.|toJson}}
+
+# The plugin will make requests to this url, eg:  https://www.example.com/
+url: <HTTP_url>
+
+# Any of the http verbs: "POST", "GET", "PUT"...
+method: POST
+
+# headers:
+#   Authorization: token 0x64312313
+
+# skip_tls_verification:  # true or false. Default is false
+

tests/config-templates/notifications/slack.yaml

@@ -0,0 +1,30 @@
+type: slack           # Don't change
+name: slack_default   # Must match the registered plugin in the profile
+
+# One of "trace", "debug", "info", "warn", "error", "off"
+log_level: info
+
+# group_wait:         # Time to wait collecting alerts before relaying a message to this plugin, eg "30s"
+# group_threshold:    # Amount of alerts that triggers a message before <group_wait> has expired, eg "10"
+# max_retry:          # Number of attempts to relay messages to plugins in case of error
+# timeout:            # Time to wait for response from the plugin before considering the attempt a failure, eg "10s"
+
+#-------------------------
+# plugin-specific options
+
+# The following template receives a list of models.Alert objects
+# The output goes in the slack message
+format: |
+  {{range . -}}
+  {{$alert := . -}}
+  {{range .Decisions -}}
+  {{if $alert.Source.Cn -}}
+  :flag-{{$alert.Source.Cn}}: <https://www.whois.com/whois/{{.Value}}|{{.Value}}> will get {{.Type}} for next {{.Duration}} for triggering {{.Scenario}} on machine '{{$alert.MachineID}}'. <https://www.shodan.io/host/{{.Value}}|Shodan>{{end}}
+  {{if not $alert.Source.Cn -}}
+  :pirate_flag: <https://www.whois.com/whois/{{.Value}}|{{.Value}}> will get {{.Type}} for next {{.Duration}} for triggering {{.Scenario}} on machine '{{$alert.MachineID}}'.  <https://www.shodan.io/host/{{.Value}}|Shodan>{{end}}
+  {{end -}}
+  {{end -}}
+
+
+webhook: <WEBHOOK_URL>
+

tests/config-templates/notifications/splunk.yaml

@@ -0,0 +1,21 @@
+type: splunk          # Don't change
+name: splunk_default  # Must match the registered plugin in the profile
+
+# One of "trace", "debug", "info", "warn", "error", "off"
+log_level: info
+
+# group_wait:         # Time to wait collecting alerts before relaying a message to this plugin, eg "30s"
+# group_threshold:    # Amount of alerts that triggers a message before <group_wait> has expired, eg "10"
+# max_retry:          # Number of attempts to relay messages to plugins in case of error
+# timeout:            # Time to wait for response from the plugin before considering the attempt a failure, eg "10s"
+
+#-------------------------
+# plugin-specific options
+
+# The following template receives a list of models.Alert objects
+# The output goes in the splunk notification
+format: |
+  {{.|toJson}}
+
+url: <SPLUNK_HTTP_URL>
+token: <SPLUNK_TOKEN>

scripts/func_tests/config/profiles.yaml → tests/config-templates/profiles.yaml

@@ -1,12 +1,13 @@
 name: default_ip_remediation
 #debug: true
 filters:
- - 1==1
+ - Alert.Remediation == true && Alert.GetScope() == "Ip"
 decisions:
  - type: ban
    duration: 4h
-notifications:
+# notifications:
 #   - slack_default  # Set the webhook in /etc/crowdsec/notifications/slack.yaml before enabling this.
 #   - splunk_default # Set the splunk url and token in /etc/crowdsec/notifications/splunk.yaml  before enabling this.
-  - http_default # Set the required http parameters in  /etc/crowdsec/notifications/http.yaml before enabling this.
+#   - http_default # Set the required http parameters in  /etc/crowdsec/notifications/http.yaml before enabling this.
+#   - email_default # Set the required http parameters in  /etc/crowdsec/notifications/email.yaml before enabling this.
 on_success: break

tests/config-templates/simulation.yaml

@@ -0,0 +1,4 @@
+simulation: off
+# exclusions:
+#  - crowdsecurity/ssh-bf
+ 

tests/dyn-bats/README.md

@@ -0,0 +1,2 @@
+This directory is for dynamically generated tests. Do not commit them.
+Any `*.bats` file here will be removed by the Makefile.

tests/generate-hub-tests

@@ -0,0 +1,53 @@
+#!/bin/sh
+
+set -eu
+
+# shellcheck disable=SC1007
+TEST_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+# shellcheck source=./.environment.sh
+. "${TEST_DIR}/.environment.sh"
+
+CSCLI="${BIN_DIR}/cscli"
+cscli() {
+    "${BIN_DIR}/cscli" "$@"
+}
+
+CROWDSEC="${BIN_DIR}/crowdsec"
+
+"${TEST_DIR}/instance-data" load
+
+hubdir="${LOCAL_DIR}/hub-tests"
+git clone --depth 1 https://github.com/crowdsecurity/hub.git "${hubdir}" >/dev/null 2>&1 || (cd "${hubdir}"; git pull)
+
+HUBTESTS_BATS="${TEST_DIR}/dyn-bats/99_hub.bats"
+
+cat << EOT > "${HUBTESTS_BATS}"
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh" >&3 2>&1
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh" >&3 2>&1
+}
+
+setup() {
+    load "../lib/setup.sh"
+}
+
+EOT
+
+echo "Generating hub tests..."
+
+for testname in $("${CSCLI}" --crowdsec "${CROWDSEC}" --cscli "${CSCLI}" hubtest --hub "${hubdir}" list -o json | grep -v NAME | grep -v -- '-------' | awk '{print $1}'); do
+    cat << EOT >> "${HUBTESTS_BATS}"
+
+@test "\$FILE $testname" {
+    run "\${CSCLI}" --crowdsec "\${CROWDSEC}" --cscli "\${CSCLI}" --hub "${hubdir}" hubtest run "${testname}" --clean
+    # in case of error, need to see what went wrong
+    echo "\$output"
+    assert_success
+}
+EOT
+done

tests/instance-crowdsec

@@ -0,0 +1,86 @@
+#!/usr/bin/env bash
+
+set -eu
+
+die() {
+    echo >&2 "$@"
+    exit 1
+}
+
+about() {
+    die "usage: $0 [ start | stop ]"
+}
+
+#shellcheck disable=SC1007
+THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+#shellcheck disable=SC1090
+. "${THIS_DIR}/.environment.sh"
+
+# you have not removed set -u above, have you?
+
+[ -z "${BIN_DIR-}" ] && die "\$BIN_DIR must be defined."
+[ -z "${LOG_DIR-}" ] && die "\$LOG_DIR must be defined."
+[ -z "${PID_DIR-}" ] && die "\$PID_DIR must be defined."
+
+if [ ! -e "${BIN_DIR}/crowdsec" ]; then
+    die "${BIN_DIR}/crowdsec is missing. Please run 'make bats-build' to create it."
+fi
+
+wait_for_port() {
+    for _ in $(seq 40); do
+        nc -z localhost "$1" && return
+        sleep .05
+    done
+
+    # send to &3 if open
+    if { true >&3; } 2>/dev/null; then
+        # cat "${LOG_DIR}/crowdsec.out" >&3
+        # cat "${LOG_DIR}/crowdsec.log" >&3
+        echo "Can't connect to port $1" >&3
+    else
+        # cat "${LOG_DIR}/crowdsec.out" >&2
+        # cat "${LOG_DIR}/crowdsec.log" >&2
+        echo "Can't connect to port $1" >&2
+    fi
+
+    return 1
+}
+
+DAEMON_PID=${PID_DIR}/crowdsec.pid
+
+start_instance() {
+    OUT_FILE="${LOG_DIR}/crowdsec.out" \
+        DAEMON_PID="${DAEMON_PID}" \
+        "${TEST_DIR}/run-as-daemon" "${BIN_DIR}/crowdsec"
+    wait_for_port 6060
+}
+
+stop_instance() {
+    if [ -f "${DAEMON_PID}" ]; then
+       # terminate quickly with extreme prejudice, all the application data will be
+       # thrown away anyway. also terminate the child processes (notification plugin).
+       PGID="$(ps --no-headers -p "$(cat "${DAEMON_PID}")" -o pgid | tr -d ' ')"
+       if [ -n "${PGID}" ]; then
+           kill -- "-${PGID}"
+       fi
+       rm -f -- "${DAEMON_PID}"
+    fi
+}
+
+
+# ---------------------------
+
+[ $# -lt 1 ] && about
+
+case "$1" in
+    start)
+        start_instance
+        ;;
+    stop)
+        stop_instance
+        ;;
+    *)
+        about
+        ;;
+esac;
+

tests/instance-data

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+
+set -eu
+script_name=$0
+
+die() {
+    echo >&2 "$@"
+    exit 1
+}
+
+about() {
+    die "usage: $0 [make | load | clean]"
+}
+
+#shellcheck disable=SC1007
+THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+cd "${THIS_DIR}"
+#shellcheck disable=SC1090
+. ./.environment.sh
+
+# you have not removed set -u above, have you?
+
+[ -z "${TEST_DIR-}" ] && die "\$TEST_DIR must be defined."
+[ -z "${LOCAL_DIR-}" ] && die "\$LOCAL_DIR must be defined."
+[ -z "${BIN_DIR-}" ] && die "\$BIN_DIR must be defined."
+[ -z "${CONFIG_DIR-}" ] && die "\$CONFIG_DIR must be defined."
+[ -z "${DATA_DIR-}" ] && die "\$DATA_DIR must be defined."
+[ -z "${LOCAL_INIT_DIR-}" ] && die "\$LOCAL_INIT_DIR must be defined."
+[ -z "${PLUGIN_DIR-}" ] && die "\$PLUGIN_DIR must be defined."
+
+if [ ! -e "${BIN_DIR}/cscli" ]; then
+    die "${BIN_DIR}/cscli is missing. Please run 'make bats-build' to create it."
+fi
+
+# fails if config_dir or data_dir are not subpaths of local_dir
+REL_CONFIG_DIR=${CONFIG_DIR#$LOCAL_DIR}
+REL_CONFIG_DIR=${REL_CONFIG_DIR#/}
+REL_DATA_DIR=${DATA_DIR#$LOCAL_DIR}
+REL_DATA_DIR=${REL_DATA_DIR#/}
+
+remove_init_data() {
+    rm -rf -- "${CONFIG_DIR:?}"/* "${DATA_DIR:?}"/*
+}
+
+make_init_data() {
+    remove_init_data
+
+    mkdir -p "${CONFIG_DIR}/notifications"
+
+    envsubst < "./config-templates/acquis.yaml" > "${CONFIG_DIR}/acquis.yaml"
+    envsubst < "./config-templates/config.yaml" > "${CONFIG_DIR}/config.yaml"
+    envsubst < "./config-templates/simulation.yaml" > "${CONFIG_DIR}/simulation.yaml"
+    envsubst < "./config-templates/local_api_credentials.yaml" > "${CONFIG_DIR}/local_api_credentials.yaml"
+    envsubst < "./config-templates/online_api_credentials.yaml" > "${CONFIG_DIR}/online_api_credentials.yaml"
+    envsubst < "./config-templates/profiles.yaml" > "${CONFIG_DIR}/profiles.yaml"
+    envsubst < "./config-templates/notifications/http.yaml" > "${CONFIG_DIR}/notifications/http.yaml"
+    envsubst < "./config-templates/notifications/email.yaml" > "${CONFIG_DIR}/notifications/email.yaml"
+    envsubst < "./config-templates/notifications/slack.yaml" > "${CONFIG_DIR}/notifications/slack.yaml"
+    envsubst < "./config-templates/notifications/splunk.yaml" > "${CONFIG_DIR}/notifications/splunk.yaml"
+
+    mkdir -p "${CONFIG_DIR}/hub"
+    "${BIN_DIR}/cscli" machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto
+    "${BIN_DIR}/cscli" capi register
+    "${BIN_DIR}/cscli" hub update
+    "${BIN_DIR}/cscli" collections install crowdsecurity/linux
+    mkdir -p "${CONFIG_DIR}/patterns"
+    cp -ax "../config/patterns" "${CONFIG_DIR}/"
+
+    "${TEST_DIR}/instance-crowdsec" start
+    "${BIN_DIR}/cscli" lapi status
+    "${TEST_DIR}/instance-crowdsec" stop
+
+    tar -C "${LOCAL_DIR}" --create --file "${LOCAL_INIT_DIR}/init-config-data.tar" "$REL_CONFIG_DIR" "$REL_DATA_DIR"
+
+    remove_init_data
+}
+
+load_init_data() {
+    if [ ! -f "${LOCAL_INIT_DIR}/init-config-data.tar" ]; then
+        die "Initial data not found; did you run '$script_name make' ?"
+    fi
+
+    remove_init_data
+
+    tar -C "${LOCAL_DIR}" --extract --file "${LOCAL_INIT_DIR}/init-config-data.tar"
+}
+
+
+# ---------------------------
+
+[ $# -lt 1 ] && about
+
+./assert-crowdsec-not-running
+
+case "$1" in
+    make)
+        make_init_data
+        ;;
+    load)
+        load_init_data
+        ;;
+    clean)
+        remove_init_data
+        ;;
+    *)
+        about
+        ;;
+esac;
+

tests/instance-mock-http

@@ -0,0 +1,82 @@
+#!/bin/sh
+
+set -eu
+
+die() {
+    echo >&2 "$@"
+    exit 1
+}
+
+about() {
+    die "usage: $0 [ start <port> | stop ]"
+}
+
+#shellcheck disable=SC1007
+THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+#shellcheck disable=SC1090
+. "${THIS_DIR}/.environment.sh"
+
+# you have not removed set -u above, have you?
+
+[ -z "${LOG_DIR-}" ] && die "\$LOG_DIR must be defined."
+[ -z "${PID_DIR-}" ] && die "\$PID_DIR must be defined."
+
+if ! command -v python3 >/dev/null 2>&2; then
+    die "The python3 executable is is missing. Please install it and try again."
+fi
+
+wait_for_port() {
+    for _ in $(seq 40); do
+        nc -z localhost "$1" && return
+        sleep .05
+    done
+
+    # send to &3 if open
+    if { true >&3; } 2>/dev/null; then
+        # cat "${LOG_DIR}/mock-http.out" >&3
+        echo "Can't connect to port $1" >&3
+    else
+        # cat "${LOG_DIR}/mock-http.out" >&2
+        echo "Can't connect to port $1" >&2
+    fi
+
+    return 1
+}
+
+DAEMON_PID=${PID_DIR}/mock-http.pid
+
+start_instance() {
+    [ $# -lt 1 ] && about
+    OUT_FILE="${LOG_DIR}/mock-http.out" \
+        DAEMON_PID="${DAEMON_PID}" \
+        "${TEST_DIR}/run-as-daemon" /usr/bin/env python3 -u "${THIS_DIR}/mock-http.py" "$1"
+    wait_for_port "$1"
+    echo "mock http started on port $1"
+}
+
+stop_instance() {
+    if [ -f "${DAEMON_PID}" ]; then
+        # terminate with extreme prejudice, all the application data will be thrown away anyway
+        kill -9 "$(cat "${DAEMON_PID}")" > /dev/null 2>&1
+        rm -f -- "${DAEMON_PID}"
+    fi
+}
+
+
+# ---------------------------
+
+[ $# -lt 1 ] && about
+
+case "$1" in
+    start)
+        shift
+        start_instance "$@"
+        ;;
+    stop)
+        stop_instance
+        ;;
+    *)
+        about
+        ;;
+esac;
+

tests/lib/bats-assert

@@ -0,0 +1 @@
+Subproject commit 4bdd58d3fbcdce3209033d44d884e87add1d8405

tests/lib/bats-core

@@ -0,0 +1 @@
+Subproject commit 210acf3a8ed318ddedad3137c15451739beba7d4

tests/lib/bats-file

@@ -0,0 +1 @@
+Subproject commit 17fa557f6fe28a327933e3fa32efef1d211caa5a

tests/lib/bats-support

@@ -0,0 +1 @@
+Subproject commit d140a65044b2d6810381935ae7f0c94c7023c8c3

tests/lib/setup.sh

@@ -0,0 +1,6 @@
+
+# these plugins are always available
+
+load "../lib/bats-support/load.bash"
+load "../lib/bats-assert/load.bash"
+#load "../lib/bats-file/load.bash"

tests/lib/setup_file.sh

@@ -0,0 +1,45 @@
+
+# Allow tests to use relative paths for helper scripts.
+# Must redirect output to &3 otherwise errors in setup_file, teardown_file go unreported
+
+# shellcheck disable=SC2164
+cd "${TEST_DIR}" >&3 2>&1
+
+# complain if there's a crowdsec running system-wide or leftover from a previous test
+./assert-crowdsec-not-running
+
+# we can use the filename in test descriptions
+FILE="$(basename "${BATS_TEST_FILENAME}" .bats):"
+export FILE
+
+# the variables exported here can be seen in other setup/teardown/test functions
+CROWDSEC="${BIN_DIR}/crowdsec"
+export CROWDSEC
+CSCLI="${BIN_DIR}/cscli"
+export CSCLI
+
+# functions too
+cscli() {
+    "${CSCLI}" "$@"
+}
+export -f cscli
+
+# We use these functions like this:
+#    somecommand <(stderr)
+# to provide a standard input to "somecommand".
+# The alternatives echo "$stderr" or <<<"$stderr"
+# ("here string" in bash jargon)
+# are worse because they add a newline,
+# even if the variable is empty.
+
+# shellcheck disable=SC2154
+stderr() {
+    printf '%s' "$stderr"
+}
+export -f stderr
+
+# shellcheck disable=SC2154
+output() {
+    printf '%s' "$output"
+}
+export -f output

tests/lib/teardown_file.sh

@@ -0,0 +1,4 @@
+
+# ensure we don't leave crowdsec running if tests are broken or interrupted
+./instance-crowdsec stop
+

scripts/func_tests/mock_http_server.py → tests/mock-http.py

@@ -1,4 +1,9 @@
+#!/usr/bin/env python3
+
 import json
+import logging
+import sys
+
 from http.server import HTTPServer, BaseHTTPRequestHandler
 
 class RequestHandler(BaseHTTPRequestHandler):
@@ -8,7 +13,7 @@ class RequestHandler(BaseHTTPRequestHandler):
         request_body = json.loads(request_body.decode())
         log = {
             "path": request_path,
-            "status": 200, 
+            "status": 200,
             "request_body": request_body,
         }
         print(json.dumps(log))
@@ -17,10 +22,25 @@ class RequestHandler(BaseHTTPRequestHandler):
         self.end_headers()
         self.wfile.write(json.dumps({}).encode())
         return
-    
+
     def log_message(self, format, *args):
         return
 
+def main(argv):
+    try:
+        port = int(argv[1])
+    except IndexError:
+        logging.fatal("Missing port number")
+        return 1
+    except ValueError:
+        logging.fatal("Invalid port number '%s'", argv[1])
+        return 1
+    server = HTTPServer(('', port), RequestHandler)
+    # logging.info('Listening on port %s', port)
+    server.serve_forever()
+    return 0
+
+
 if __name__ == "__main__" :
-    server = HTTPServer(('', 9999), RequestHandler)
-    server.serve_forever()
+    logging.basicConfig(level=logging.INFO)
+    sys.exit(main(sys.argv))

tests/run-as-daemon

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+
+SYSTEM=$(uname -s)
+
+die() {
+    echo >&2 "$@"
+    exit 1
+}
+
+# Simplified dudeist daemonizer. Don't care about lock files, separate
+# stdout/stderr and fancy stuff. #YOLO
+
+case "${SYSTEM,,}" in
+    linux)
+        daemonize -p "${DAEMON_PID}" -e "${OUT_FILE}" -o "${OUT_FILE}" "$@"
+        ;;
+    freebsd)
+        daemon -p "${DAEMON_PID}" -o "${OUT_FILE}" "$@"
+        ;;
+    *)
+        die "unsupported system: $SYSTEM"
+        ;;
+esac
+

tests/run-tests

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+set -eu
+
+die() {
+    echo >&2 "$@"
+    exit 1
+}
+
+# shellcheck disable=SC1007
+TEST_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+# shellcheck source=./.environment.sh
+. "${TEST_DIR}/.environment.sh"
+
+
+
+check_requirements() {
+    if ! command -v nc >/dev/null; then
+        die "missing required program 'nc' (package 'netcat-openbsd')"
+    fi
+
+    if ! command -v yq >/dev/null; then
+        die "missing required program 'yq'. You can install it with 'GO111MODULE=on go get github.com/mikefarah/yq/v4'"
+    fi
+
+    SYSTEM=$(uname -s)
+    case "${SYSTEM,,}" in
+        linux)
+            if ! command -v daemonize >/dev/null; then
+                die "missing required program 'daemonize' (package 'daemonize')"
+            fi
+            ;;
+        freebsd)
+            if ! command -v daemon >/dev/null; then
+                die "missing required program 'daemon'"
+            fi
+            ;;
+        *)
+            die "unsupported system: $SYSTEM"
+            ;;
+    esac
+}
+
+
+check_requirements
+
+if [ $# -ge 1 ]; then
+    "${TEST_DIR}/lib/bats-core/bin/bats" \
+        --jobs 1 \
+        --print-output-on-failure \
+        -T "$@"
+else
+    "${TEST_DIR}/lib/bats-core/bin/bats" \
+        --jobs 1 \
+        --print-output-on-failure \
+        -T "${TEST_DIR}/bats" "${TEST_DIR}/dyn-bats"
+fi