Update bats tests to work with systemd (#1394)

* update for making it work with systemd
* take DB_BACKEND from file name; reduce duplicated code; configure db_type=sqlite
* define PACKAGE_TESTING

Co-authored-by: sabban <15465465+sabban@users.noreply.github.com>
Co-authored-by: mmetc <92726601+mmetc@users.noreply.github.com>

tests/bats.mk

@@ -2,18 +2,30 @@
 # contains scripts, bats submodules, local instances and functional test suite
 TEST_DIR = $(CURDIR)/tests
 
-# contains a local instance of crowdsec, complete with configuration and data
-LOCAL_DIR = $(TEST_DIR)/local
+ifdef PACKAGE_TESTING
+  # define PACKAGE_TESTING to test the executables already installed with
+  # *.deb, *.rpm...
+  LOCAL_DIR = /
+  BIN_DIR = /usr/bin
+  INIT_BACKEND = systemd
+  CONFIG_BACKEND = global
+else
+  # LOCAL_DIR will contain contains a local instance of crowdsec, complete with
+  # configuration and data
+  LOCAL_DIR = $(TEST_DIR)/local
+  BIN_DIR = $(LOCAL_DIR)/bin
+  INIT_BACKEND = daemon
+  CONFIG_BACKEND = local
+  PACKAGE_TESTING =
+endif
 
-BIN_DIR = $(LOCAL_DIR)/bin
 CONFIG_DIR = $(LOCAL_DIR)/etc/crowdsec
 DATA_DIR = $(LOCAL_DIR)/var/lib/crowdsec/data
-
 LOCAL_INIT_DIR = $(TEST_DIR)/local-init
 LOG_DIR = $(LOCAL_DIR)/var/log
 PID_DIR = $(LOCAL_DIR)/var/run
 PLUGIN_DIR = $(LOCAL_DIR)/lib/crowdsec/plugins
-DB_BACKEND ?= "sqlite"
+DB_BACKEND ?= sqlite
 
 define ENV :=
 export TEST_DIR="$(TEST_DIR)"
@@ -26,6 +38,9 @@ export LOG_DIR="$(LOG_DIR)"
 export PID_DIR="$(PID_DIR)"
 export PLUGIN_DIR="$(PLUGIN_DIR)"
 export DB_BACKEND="$(DB_BACKEND)"
+export INIT_BACKEND="$(INIT_BACKEND)"
+export CONFIG_BACKEND="$(CONFIG_BACKEND)"
+export PACKAGE_TESTING="$(PACKAGE_TESTING)"
 endef
 
 bats-all: bats-clean bats-build bats-test bats-test-hub
@@ -66,3 +81,8 @@ bats-lint:
 	@shellcheck --version >/dev/null 2>&1 || (echo "ERROR: shellcheck is required."; exit 1)
 	@shellcheck -x $(TEST_DIR)/bats/*.bats
 
+
+bats-test-package: bats-environment
+	$(TEST_DIR)/instance-data make
+	$(TEST_DIR)/run-tests $(TEST_DIR)/bats
+	$(TEST_DIR)/run-tests $(TEST_DIR)/dyn-bats

tests/bats/30_machines.bats

@@ -29,7 +29,7 @@ teardown() {
 
 @test "$FILE we have exactly one machine, localhost" {
     run -0 cscli machines list -o json
-    run -0 jq -c '[. | length, .[0].machineId, .[0].isValidated, .[0].ipAddress]' <(output)
+    run -0 jq -c '[. | length, .[0].machineId[0:32], .[0].isValidated, .[0].ipAddress]' <(output)
     assert_output '[1,"githubciXXXXXXXXXXXXXXXXXXXXXXXX",true,"127.0.0.1"]'
     # the machine gets an IP address when it talks to the LAPI
     # XXX already done in instance-data make

tests/bats/71_dummy_plugin.bats

@@ -5,27 +5,28 @@ set -u
 
 setup_file() {
     load "../lib/setup_file.sh"
-    eval "$(debug)"
+    [ -n "${PACKAGE_TESTING}" ] && return
+
     ./instance-data load
 
     tempfile=$(TMPDIR="${BATS_FILE_TMPDIR}" mktemp)
     export tempfile
 
     yq '
-        .group_wait="5s" |
-        .group_threshold=2 |
-        .output_file=strenv(tempfile)
-    ' -i "$(config_yq '.config_paths.notification_dir')/dummy.yaml"
+       .group_wait="5s" |
+       .group_threshold=2 |
+       .output_file=strenv(tempfile)
+       ' -i "$(config_yq '.config_paths.notification_dir')/dummy.yaml"
 
     yq '
-        .notifications=["dummy_default"] |
-        .filters=["Alert.GetScope() == \"Ip\""]
-    ' -i "$(config_yq '.api.server.profiles_path')"
+       .notifications=["dummy_default"] |
+       .filters=["Alert.GetScope() == \"Ip\""]
+       ' -i "$(config_yq '.api.server.profiles_path')"
 
     yq '
-        .plugin_config.user="" |
-        .plugin_config.group=""
-    ' -i "${CONFIG_YAML}"
+       .plugin_config.user="" |
+       .plugin_config.group=""
+       ' -i "${CONFIG_YAML}"
 
     ./instance-crowdsec start
 }
@@ -35,6 +36,7 @@ teardown_file() {
 }
 
 setup() {
+    [ -n "${PACKAGE_TESTING}" ] && skip
     load "../lib/setup.sh"
 }
 

tests/bats/80_alerts.bats

@@ -29,16 +29,16 @@ teardown() {
     run -0 cscli alerts list
     refute_output --partial 'MACHINE'
     # machine name appears quoted in the "REASON" column
-    assert_output --partial "| 'githubciXXXXXXXXXXXXXXXXXXXXXXXX' |"
-    refute_output --partial "| githubciXXXXXXXXXXXXXXXXXXXXXXXX |"
+    assert_output --regexp "\| 'githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})?' \|"
+    refute_output --regexp "\| githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})? \|"
 
     run -0 cscli alerts list -m
     assert_output --partial 'MACHINE'
-    assert_output --partial "| 'githubciXXXXXXXXXXXXXXXXXXXXXXXX' |"
-    assert_output --partial "| githubciXXXXXXXXXXXXXXXXXXXXXXXX |"
+    assert_output --regexp "\| 'githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})?' \|"
+    assert_output --regexp "\| githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})? \|"
 
     run -0 cscli alerts list --machine
     assert_output --partial 'MACHINE'
-    assert_output --partial "| 'githubciXXXXXXXXXXXXXXXXXXXXXXXX' |"
-    assert_output --partial "| githubciXXXXXXXXXXXXXXXXXXXXXXXX |"
+    assert_output --regexp "\| 'githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})?' \|"
+    assert_output --regexp "\| githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})? \|"
 }

tests/bats/90_decisions.bats

@@ -43,16 +43,16 @@ declare stderr
     run -0 cscli decisions list
     refute_output --partial 'MACHINE'
     # machine name appears quoted in the "REASON" column
-    assert_output --partial "| 'githubciXXXXXXXXXXXXXXXXXXXXXXXX' |"
-    refute_output --partial "| githubciXXXXXXXXXXXXXXXXXXXXXXXX |"
+    assert_output --regexp "\| 'githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})?' \|"
+    refute_output --regexp "\| githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})? \|"
 
     run -0 cscli decisions list -m
     assert_output --partial 'MACHINE'
-    assert_output --partial "| 'githubciXXXXXXXXXXXXXXXXXXXXXXXX' |"
-    assert_output --partial "| githubciXXXXXXXXXXXXXXXXXXXXXXXX |"
+    assert_output --regexp "\| 'githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})?' \|"
+    assert_output --regexp "\| githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})? \|"
 
     run -0 cscli decisions list --machine
     assert_output --partial 'MACHINE'
-    assert_output --partial "| 'githubciXXXXXXXXXXXXXXXXXXXXXXXX' |"
-    assert_output --partial "| githubciXXXXXXXXXXXXXXXXXXXXXXXX |"
+    assert_output --regexp "\| 'githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})?' \|"
+    assert_output --regexp "\| githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})? \|"
 }

tests/instance-crowdsec

@@ -1,88 +1,16 @@
 #!/usr/bin/env bash
 
-set -eu
-script_name=$0
-
-die() {
-    echo >&2 "$@"
-    exit 1
-}
-
-about() {
-    die "usage: $script_name [ start | stop ]"
-}
-
 #shellcheck disable=SC1007
 THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+cd "${THIS_DIR}"
 #shellcheck disable=SC1090
-. "${THIS_DIR}/.environment.sh"
+. ./.environment.sh
 
-# you have not removed set -u above, have you?
+backend_script="./lib/init/crowdsec-${INIT_BACKEND}"
 
-[ -z "${CROWDSEC-}" ] && die "\$CROWDSEC must be defined."
-[ -z "${LOG_DIR-}" ] && die "\$LOG_DIR must be defined."
-[ -z "${PID_DIR-}" ] && die "\$PID_DIR must be defined."
-
-if [ ! -f "${CROWDSEC}" ]; then
-    die "${CROWDSEC} is missing. Please build (with 'make bats-build') or install it."
+if [ ! -x "$backend_script" ]; then
+    echo "unknown init system '$INIT_BACKEND'" >&2
+    exit 1
 fi
 
-wait_for_port() {
-    for _ in $(seq 40); do
-        nc -z localhost "$1" >/dev/null 2>&1 && return
-        sleep .05
-    done
-
-    # send to &3 if open
-    if { true >&3; } 2>/dev/null; then
-        # cat "${LOG_DIR}/crowdsec.out" >&3
-        # cat "${LOG_DIR}/crowdsec.log" >&3
-        echo "Can't connect to port $1" >&3
-    else
-        # cat "${LOG_DIR}/crowdsec.out" >&2
-        # cat "${LOG_DIR}/crowdsec.log" >&2
-        echo "Can't connect to port $1" >&2
-    fi
-
-    return 1
-}
-
-DAEMON_PID=${PID_DIR}/crowdsec.pid
-
-start_instance() {
-    OUT_FILE="${LOG_DIR}/crowdsec.out" \
-        DAEMON_PID="${DAEMON_PID}" \
-        "${TEST_DIR}/run-as-daemon" "${CROWDSEC}"
-    wait_for_port 6060
-}
-
-stop_instance() {
-    if [ -f "${DAEMON_PID}" ]; then
-       # terminate quickly with extreme prejudice, all the application data will be
-       # thrown away anyway. also terminate the child processes (notification plugin).
-       PGID="$(ps -o pgid= -p "$(cat "${DAEMON_PID}")" | tr -d ' ')"
-       # ps above should work on linux, freebsd, busybox..
-       if [ -n "${PGID}" ]; then
-           kill -- "-${PGID}"
-       fi
-       rm -f -- "${DAEMON_PID}"
-    fi
-}
-
-
-# ---------------------------
-
-[ $# -lt 1 ] && about
-
-case "$1" in
-    start)
-        start_instance
-        ;;
-    stop)
-        stop_instance
-        ;;
-    *)
-        about
-        ;;
-esac;
-
+exec "$backend_script" "$@"

tests/instance-data

@@ -1,150 +1,16 @@
 #!/usr/bin/env bash
 
-set -eu
-script_name=$0
-
-die() {
-    echo >&2 "$@"
-    exit 1
-}
-
-about() {
-    die "usage: $script_name [make | load | clean]"
-}
-
 #shellcheck disable=SC1007
 THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
 cd "${THIS_DIR}"
 #shellcheck disable=SC1090
 . ./.environment.sh
 
-# you have not removed set -u above, have you?
+backend_script="./lib/config/config-${CONFIG_BACKEND}"
 
-[ -z "${TEST_DIR-}" ] && die "\$TEST_DIR must be defined."
-[ -z "${LOCAL_DIR-}" ] && die "\$LOCAL_DIR must be defined."
-[ -z "${CSCLI-}" ] && die "\$CSCLI must be defined."
-[ -z "${LOCAL_INIT_DIR-}" ] && die "\$LOCAL_INIT_DIR must be defined."
-[ -z "${PLUGIN_DIR-}" ] && die "\$PLUGIN_DIR must be defined."
-[ -z "${DB_BACKEND-}" ] && die "\$DB_BACKEND must be defined."
-
-if [ ! -f "${CSCLI}" ]; then
-    die "${CSCLI} is missing. Please build (with 'make bats-build') or install it."
+if [ ! -x "$backend_script" ]; then
+    echo "unknown config backend '$CONFIG_BACKEND'" >&2
+    exit 1
 fi
 
-REL_CONFIG_DIR="etc/crowdsec"
-REL_DATA_DIR="var/lib/crowdsec/data"
-
-DATA_DIR="${LOCAL_DIR}/${REL_DATA_DIR}"
-export DATA_DIR
-CONFIG_DIR="${LOCAL_DIR}/${REL_CONFIG_DIR}"
-export CONFIG_DIR
-
-remove_init_data() {
-    rm -rf -- "${LOCAL_DIR:?}/${REL_CONFIG_DIR}"/* "${LOCAL_DIR:?}/${REL_DATA_DIR:?}"/*
-}
-
-config_generate() {
-    mkdir -p "${CONFIG_DIR}"
-
-    cp ../config/acquis.yaml \
-       ../config/profiles.yaml \
-       ../config/simulation.yaml \
-       ../config/local_api_credentials.yaml \
-       ../config/online_api_credentials.yaml \
-       "${CONFIG_DIR}/"
-
-    cp ../plugins/notifications/*/{http,email,slack,splunk,dummy}.yaml \
-       "${CONFIG_DIR}/notifications/"
-
-    yq '
-    .common.daemonize=false |
-    del(.common.pid_dir) |
-    .common.log_dir=strenv(LOG_DIR) |
-    .config_paths.config_dir=strenv(CONFIG_DIR) |
-    .config_paths.data_dir=strenv(DATA_DIR) |
-    .config_paths.simulation_path=strenv(CONFIG_DIR)+"/simulation.yaml" |
-    .config_paths.hub_dir=strenv(CONFIG_DIR)+"/hub/" |
-    .config_paths.index_path=strenv(CONFIG_DIR)+"/hub/.index.json" |
-    .config_paths.notification_dir=strenv(CONFIG_DIR)+"/notifications/" |
-    .config_paths.plugin_dir=strenv(PLUGIN_DIR) |
-    .crowdsec_service.acquisition_path=strenv(CONFIG_DIR)+"/acquis.yaml" |
-    .db_config.db_path=strenv(DATA_DIR)+"/crowdsec.db" |
-    .api.client.credentials_path=strenv(CONFIG_DIR)+"/local_api_credentials.yaml" |
-    .api.server.profiles_path=strenv(CONFIG_DIR)+"/profiles.yaml" |
-    .api.server.console_path=strenv(CONFIG_DIR)+"/console.yaml" |
-    .api.server.online_client.credentials_path=strenv(CONFIG_DIR)+"/online_api_credentials.yaml"
-    ' <../config/config.yaml >"${CONFIG_DIR}/config.yaml"
-}
-
-make_init_data() {
-    remove_init_data
-
-    mkdir -p "${DATA_DIR}"
-    mkdir -p "${CONFIG_DIR}/notifications"
-    mkdir -p "${CONFIG_DIR}/hub"
-    mkdir -p "${CONFIG_DIR}/patterns"
-    cp -ax "../config/patterns" "${CONFIG_DIR}/"
-    config_generate
-    # XXX errors from instance-db should be reported...
-    ./instance-db config-yaml
-    ./instance-db setup
-
-    "${CSCLI}" machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto
-    "${CSCLI}" capi register --schmilblick githubciXXXXXXXXXXXXXXXXXXXXXXXX
-    "${CSCLI}" hub update
-    "${CSCLI}" collections install crowdsecurity/linux
-
-    "${TEST_DIR}/instance-crowdsec" start
-    "${CSCLI}" lapi status
-    "${TEST_DIR}/instance-crowdsec" stop
-
-    mkdir -p "${LOCAL_INIT_DIR}"
-
-    ./instance-db dump "${LOCAL_INIT_DIR}/database"
-    echo "${DB_BACKEND}" > "${LOCAL_INIT_DIR}/.backend"
-
-    tar -C "${LOCAL_DIR}" --create \
-        --exclude "$REL_DATA_DIR"/crowdsec.db \
-        --file "${LOCAL_INIT_DIR}/init-config-data.tar" "$REL_CONFIG_DIR" "$REL_DATA_DIR"
-
-    remove_init_data
-}
-
-load_init_data() {
-    if [ ! -f "${LOCAL_INIT_DIR}/init-config-data.tar" ]; then
-        die "Initial data not found; did you run '$script_name make' ?"
-    fi
-
-    dump_backend="$(cat "${LOCAL_INIT_DIR}/.backend")"
-    if [ "$DB_BACKEND" != "$dump_backend" ]; then
-        die "Can't run with backend '$DB_BACKEND' because the test data was build with '$dump_backend'"
-    fi
-
-    remove_init_data
-
-    tar -C "${LOCAL_DIR}" --extract --file "${LOCAL_INIT_DIR}/init-config-data.tar"
-    ./instance-db restore "${LOCAL_INIT_DIR}/database"
-}
-
-
-# ---------------------------
-
-[ $# -lt 1 ] && about
-
-./assert-crowdsec-not-running
-
-case "$1" in
-    make)
-        make_init_data
-        ;;
-    load)
-        load_init_data
-        ;;
-    clean)
-        remove_init_data
-        ;;
-    *)
-        about
-        ;;
-esac;
-
+exec "$backend_script" "$@"

tests/instance-mock-http

@@ -13,6 +13,8 @@ about() {
 
 #shellcheck disable=SC1007
 THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+cd "${THIS_DIR}"
+
 #shellcheck disable=SC1090
 . "${THIS_DIR}/.environment.sh"
 
@@ -25,24 +27,6 @@ if ! command -v python3 >/dev/null 2>&2; then
     die "The python3 executable is is missing. Please install it and try again."
 fi
 
-wait_for_port() {
-    for _ in $(seq 40); do
-        nc -z localhost "$1" && return
-        sleep .05
-    done
-
-    # send to &3 if open
-    if { true >&3; } 2>/dev/null; then
-        # cat "${LOG_DIR}/mock-http.out" >&3
-        echo "Can't connect to port $1" >&3
-    else
-        # cat "${LOG_DIR}/mock-http.out" >&2
-        echo "Can't connect to port $1" >&2
-    fi
-
-    return 1
-}
-
 DAEMON_PID=${PID_DIR}/mock-http.pid
 
 start_instance() {
@@ -50,7 +34,7 @@ start_instance() {
     OUT_FILE="${LOG_DIR}/mock-http.out" \
         DAEMON_PID="${DAEMON_PID}" \
         "${TEST_DIR}/run-as-daemon" /usr/bin/env python3 -u "${THIS_DIR}/mock-http.py" "$1"
-    wait_for_port "$1"
+    ./lib/util/wait-for-port "$1"
 #    echo "mock http started on port $1"
 }
 

tests/lib/config/config-global

@@ -0,0 +1,109 @@
+#!/usr/bin/env bash
+
+set -eu
+script_name=$0
+
+die() {
+    echo >&2 "$@"
+    exit 1
+}
+
+about() {
+    die "usage: $script_name [make | load | clean]"
+}
+
+#shellcheck disable=SC1007
+THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+cd "${THIS_DIR}"/../../
+#shellcheck disable=SC1090
+. ./.environment.sh
+
+# you have not removed set -u above, have you?
+
+[ -z "${TEST_DIR-}" ] && die "\$TEST_DIR must be defined."
+[ -z "${LOCAL_DIR-}" ] && die "\$LOCAL_DIR must be defined."
+[ -z "${CSCLI-}" ] && die "\$CSCLI must be defined."
+[ -z "${LOCAL_INIT_DIR-}" ] && die "\$LOCAL_INIT_DIR must be defined."
+[ -z "${PLUGIN_DIR-}" ] && die "\$PLUGIN_DIR must be defined."
+[ -z "${DB_BACKEND-}" ] && die "\$DB_BACKEND must be defined."
+
+if [ ! -f "${CSCLI}" ]; then
+    die "${CSCLI} is missing. Please build (with 'make bats-build') or install it."
+fi
+
+REL_CONFIG_DIR="etc/crowdsec"
+REL_DATA_DIR="var/lib/crowdsec/data"
+
+DATA_DIR="${LOCAL_DIR}/${REL_DATA_DIR}"
+export DATA_DIR
+CONFIG_DIR="${LOCAL_DIR}/${REL_CONFIG_DIR}"
+export CONFIG_DIR
+
+remove_init_data() {
+    rm -rf -- "${LOCAL_DIR:?}/${REL_CONFIG_DIR}"/* "${LOCAL_DIR:?}/${REL_DATA_DIR:?}"/*
+}
+
+#we need a separate function for initializing config when testing package
+#because we want to test the configuration as well
+make_init_data() {
+    "${TEST_DIR}/instance-crowdsec" stop
+
+    ./instance-db config-yaml
+    ./instance-db setup
+
+    "${TEST_DIR}/instance-crowdsec" start
+    "${CSCLI}" lapi status
+    "${TEST_DIR}/instance-crowdsec" stop
+
+    mkdir -p "${LOCAL_INIT_DIR}"
+
+    ./instance-db dump "${LOCAL_INIT_DIR}/database"
+
+    echo "${DB_BACKEND}" > "${LOCAL_INIT_DIR}/.backend"
+    tar -C "${LOCAL_DIR}" --create \
+        --exclude "$REL_DATA_DIR"/crowdsec.db \
+        --file "${LOCAL_INIT_DIR}/init-config-data.tar" "$REL_CONFIG_DIR" "$REL_DATA_DIR"
+
+    remove_init_data
+}
+
+
+load_init_data() {
+    if [ ! -f "${LOCAL_INIT_DIR}/init-config-data.tar" ]; then
+        die "Initial data not found; did you run '$script_name make' ?"
+    fi
+
+    dump_backend="$(cat "${LOCAL_INIT_DIR}/.backend")"
+    if [ "$DB_BACKEND" != "$dump_backend" ]; then
+        die "Can't run with backend '$DB_BACKEND' because the test data was build with '$dump_backend'"
+    fi
+
+    remove_init_data
+
+    tar -C "${LOCAL_DIR}" --extract --file "${LOCAL_INIT_DIR}/init-config-data.tar"
+    ./instance-db restore "${LOCAL_INIT_DIR}/database"
+}
+
+
+# ---------------------------
+
+[ $# -lt 1 ] && about
+
+
+case "$1" in
+    make)
+        make_init_data
+        ;;
+    load)
+        ./assert-crowdsec-not-running
+        load_init_data
+        ;;
+    clean)
+        ./assert-crowdsec-not-running
+        remove_init_data
+        ;;
+    *)
+        about
+        ;;
+esac;
+

tests/lib/config/config-local

@@ -0,0 +1,152 @@
+#!/usr/bin/env bash
+
+set -eu
+script_name=$0
+
+die() {
+    echo >&2 "$@"
+    exit 1
+}
+
+about() {
+    die "usage: $script_name [make | load | clean]"
+}
+
+#shellcheck disable=SC1007
+THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+cd "${THIS_DIR}"/../../
+#shellcheck disable=SC1090
+. ./.environment.sh
+
+# you have not removed set -u above, have you?
+
+[ -z "${TEST_DIR-}" ] && die "\$TEST_DIR must be defined."
+[ -z "${LOCAL_DIR-}" ] && die "\$LOCAL_DIR must be defined."
+[ -z "${CSCLI-}" ] && die "\$CSCLI must be defined."
+[ -z "${LOCAL_INIT_DIR-}" ] && die "\$LOCAL_INIT_DIR must be defined."
+[ -z "${PLUGIN_DIR-}" ] && die "\$PLUGIN_DIR must be defined."
+[ -z "${DB_BACKEND-}" ] && die "\$DB_BACKEND must be defined."
+
+if [ ! -f "${CSCLI}" ]; then
+    die "${CSCLI} is missing. Please build (with 'make bats-build') or install it."
+fi
+
+REL_CONFIG_DIR="etc/crowdsec"
+REL_DATA_DIR="var/lib/crowdsec/data"
+
+DATA_DIR="${LOCAL_DIR}/${REL_DATA_DIR}"
+export DATA_DIR
+CONFIG_DIR="${LOCAL_DIR}/${REL_CONFIG_DIR}"
+export CONFIG_DIR
+
+remove_init_data() {
+    rm -rf -- "${LOCAL_DIR:?}/${REL_CONFIG_DIR}"/* "${LOCAL_DIR:?}/${REL_DATA_DIR:?}"/*
+}
+
+config_generate() {
+    mkdir -p "${CONFIG_DIR}"
+
+    cp ../config/acquis.yaml \
+       ../config/profiles.yaml \
+       ../config/simulation.yaml \
+       ../config/local_api_credentials.yaml \
+       ../config/online_api_credentials.yaml \
+       "${CONFIG_DIR}/"
+
+    cp ../plugins/notifications/*/{http,email,slack,splunk,dummy}.yaml \
+       "${CONFIG_DIR}/notifications/"
+
+    yq '
+    .common.daemonize=false |
+    del(.common.pid_dir) |
+    .common.log_dir=strenv(LOG_DIR) |
+    .config_paths.config_dir=strenv(CONFIG_DIR) |
+    .config_paths.data_dir=strenv(DATA_DIR) |
+    .config_paths.simulation_path=strenv(CONFIG_DIR)+"/simulation.yaml" |
+    .config_paths.hub_dir=strenv(CONFIG_DIR)+"/hub/" |
+    .config_paths.index_path=strenv(CONFIG_DIR)+"/hub/.index.json" |
+    .config_paths.notification_dir=strenv(CONFIG_DIR)+"/notifications/" |
+    .config_paths.plugin_dir=strenv(PLUGIN_DIR) |
+    .crowdsec_service.acquisition_path=strenv(CONFIG_DIR)+"/acquis.yaml" |
+    .db_config.db_path=strenv(DATA_DIR)+"/crowdsec.db" |
+    .api.client.credentials_path=strenv(CONFIG_DIR)+"/local_api_credentials.yaml" |
+    .api.server.profiles_path=strenv(CONFIG_DIR)+"/profiles.yaml" |
+    .api.server.console_path=strenv(CONFIG_DIR)+"/console.yaml" |
+    .api.server.online_client.credentials_path=strenv(CONFIG_DIR)+"/online_api_credentials.yaml"
+    ' <../config/config.yaml >"${CONFIG_DIR}/config.yaml"
+}
+
+make_init_data() {
+    remove_init_data
+
+    mkdir -p "${DATA_DIR}"
+    mkdir -p "${CONFIG_DIR}/notifications"
+    mkdir -p "${CONFIG_DIR}/hub"
+    mkdir -p "${CONFIG_DIR}/patterns"
+    cp -ax "../config/patterns" "${CONFIG_DIR}/"
+    config_generate
+    # XXX errors from instance-db should be reported...
+    ./instance-db config-yaml
+    ./instance-db setup
+
+    "${CSCLI}" machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto
+    "${CSCLI}" capi register --schmilblick githubciXXXXXXXXXXXXXXXXXXXXXXXX
+    "${CSCLI}" hub update
+    "${CSCLI}" collections install crowdsecurity/linux
+
+    "${TEST_DIR}/instance-crowdsec" start
+    "${CSCLI}" lapi status
+    "${TEST_DIR}/instance-crowdsec" stop
+
+    mkdir -p "${LOCAL_INIT_DIR}"
+
+    ./instance-db dump "${LOCAL_INIT_DIR}/database"
+    echo "${DB_BACKEND}" > "${LOCAL_INIT_DIR}/.backend"
+
+    tar -C "${LOCAL_DIR}" --create \
+        --exclude "$REL_DATA_DIR"/crowdsec.db \
+        --file "${LOCAL_INIT_DIR}/init-config-data.tar" "$REL_CONFIG_DIR" "$REL_DATA_DIR"
+
+    remove_init_data
+}
+
+load_init_data() {
+    if [ ! -f "${LOCAL_INIT_DIR}/init-config-data.tar" ]; then
+        die "Initial data not found; did you run '$script_name make' ?"
+    fi
+
+    dump_backend="$(cat "${LOCAL_INIT_DIR}/.backend")"
+    if [ "$DB_BACKEND" != "$dump_backend" ]; then
+        die "Can't run with backend '$DB_BACKEND' because the test data was build with '$dump_backend'"
+    fi
+
+    remove_init_data
+
+    tar -C "${LOCAL_DIR}" --extract --file "${LOCAL_INIT_DIR}/init-config-data.tar"
+    ./instance-db restore "${LOCAL_INIT_DIR}/database"
+}
+
+
+# ---------------------------
+
+[ $# -lt 1 ] && about
+
+
+case "$1" in
+    make)
+        ./assert-crowdsec-not-running
+        make_init_data
+        ;;
+    load)
+        ./assert-crowdsec-not-running
+        load_init_data
+        ;;
+    clean)
+        ./assert-crowdsec-not-running
+        remove_init_data
+        ;;
+    *)
+        about
+        ;;
+esac;
+

tests/lib/init/crowdsec-daemon

@@ -0,0 +1,69 @@
+    #!/usr/bin/env bash
+
+    set -eu
+    script_name=$0
+
+    die() {
+        echo >&2 "$@"
+        exit 1
+    }
+
+    about() {
+        die "usage: $script_name [ start | stop ]"
+    }
+
+    #shellcheck disable=SC1007
+    THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+    cd "${THIS_DIR}"/../../
+#shellcheck disable=SC1091
+. ./.environment.sh
+
+# you have not removed set -u above, have you?
+
+[ -z "${CROWDSEC-}" ] && die "\$CROWDSEC must be defined."
+[ -z "${LOG_DIR-}" ] && die "\$LOG_DIR must be defined."
+[ -z "${PID_DIR-}" ] && die "\$PID_DIR must be defined."
+
+if [ ! -f "${CROWDSEC}" ]; then
+    die "${CROWDSEC} is missing. Please build (with 'make bats-build') or install it."
+fi
+
+DAEMON_PID=${PID_DIR}/crowdsec.pid
+
+start() {
+    OUT_FILE="${LOG_DIR}/crowdsec.out" \
+            DAEMON_PID="${DAEMON_PID}" \
+            "${TEST_DIR}/run-as-daemon" "${CROWDSEC}"
+    ./lib/util/wait-for-port 6060
+}
+
+stop() {
+    if [ -f "${DAEMON_PID}" ]; then
+       # terminate quickly with extreme prejudice, all the application data will be
+       # thrown away anyway. also terminate the child processes (notification plugin).
+       PGID="$(ps -o pgid= -p "$(cat "${DAEMON_PID}")" | tr -d ' ')"
+       # ps above should work on linux, freebsd, busybox..
+       if [ -n "${PGID}" ]; then
+           kill -- "-${PGID}"
+       fi
+       rm -f -- "${DAEMON_PID}"
+    fi
+}
+
+
+# ---------------------------
+
+[ $# -lt 1 ] && about
+
+case "$1" in
+    start)
+        start
+        ;;
+    stop)
+        stop
+        ;;
+    *)
+        about
+        ;;
+esac;
+

tests/lib/init/crowdsec-systemd

@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+
+set -eu
+script_name=$0
+
+die() {
+    echo >&2 "$@"
+    exit 1
+}
+
+about() {
+    die "usage: $script_name [ start | stop ]"
+}
+
+#shellcheck disable=SC1007
+THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
+cd "${THIS_DIR}"/../../
+#shellcheck disable=SC1091
+. ./.environment.sh
+
+# you have not removed set -u above, have you?
+
+[ -z "${CROWDSEC-}" ] && die "\$CROWDSEC must be defined."
+[ -z "${LOG_DIR-}" ] && die "\$LOG_DIR must be defined."
+[ -z "${PID_DIR-}" ] && die "\$PID_DIR must be defined."
+
+if [ ! -f "${CROWDSEC}" ]; then
+    die "${CROWDSEC} is missing. Please build (with 'make bats-build') or install it."
+fi
+
+start() {
+    systemctl start crowdsec
+    ./lib/util/wait-for-port 6060
+}
+
+stop() {
+    systemctl stop crowdsec # systemd doesn't throw error when stopping already stopped stuff
+    while pidof /usr/bin/crowdsec ; do sleep 0.1; done
+}
+
+
+# ---------------------------
+
+[ $# -lt 1 ] && about
+
+case "$1" in
+    start)
+        start
+        ;;
+    stop)
+        stop
+        ;;
+    *)
+        about
+        ;;
+esac;
+

tests/lib/util/wait-for-port

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+set -eu
+script_name=$0
+
+die() {
+    echo >&2 "$@"
+    exit 1
+}
+
+about() {
+    die "usage: $script_name <port_number>"
+}
+
+[ $# -lt 1 ] && about
+
+port_number=$1
+
+for _ in $(seq 40); do
+    nc -z localhost "$port_number" >/dev/null 2>&1 && exit 0
+    sleep .05
+done
+
+# send to &3 if open
+if { true >&3; } 2>/dev/null; then
+    echo "Can't connect to port $port_number" >&3
+else
+    echo "Can't connect to port $port_number" >&2
+fi
+
+exit 1
+