update functional tests for build pipeline (#2442)

test/ansible/roles/make_fixture/tasks/main.yml

@@ -52,7 +52,7 @@
         # daemonize -> /usr/bin or /usr/local/sbin
         # pidof -> /usr/sbin
         # bash -> /opt/bash/bin
-        PATH: "/opt/bash/bin:{{ ansible_env.PATH }}:/usr/sbin:/usr/local/sbin"
+        PATH: "/opt/bash/bin:{{ ansible_env.PATH }}:{{ golang_install_dir }}/bin/:/usr/sbin:/usr/local/sbin"
   rescue:
     - name: "Read crowdsec.log"
       ansible.builtin.slurp:

test/ansible/vagrant/ubuntu-22.04-jammy/Vagrantfile

@@ -3,6 +3,7 @@
 Vagrant.configure('2') do |config|
   config.vm.box = 'generic/ubuntu2204'
   config.vm.provision "shell", inline: <<-SHELL
+    sudo apt install -y kitty-terminfo
   SHELL
 end
 

test/ansible/vagrant/ubuntu-22.10-kinetic/Vagrantfile

@@ -3,6 +3,7 @@
 Vagrant.configure('2') do |config|
   config.vm.box = 'generic/ubuntu2210'
   config.vm.provision "shell", inline: <<-SHELL
+    sudo apt install -y kitty-terminfo
   SHELL
 end
 

test/ansible/vagrant/ubuntu-23.04-lunar/Vagrantfile

@@ -3,6 +3,7 @@
 Vagrant.configure('2') do |config|
   config.vm.box = 'bento/ubuntu-23.04'
   config.vm.provision "shell", inline: <<-SHELL
+    sudo apt install -y kitty-terminfo
   SHELL
 end
 

test/bats/01_crowdsec.bats

@@ -79,7 +79,7 @@ teardown() {
 
     rune -0 ./instance-crowdsec start-pid
     PID="$output"
-    assert_file_exist "$log_old"
+    assert_file_exists "$log_old"
     assert_file_contains "$log_old" "Starting processing data"
 
     logdir2=$(TMPDIR="${BATS_TEST_TMPDIR}" mktemp -u)
@@ -113,7 +113,7 @@ teardown() {
 
     sleep 5
 
-    assert_file_exist "$log_new"
+    assert_file_exists "$log_new"
 
     for ((i=0; i<10; i++)); do
         sleep 1
@@ -192,6 +192,12 @@ teardown() {
 }
 
 @test "crowdsec (disabled datasources)" {
+    if is_package_testing; then
+        # we can't hide journalctl in package testing
+        # because crowdsec is run from systemd
+        skip "n/a for package testing"
+    fi
+
     config_set '.common.log_media="stdout"'
 
     # a datasource cannot run - missing journalctl command

test/bats/01_cscli.bats

@@ -102,12 +102,12 @@ teardown() {
 
     rune -0 cscli config show -o human
     assert_line --regexp ".*- URL +: http://127.0.0.1:8080/"
-    assert_line --regexp ".*- Login +: githubciXXXXXXXXXXXXXXXXXXXXXXXX"
+    assert_line --regexp ".*- Login +: githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})?"
     assert_line --regexp ".*- Credentials File +: .*/local_api_credentials.yaml"
 
     rune -0 cscli config show -o json
-    rune -0 jq -c '.API.Client.Credentials | [.url,.login]' <(output)
-    assert_output '["http://127.0.0.1:8080/","githubciXXXXXXXXXXXXXXXXXXXXXXXX"]'
+    rune -0 jq -c '.API.Client.Credentials | [.url,.login[0:32]]' <(output)
+    assert_json '["http://127.0.0.1:8080/","githubciXXXXXXXXXXXXXXXXXXXXXXXX"]'
 }
 
 @test "cscli config show-yaml" {
@@ -146,7 +146,7 @@ teardown() {
     # restore
     rm "${SIMULATION_YAML}"
     rune -0 cscli config restore "${backupdir}"
-    assert_file_exist "${SIMULATION_YAML}"
+    assert_file_exists "${SIMULATION_YAML}"
 
     # cleanup
     rm -rf -- "${backupdir:?}"
@@ -283,7 +283,7 @@ teardown() {
 
 @test "cscli support dump (smoke test)" {
     rune -0 cscli support dump -f "$BATS_TEST_TMPDIR"/dump.zip
-    assert_file_exist "$BATS_TEST_TMPDIR"/dump.zip
+    assert_file_exists "$BATS_TEST_TMPDIR"/dump.zip
 }
 
 @test "cscli explain" {
@@ -321,14 +321,14 @@ teardown() {
     rune -0 cscli doc
     refute_output
     refute_stderr
-    assert_file_exist "doc/cscli.md"
+    assert_file_exists "doc/cscli.md"
     assert_file_not_exist "doc/cscli_setup.md"
 
     # commands guarded by feature flags are not documented unless the feature flag is set
 
     export CROWDSEC_FEATURE_CSCLI_SETUP="true"
     rune -0 cscli doc
-    assert_file_exist "doc/cscli_setup.md"
+    assert_file_exists "doc/cscli_setup.md"
 }
 
 @test "feature.yaml for subcommands" {

test/bats/04_capi.bats

@@ -45,7 +45,7 @@ setup() {
 @test "we have exactly one machine, localhost" {
     rune -0 cscli machines list -o json
     rune -0 jq -c '[. | length, .[0].machineId[0:32], .[0].isValidated, .[0].ipAddress]' <(output)
-    assert_output '[1,"githubciXXXXXXXXXXXXXXXXXXXXXXXX",true,"127.0.0.1"]'
+    assert_json '[1,"githubciXXXXXXXXXXXXXXXXXXXXXXXX",true,"127.0.0.1"]'
 }
 
 @test "no agent: capi status should be ok" {

test/bats/07_setup.bats

@@ -7,6 +7,8 @@ setup_file() {
     load "../lib/setup_file.sh"
     ./instance-data load
     HUB_DIR=$(config_get '.config_paths.hub_dir')
+    # remove trailing slash if any (like in default config.yaml from package)
+    HUB_DIR=${HUB_DIR%/}
     export HUB_DIR
     DETECT_YAML="${HUB_DIR}/detect.yaml"
     export DETECT_YAML

test/bats/71_dummy_plugin.bats

@@ -5,7 +5,7 @@ set -u
 
 setup_file() {
     load "../lib/setup_file.sh"
-    [[ -n "${PACKAGE_TESTING}" ]] && return
+    is_package_testing && return
 
     ./instance-data load
 
@@ -51,7 +51,7 @@ teardown_file() {
 }
 
 setup() {
-    [[ -n "${PACKAGE_TESTING}" ]] && skip
+    is_package_testing && skip
     load "../lib/setup.sh"
 }
 

test/bats/72_plugin_badconfig.bats

@@ -73,11 +73,9 @@ teardown() {
     rune -0 yq -i '.name="email_default"' "$CONFIG_DIR/notifications/http.yaml"
     # enable a notification, otherwise plugins are ignored
     config_set "${PROFILES_PATH}" '.notifications=["slack_default"]'
-    # we want to check the logs
+    # the slack plugin may fail or not, but we just need the logs
     config_set '.common.log_media="stdout"'
-    # the command will fail because slack_deault is not working
-    run -1 --separate-stderr timeout 2s "${CROWDSEC}"
-    # but we have what we wanted
+    rune timeout 2s "${CROWDSEC}"
     assert_stderr --partial "notification 'email_default' is defined multiple times"
 }
 

test/bats/81_alert_context.bats

@@ -40,7 +40,12 @@ teardown() {
 	  type: syslog
 	EOT
 
-    CONTEXT_YAML=$(config_get '.crowdsec_service.console_context_path')
+    # we set the path here because the default is empty
+    CONFIG_DIR=$(dirname "$CONFIG_YAML")
+    CONTEXT_YAML="$CONFIG_DIR/console/context.yaml"
+    export CONTEXT_YAML
+    config_set '.crowdsec_service.console_context_path=strenv(CONTEXT_YAML)'
+    mkdir -p "$CONFIG_DIR/console"
 
     cat <<-EOT >"${CONTEXT_YAML}"
 	target_user:

test/bats/testdata/explain/explain-log.txt

@@ -5,7 +5,8 @@ line: Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authenticatio
 	|	└ 🟢 crowdsecurity/sshd-logs (+8 ~1)
 	├ s02-enrich
 	|	├ 🟢 crowdsecurity/dateparse-enrich (+2 ~2)
-	|	└ 🟢 crowdsecurity/geoip-enrich (+10)
+	|	├ 🟢 crowdsecurity/geoip-enrich (+10)
+	|	└ 🟢 crowdsecurity/whitelists (unchanged)
 	├-------- parser success 🟢
 	├ Scenarios
 		├ 🟢 crowdsecurity/ssh-bf

test/lib/bats-assert

@@ -1 +1 @@
-Subproject commit 397c735212bf1a06cfdd0cb7806c5a6ea79582bf
+Subproject commit 78fa631d1370562d2cd4a1390989e706158e7bf0

test/lib/bats-core

@@ -1 +1 @@
-Subproject commit 6636e2c2ef5ffe361535cb45fc61682c5ef46b71
+Subproject commit f7defb94362f2053a3e73d13086a167448ea9133

test/lib/bats-file

@@ -1 +1 @@
-Subproject commit 17fa557f6fe28a327933e3fa32efef1d211caa5a
+Subproject commit cb914cdc176da00e321d3bc92f88383698c701d6

test/lib/bats-support

@@ -1 +1 @@
-Subproject commit d140a65044b2d6810381935ae7f0c94c7023c8c3
+Subproject commit 3c8fadc5097c9acfc96d836dced2bb598e48b009

test/lib/config/config-local

@@ -61,8 +61,6 @@ config_generate() {
        ../config/online_api_credentials.yaml \
        "${CONFIG_DIR}/"
 
-    cp ../config/context.yaml "${CONFIG_DIR}/console/"
-
     cp ../config/detect.yaml \
        "${HUB_DIR}"
 
@@ -99,7 +97,6 @@ config_generate() {
     .api.client.credentials_path=strenv(CONFIG_DIR)+"/local_api_credentials.yaml" |
     .api.server.profiles_path=strenv(CONFIG_DIR)+"/profiles.yaml" |
     .api.server.console_path=strenv(CONFIG_DIR)+"/console.yaml" |
-    .crowdsec_service.console_context_path=strenv(CONFIG_DIR) + "/console/context.yaml" |
     del(.api.server.online_client)
     ' ../config/config.yaml >"${CONFIG_DIR}/config.yaml"
 }
@@ -113,7 +110,6 @@ make_init_data() {
     mkdir -p "${CONFIG_DIR}/notifications"
     mkdir -p "${CONFIG_DIR}/hub"
     mkdir -p "${CONFIG_DIR}/patterns"
-    mkdir -p "${CONFIG_DIR}/console"
     cp -a "../config/patterns" "${CONFIG_DIR}/"
     config_generate
     # XXX errors from instance-db should be reported...
@@ -123,6 +119,8 @@ make_init_data() {
     "$CSCLI" --warning machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto
     "$CSCLI" --warning hub update
     "$CSCLI" --warning collections install crowdsecurity/linux
+    # the whitelists are installed by the deb & rpm packages, so we test with the same config
+    "$CSCLI" --warning parsers install crowdsecurity/whitelists
 
     mkdir -p "$LOCAL_INIT_DIR"
 

test/lib/setup_file.sh

@@ -116,18 +116,23 @@ output() {
 }
 export -f output
 
+is_package_testing() {
+    [[ "$PACKAGE_TESTING" != "" ]]
+}
+export -f is_package_testing
+
 is_db_postgres() {
-    [[ "${DB_BACKEND}" =~ ^postgres|pgx$ ]]
+    [[ "$DB_BACKEND" =~ ^postgres|pgx$ ]]
 }
 export -f is_db_postgres
 
 is_db_mysql() {
-    [[ "${DB_BACKEND}" == "mysql" ]]
+    [[ "$DB_BACKEND" == "mysql" ]]
 }
 export -f is_db_mysql
 
 is_db_sqlite() {
-    [[ "${DB_BACKEND}" == "sqlite" ]]
+    [[ "$DB_BACKEND" == "sqlite" ]]
 }
 export -f is_db_sqlite