Ansible testing improvements (#1700)

platform/openbsd.mk

@@ -0,0 +1,6 @@
+# OpenBSD specific
+#
+
+Make=gmake
+
+$(info building for OpenBSD)

tests/README.md

@@ -56,14 +56,17 @@ architectures.
 ## pre-requisites
 
  - `git submodule init; git submodule update`
- - `daemonize (linux) or daemon (freebsd), bash>=4.4, python3, openbsd-netcat`
  - `go install github.com/cloudflare/cfssl/cmd/cfssl@latest`
  - `go install github.com/cloudflare/cfssl/cmd/cfssljson@latest`
  - `go install github.com/mikefarah/yq/v4@latest`
  - `base64`
+ - `bash>=4.4`
  - `curl`
+ - `daemonize`
  - `jq`
  - `nc`
+ - `openssl`
+ - `openbsd-netcat`
  - `python3`
 
 ## Running all tests

tests/ansible/README.md

@@ -104,10 +104,11 @@ To test with Vagrant, you need to:
   space taken by the base VM images, they are in
   `/var/lib/libvirt/images/*VAGRANT*`
 
-The above steps are automated in the script `./prepare-run` (requires bash >=4.4).
-It takes an enviroment file, and optionally a list of directories with vagrant
-configurations. With a single parameter, it loops over all the directories in
-alphabetical order. Watch out for leftover VMs if you break the loop by hand.
+The above steps are automated in the script `./prepare-run` (requires bash
+>=4.4). It takes an enviroment file, and optionally a list of directories with
+vagrant configurations. With a single parameter, it loops over all the
+directories in alphabetical order, excluding those in the `experimental`
+directory. Watch out for running VMs if you break the loop by hand.
 
 After this, you will find up to 30GB of base images in `/var/lib/libvirt/images`,
 which you need to remove by hand when you have finished testing or leave them
@@ -132,16 +133,16 @@ The data was created with crowdsec v1.4.1.
 | Debian 9 (stretch)        | ✓             | ✓          | old-db          | old-db     | wip              |
 | Debian 10 (buster)        | ✓             | ✓          | ✓               | ✓          | ✓                |
 | Debian 11 (bullseye)      | ✓             | ✓          | ✓               | ✓          | ✓                |
-| Debian (testing/bookworm) | ✓             | ✓          | wip             | wip        | wip              |
+| Debian (testing/bookworm) | ✓             | ✓          | ✓               | ✓          | wip              |
 | Fedora 33                 | ✓             | ✓          | wip             | wip        | wip              |
 | Fedora 34                 | ✓             | ✓          | ✓               | ✓          | wip              |
 | Fedora 35                 | ✓             | ✓          | ✓               | ✓          | wip              |
 | Fedora 36                 | ✓             | ✓          | ✓               | ✓          | wip              |
 | FreeBSD 12                | ✓             | wip        | wip             | wip        | wip              |
 | FreeBSD 13                | ✓             | wip        | wip             | wip        | wip              |
-| Oracle 7                  | ✓             | ✓          | wip             | wip        | ✓                |
+| Oracle 7                  | ✓             | ✓          | old-db          | old-db     | ✓                |
 | Oracle 8                  | ✓             | ✓          | ✓               | ✓          | ✓                |
-| Ubuntu 16.04 (xenial)     | ✓             | ✓          | wip             | wip        | ✓                |
+| Ubuntu 16.04 (xenial)     | ✓             | ✓          | old-db          | old-db     | ✓                |
 | Ubuntu 18.04 (bionic)     | ✓             | ✓          | ✓               | ✓          | ✓                |
 | Ubuntu 20.04 (focal)      | ✓             | ✓          | ✓               | ✓          | ✓                |
 | Ubuntu 22.04 (jammy)      | ✓             | ✓          | ✓               | ✓          | ✓                |

tests/ansible/ansible.cfg

@@ -1,14 +1,15 @@
 [defaults]
 pipelining = True
 force_color = True
+nocows = True
 
 # inventory = inventory.yml
 callbacks_enabled = timer
 
 # more compact and readable output
 stdout_callback = debug
-display_skipped_hosts = no
-display_ok_hosts = yes
+display_skipped_hosts = False
+display_ok_hosts = True
 
 [ssh_connection]
 ssh_args = -o ControlMaster=auto -o ControlPersist=60s

tests/ansible/prepare-run

@@ -14,7 +14,8 @@ shift
 
 vagrant_dirs=("$@")
 if [[ $# -eq 0 ]]; then
-    readarray -d '' vagrant_dirs < <(find vagrant -mindepth 1 -maxdepth 1 -type d -print0 | sort -z)
+    # find all targets, with possibly weird names, don't go into subfolders (like 'experimental/')
+    readarray -d '' vagrant_dirs < <(find vagrant -mindepth 1 -maxdepth 1 -type d -print0 | sort -z | grep -z -v .vagrant)
 fi
 
 #shellcheck disable=SC1090
@@ -26,12 +27,23 @@ export VAGRANT_FORCE_COLOR
 for vm in "${vagrant_dirs[@]}"; do
     outfile="$(basename "${env}").out"
     pushd "${vm}" >/dev/null || exit
+    if [[ ! -f "Vagrantfile" ]]; then
+            popd >/dev/null || exit
+            continue
+    fi
+    echo "Prepare and run tests on ${vm}..."
+    if [[ -x "skip" ]]; then
+        if ! ./skip; then
+            popd >/dev/null || exit
+            continue
+        fi
+    fi
     if [[ ! -f "${outfile}" ]]; then
         vagrant up --no-provision
         vagrant provision 2>&1 | tee "${outfile}"
         vagrant destroy -f
     else
-        echo "Skipping: ${vm}, file ${outfile} already exists." >&2
+        echo "skipping: ${vm}, file ${outfile} already exists." >&2
     fi
     popd >/dev/null || exit
 done

tests/ansible/provision_dependencies.yml

@@ -9,8 +9,9 @@
         name: gantsign.golang
       when:
         - ansible_facts.system == 'Linux'
+        - ansible_facts.distribution != 'Alpine'
 
-- name: "install Go (bsd)"
+- name: "install Go (bsd, alpine)"
   hosts: all
   gather_facts: true
   become: true
@@ -19,7 +20,7 @@
         name: go
         state: present
       when:
-        - ansible_facts.system == 'FreeBSD'
+        - ansible_facts.system in ['FreeBSD', 'OpenBSD'] or ansible_facts.distribution == 'Alpine'
 
 # required for jq and basic tools on centos and rhel
 - name: "enable EPEL repository (RedHat)"
@@ -32,7 +33,7 @@
         epel_repo_disable: false
       when:
         - ansible_facts.os_family == 'RedHat'
-        - ansible_facts.distribution != 'Fedora'
+        - (ansible_facts.distribution != 'Fedora') and (ansible_facts.distribution != 'Amazon')
 
 - name: "apply common configuration to all nodes"
   hosts: all
@@ -51,6 +52,9 @@
   tasks:
     - ansible.builtin.include_role:
         name: geerlingguy.postgresql
+      # enable this for debugging
+      #vars:
+      #  postgres_users_no_log: false
       when:
         - lookup('ansible.builtin.env', 'DB_BACKEND') in ['pgx', 'postgres']
 

tests/ansible/requirements.yml

@@ -1,6 +1,9 @@
 ---
 - src: geerlingguy.mysql
-- src: geerlingguy.postgresql
 - src: geerlingguy.repo-epel
 - src: gantsign.golang
+- src: https://github.com/crowdsecurity/ansible-role-postgresql
+  version: crowdsec
+  name: geerlingguy.postgresql
+
 

tests/ansible/roles/bats_requirements/defaults/main.yml

@@ -1,4 +1,5 @@
 ---
 build_bash: false
+bash_version: 5.1.16
 
 build_daemonize: (ansible_facts.distribution == "Ubuntu" and ansible_facts.distribution == '16.04') or ansible_facts.distribution == 'Amazon'

tests/ansible/roles/bats_requirements/tasks/bash.yml

@@ -1,51 +1,42 @@
 ---
+- name: "check if we need to build bash"
+  become: false
+  block:
     - name: "look up bash version"
-  become: false
       ansible.builtin.package_facts:
-
     - name: "bash version found"
-  become: false
       ansible.builtin.debug:
         var: ansible_facts.packages['bash'][0].version
-
-- name: "check if bash needs building (<4.4)"
-  become: false
+    - name: "check if bash is too old (<4.4)"
       ansible.builtin.set_fact:
         build_bash: "{{ ansible_facts.packages['bash'][0].version is version('4.4', '<') }}"
 
+- name: "build bash"
+  block:
     - name: "build bash: download"
       become: false
       ansible.builtin.unarchive:
-    src: http://ftp.gnu.org/gnu/bash/bash-5.1.16.tar.gz
+        src: http://ftp.gnu.org/gnu/bash/bash-{{ bash_version }}.tar.gz
         dest: "{{ ansible_env.HOME }}"
         remote_src: true
-    creates: "{{ ansible_env.HOME }}/bash-5.1.16"
-  when:
-    - build_bash
-
+        creates: "{{ ansible_env.HOME }}/bash-{{ bash_version }}"
     - name: "build bash: configure"
       become: false
       ansible.builtin.command:
         cmd: "./configure --prefix=/opt/bash"
         creates: ./Makefile
-    chdir: "{{ ansible_env.HOME }}/bash-5.1.16"
-  when:
-    - build_bash
-
+        chdir: "{{ ansible_env.HOME }}/bash-{{ bash_version }}"
     - name: "build bash: create /opt/bash"
       become: true
       ansible.builtin.file:
         path: /opt/bash
         state: directory
         mode: 0o755
-  when:
-    - build_bash
-
     - name: "build bash: make install"
       become: true
       ansible.builtin.command:
         cmd: "make install"
         creates: /opt/bash/bin/bash
-    chdir: "{{ ansible_env.HOME }}/bash-5.1.16"
+        chdir: "{{ ansible_env.HOME }}/bash-{{ bash_version }}"
   when:
     - build_bash

tests/ansible/roles/bats_requirements/tasks/daemonize.yml

@@ -7,29 +7,36 @@
   when:
     - not build_daemonize
 
+- name: "build daemonize"
+  block:
     - name: "build daemonize: git checkout"
       become: false
       ansible.builtin.git:
         repo: https://github.com/bmc/daemonize
         dest: "{{ ansible_env.HOME }}/daemonize"
         version: release-1.7.8
-  when:
-    - build_daemonize
-
     - name: "build daemonize: configure"
       become: false
       ansible.builtin.command:
         cmd: "./configure --prefix=/usr/local"
         creates: ./Makefile
         chdir: "{{ ansible_env.HOME }}/daemonize"
-  when:
-    - build_daemonize
-
-- name: "build daemonize: make install"
+    - name: "build daemonize: make install (linux)"
       become: true
       ansible.builtin.command:
         cmd: "make all install"
         creates: /usr/local/sbin/daemonize
         chdir: "{{ ansible_env.HOME }}/daemonize"
+      when:
+        - ansible_facts.system == 'Linux'
+    - name: "build daemonize: make install (bsd)"
+      become: true
+      ansible.builtin.command:
+        cmd: "gmake all install"
+        creates: /usr/local/sbin/daemonize
+        chdir: "{{ ansible_env.HOME }}/daemonize"
+      when:
+        - ansible_facts.system in ['FreeBSD', 'OpenBSD']
   when:
     - build_daemonize
+

tests/ansible/roles/bats_requirements/tasks/main.yml

@@ -1,11 +1,12 @@
 ---
 - name: "install bash"
   ansible.builtin.import_tasks: bash.yml
+  when:
+    # openbsd is not supported by the package_facts module, let's assume bash is ok
+    - ansible_facts.system != 'OpenBSD'
 
 - name: "Install daemonize"
   ansible.builtin.import_tasks: daemonize.yml
-  when:
-    - ansible_facts.system == 'Linux'
 
 - name: "install netcat"
   ansible.builtin.import_tasks: netcat.yml
@@ -18,6 +19,8 @@
       - jq
       - openssl
       - python3
+  when:
+    - ansible_facts.distribution != 'Gentoo'
 
 - name: "install bc (!freebsd)"
   become: true
@@ -28,13 +31,13 @@
     - ansible_facts.system == 'Linux'
 
 # base64 for linux is in coreutils
-- name: "install base64 (freebsd)"
+- name: "install base64 (bsd)"
   become: true
   ansible.builtin.package:
     name:
       - base64
   when:
-    - ansible_facts.system == 'FreeBSD'
+    - ansible_facts.system in ['FreeBSD', 'OpenBSD']
 
 - name: "install pidof (Amazon)"
   become: true
@@ -44,12 +47,6 @@
   when:
     - ansible_facts.distribution == 'Amazon'
 
-- name: "install gcc (for go-sqlite, needs cgo)"
-  become: true
-  ansible.builtin.package:
-    name:
-      - gcc
-
 - name: "install cfssl"
   become: true
   ansible.builtin.command:

tests/ansible/roles/bats_requirements/tasks/netcat.yml

@@ -1,5 +1,5 @@
 ---
-- name: "install netcat (Amazon, Fedora, CentOS)"
+- name: "install netcat (Amazon, Fedora, CentOS, Oracle)"
   become: true
   ansible.builtin.package:
     name:
@@ -13,22 +13,13 @@
     name:
       - netcat
   when:
-    - (ansible_facts.distribution == 'RedHat') or  (ansible_facts.distribution == 'AlmaLinux')
+    - (ansible_facts.distribution == 'RedHat') or (ansible_facts.distribution == 'AlmaLinux') or (ansible_facts.distribution == 'Rocky')
 
-# "netcat" does not exist in some versions (only -traditional or -openbsd)
-- name: "install netcat (Debian)"
-  become: true
-  ansible.builtin.package:
-    name:
-      - netcat-traditional
-  when:
-    - ansible_facts.os_family == "Debian"
-
-# "netcat" does not exist in some versions (only -traditional or -openbsd)
+# "netcat" does not exist in some Debian versions (only -traditional or -openbsd)
 - name: "install netcat (Suse)"
   become: true
   ansible.builtin.package:
     name:
       - netcat-openbsd
   when:
-    - ansible_facts.os_family == "Suse"
+    - ansible_facts.os_family in ["Debian", "Suse"]

tests/ansible/roles/common/tasks/main.yml

@@ -22,10 +22,10 @@
   when:
     - ansible_facts.system == "Linux"
 
-- name: "install gmake (FreeBSD)"
+- name: "install gmake (bsd)"
   become: true
   ansible.builtin.package:
     name:
       - gmake
   when:
-    - ansible_facts.system == "FreeBSD"
+    - ansible_facts.system in ['FreeBSD', 'OpenBSD']

tests/ansible/roles/install_crowdsec_package/tasks/install_from_deb.yml

@@ -5,34 +5,30 @@
   when:
     - (package_dir is defined) and (package_dir | length > 0)
 
-- name: "look for .deb file matching package_file"
+- name: "install crowdsec from package_file"
+  become: true
+  block:
+    - name: "look for file matching package_file"
       ansible.builtin.set_fact:
         found_file: "{{ item }}"
       with_fileglob:
         - "{{ package_file }}"
-  when:
-    - (package_file is defined) and (package_file | length > 0)
 
     - name: "check found_file"
       ansible.builtin.fail:
         msg: "No file found matching {{ package_file }}"
       when:
         - found_file is not defined
-    - (package_file is defined) and (package_file | length > 0)
 
-- name: "copy built file for deb-like"
-  become: false
+    - name: "copy {{ found_file }}"
       ansible.builtin.copy:
         src: "{{ found_file }}"
-    dest: "{{ ansible_env.HOME }}/crowdsec.deb"
+        dest: "/root/crowdsec.deb"
         mode: 0o644
-  when:
-    - (package_file is defined) and (package_file | length > 0)
 
-- name: "install crowdsec on deb-like"
-  become: true
+    - name: "install crowdsec"
       ansible.builtin.apt:
-    deb: "{{ ansible_env.HOME }}/crowdsec.deb"
+        deb: "/root/crowdsec.deb"
         allow_downgrade: true
   when:
     - (package_file is defined) and (package_file | length > 0)

tests/ansible/roles/install_crowdsec_package/tasks/install_from_deb_repo.yml

@@ -6,28 +6,22 @@
       - apt-transport-https
       - gnupg
 
-- name: "install crowdsec repo [1/2] (*.deb)"
+- name: "install crowdsec from the package repository"
   become: true
+  block:
+    - name: "stable apt repo key"
       ansible.builtin.apt_key:
         url: https://packagecloud.io/crowdsec/crowdsec/gpgkey
-
-- name: "add crowdsec repo [2/2] (*.deb)"
-  become: true
+    - name: "stable apt repo"
       ansible.builtin.apt_repository:
         repo: deb https://packagecloud.io/crowdsec/crowdsec/{{ ansible_facts.distribution | lower }}/ {{ ansible_facts.distribution_release }} main
-
-- name: "install crowdsec testing repo [1/2] (*.deb)"
-  become: true
+    - name: "testing apt repo key"
       ansible.builtin.apt_key:
         url: https://packagecloud.io/crowdsec/crowdsec-testing/gpgkey
-
-- name: "add crowdsec testing repo [1/2] (*.deb)"
-  become: true
+    - name: "testing apt repo"
       ansible.builtin.apt_repository:
         repo: deb https://packagecloud.io/crowdsec/crowdsec-testing/{{ ansible_facts.distribution | lower }}/ {{ ansible_facts.distribution_release }} main
-
-- name: "install crowdsec"
-  become: true
+    - name: "install crowdsec {{ package_vesion_deb }} with apt"
       ansible.builtin.package:
         name:
           - crowdsec={{ package_version_deb }}

tests/ansible/roles/install_crowdsec_package/tasks/install_from_rpm.yml

@@ -5,34 +5,30 @@
   when:
     - (package_dir is defined) and (package_dir | length > 0)
 
-- name: "look for .rpm file matching package_file"
+- name: "install crowdsec from package_file"
+  become: true
+  block:
+    - name: "look for file matching package_file"
       ansible.builtin.set_fact:
         found_file: "{{ item }}"
       with_fileglob:
         - "{{ package_file }}"
-  when:
-    - (package_file is defined) and (package_file | length > 0)
 
     - name: "check found_file"
       ansible.builtin.fail:
         msg: "No file found matching {{ package_file }}"
       when:
         - found_file is not defined
-    - (package_file is defined) and (package_file | length > 0)
 
-- name: "copy built file for rpm-like"
-  become: false
+    - name: "copy {{ found_file }}"
       ansible.builtin.copy:
         src: "{{ found_file }}"
-    dest: "{{ ansible_env.HOME }}/crowdsec.rpm"
+        dest: "/root/crowdsec.rpm"
         mode: 0o644
-  when:
-    - (package_file is defined) and (package_file | length > 0)
 
-- name: "install crowdsec on rpm-like"
-  become: true
+    - name: "install crowdsec"
       ansible.builtin.yum:
-    name: "{{ ansible_env.HOME }}/crowdsec.rpm"
+        name: "/root/crowdsec.rpm"
         disable_gpg_check: true
         allow_downgrade: true
   when:

tests/ansible/roles/install_crowdsec_package/tasks/install_from_rpm_repo.yml

@@ -1,30 +1,28 @@
 ---
-- name: "download the rpm script"
+- name: "install crowdsec from the package repository"
+  become: true
+  block:
+    - name: "stable rpm repo script"
       ansible.builtin.get_url:
         url: https://packagecloud.io/install/repositories/crowdsec/crowdsec/script.rpm.sh
-    dest: "{{ ansible_env.HOME }}/rpm.sh"
+        dest: "/root/rpm.sh"
         mode: 0o775
-
-- name: "install crowdsec rpm repo"
-  become: true
+    - name: "stable rpm repo"
       ansible.builtin.command:
-    cmd: "{{ ansible_env.HOME }}/rpm.sh"
+        cmd: "/root/rpm.sh"
       changed_when: false
 
-- name: "download the (testing) rpm script"
+    - name: "testing rpm repo script"
       ansible.builtin.get_url:
         url: https://packagecloud.io/install/repositories/crowdsec/crowdsec-testing/script.rpm.sh
-    dest: "{{ ansible_env.HOME }}/rpm-testing.sh"
+        dest: "/root/rpm-testing.sh"
         mode: 0o775
-
-- name: "install crowdsec (testing) rpm repo"
-  become: true
+    - name: "testing rpm repo"
       ansible.builtin.command:
-    cmd: "{{ ansible_env.HOME }}/rpm-testing.sh"
+        cmd: "/root/rpm-testing.sh"
       changed_when: false
 
     - name: "install crowdsec"
-  become: true
       ansible.builtin.package:
         name:
           - crowdsec-{{ package_version_rpm }}.{{ releasever.replace('amzn2', 'el7').replace('ol7', 'el7').replace('ol8', 'el8') }}

tests/ansible/roles/install_crowdsec_package/tasks/main.yml

@@ -1,12 +1,46 @@
 ---
+- name: "set releasever for RedHat family"
+  ansible.builtin.set_fact:
+    releasever: "{{ release[ansible_facts.distribution] + ansible_facts.distribution_major_version }}"
+  vars:
+    release:
+      AlmaLinux: el
+      Amazon: amzn
+      CentOS: el
+      Fedora: fc
+      OracleLinux: ol
+      Rocky: el
+  when:
+    - ansible_facts.os_family == "RedHat"
+
 - name: "system details"
   ansible.builtin.debug:
     msg: |
       Distribution: {{ ansible_facts.distribution }}
       Version: {{ ansible_facts.distribution_version }}
-      Major: {{ ansible_facts.distribution_major_version }}
+      Major: {{ ansible_facts.distribution_major_version | default('n/a') }}
       Release: {{ ansible_facts.distribution_release }}
-      Releasever: {{ releasever }}
+      Releasever: {{ releasever | default('n/a') }}
+
+- name: "hardcode master branch for the hub, temporary override before install (config.yaml.local)"
+  become: true
+  block:
+    - name: "create /etc/crowdsec"
+      ansible.builtin.file:
+        path: "/etc/crowdsec"
+        state: directory
+        mode: 0o0755
+    - name: "create /etc/crowdsec/config.yaml.local"
+      ansible.builtin.copy:
+        dest: "/etc/crowdsec/config.yaml.local"
+        content: "{{ config_yaml_local | to_nice_yaml }}"
+        mode: 0o600
+      vars:
+        config_yaml_local:
+          cscli:
+            hub_branch: master
+  when:
+    - (package_testing is defined) and (package_testing not in ['', 'false', 'False'])
 
 - name: "install from binary repository (RedHat)"
   ansible.builtin.import_tasks: install_from_rpm_repo.yml
@@ -29,3 +63,50 @@
   ansible.builtin.import_tasks: install_from_deb.yml
   when:
     - ansible_facts.os_family == "Debian"
+
+- name: "hardcode master branch for the hub, for real this time"
+  become: true
+  block:
+    - name: "read config.yaml"
+      ansible.builtin.slurp:
+        path: "/etc/crowdsec/config.yaml"
+      register: config_yaml
+    - name: "create fact from config.yaml"
+      ansible.builtin.set_fact:
+        config_data: "{{ config_yaml['content'] | b64decode | from_yaml }}"
+    - name: "patch dictionary"
+      ansible.builtin.set_fact:
+        config_data: "{{ config_data | combine(config_patch, recursive=True) }}"
+      vars: 
+        config_patch:
+          cscli:
+            hub_branch: master
+    - name: "write patched config.yaml"
+      ansible.builtin.copy:
+        content: '{{ config_data | to_nice_yaml }}'
+        dest: "/etc/crowdsec/config.yaml"
+        # preserve mode to be able to test permissions from package
+        mode: preserve
+    - name: "remove config.yaml.local"
+      ansible.builtin.file:
+        path: "/etc/crowdsec/config.yaml.local"
+        state: absent
+  when:
+    - (package_testing is defined) and (package_testing not in ['', 'false', 'False'])
+
+# this is required to avoid fatal errors in case systemctl is not working (which happens on some aws instances)
+- name: "override acquis.yaml for package testing"
+  become: true
+  ansible.builtin.copy:
+    dest: "/etc/crowdsec/acquis.yaml"
+    content: "{{ acquis_yaml | to_nice_yaml }}"
+    mode: preserve
+  vars:
+    acquis_yaml:
+      filenames:
+        - /tmp/should-not-exist.log
+      labels:
+        type: syslog
+        force_inotify: true
+  when:
+    - (package_testing is defined) and (package_testing not in ['', 'false', 'False'])

tests/ansible/roles/install_crowdsec_package/vars/main.yml

@@ -1,19 +1,8 @@
 ---
-release:
-  CentOS: el
-  Fedora: fc
-  Amazon: amzn
-  Debian: debian
-  Ubuntu: ubuntu
-  FreeBSD: freebsd
-  OracleLinux: ol
-  Rocky: el
-  AlmaLinux: el
-  openSUSE Leap: opensuse
-
-releasever: "{{ release[ansible_facts.distribution] + ansible_facts.distribution_major_version }}"
 
 package_version_deb: "{{ lookup('ansible.builtin.env', 'TEST_PACKAGE_VERSION_DEB') }}"
 package_version_rpm: "{{ lookup('ansible.builtin.env', 'TEST_PACKAGE_VERSION_RPM') }}"
 package_file: "{{ lookup('ansible.builtin.env', 'TEST_PACKAGE_FILE') }}"
 package_dir: "{{ lookup('ansible.builtin.env', 'TEST_PACKAGE_DIR') }}"
+package_testing: "{{ lookup('ansible.builtin.env', 'PACKAGE_TESTING') }}"
+

tests/ansible/roles/install_crowdsec_tests/tasks/main.yml

@@ -14,13 +14,13 @@
     suite_zip: "{{ lookup('ansible.builtin.env', 'TEST_SUITE_ZIP') }}"
   when: lookup('ansible.builtin.env', 'TEST_SUITE_ZIP') | length>0
 
+- name: "install tests from zip file"
+  block:
     - name: "install unzip"
       become: true
       ansible.builtin.package:
         name:
           - unzip
-  when:
-    - (suite_zip is defined) and (suite_zip|length > 0)
 
     - name: "install tests: create crowdsec dir"
       become: false
@@ -28,16 +28,12 @@
         path: "{{ ansible_env.HOME }}/crowdsec"
         state: directory
         mode: 0o700
-  when:
-    - (suite_zip is defined) and (suite_zip|length > 0)
 
     - name: "install tests: extract crowdsec"
       become: false
       ansible.builtin.unarchive:
         src: "{{ suite_zip }}"
         dest: "{{ ansible_env.HOME }}/crowdsec"
-  when:
-    - (suite_zip is defined) and (suite_zip|length > 0)
 
     - name: "install tests: git submodules for bats"
       become: false
@@ -50,19 +46,20 @@
   when:
     - (suite_zip is defined) and (suite_zip|length > 0)
 
+
+- name: "install tests from repository"
+  block:
     - name: "install tests: checkout crowdsec"
       become: false
       ansible.builtin.git:
         repo: "{{ suite_git }}"
         dest: "{{ ansible_env.HOME }}/crowdsec"
-    single_branch: true
         version: "{{ suite_version }}"
-  when:
-    - (suite_zip is not defined) or (suite_zip|length == 0)
 
+    # trust the dir if we need to test as root
     - name: set safe.directory on crowdsec dir"
-  become: false
-  ansible.builtin.git_config:
+      become: true
+      community.general.git_config:
         scope: global
         name: safe.directory
         value: "{{ ansible_env.HOME }}/crowdsec"

tests/ansible/roles/make_fixture/tasks/main.yml

@@ -1,19 +1,20 @@
 ---
-- name: "set make_cmd = make (!freebsd)"
+- name: "set make_cmd = make (!bsd)"
   ansible.builtin.set_fact:
     make_cmd: make
   when:
-    - ansible_facts.system != 'FreeBSD'
+    - ansible_facts.system not in ['FreeBSD', 'OpenBSD']
 
-- name: "set make_cmd = gmake (freebsd)"
+- name: "set make_cmd = gmake (bsd)"
   ansible.builtin.set_fact:
     make_cmd: gmake
   when:
-    - ansible_facts.system == 'FreeBSD'
+    - ansible_facts.system in ['FreeBSD', 'OpenBSD']
 
 - name: "build crowdsec from sources, prepare test environment and fixture"
   become: false
-  ansible.builtin.command:
+  block:
+    - ansible.builtin.command:
         cmd: "{{ make_cmd }} bats-build bats-fixture"
         chdir: "{{ ansible_env.HOME }}/crowdsec"
         creates: "{{ ansible_env.HOME }}/crowdsec/tests/local-init/init-config-data.tar"
@@ -23,11 +24,20 @@
         # pidof -> /usr/sbin
         # bash -> /opt/bash/bin
         PATH: "/opt/bash/bin:{{ ansible_env.PATH }}:{{ golang_install_dir }}/bin/:/usr/sbin:/usr/local/sbin"
+  rescue:
+    - name: "read crowdsec.log"
+      ansible.builtin.slurp:
+        path: "{{ ansible_env.HOME }}/crowdsec/tests/local/var/log/crowdsec.log"
+      register: crowdsec_log
+    - name: "show crowdsec.log"
+      ansible.builtin.fail:
+        msg: "{{ crowdsec_log['content'] | b64decode }}"
   when: (package_testing is not defined) or (package_testing in ['', 'false', 'False'])
 
 - name: "prepare test environment and fixture for binary package"
   become: true
-  ansible.builtin.command:
+  block:
+    - ansible.builtin.command:
         cmd: "{{ make_cmd }} bats-environment bats-check-requirements bats-fixture"
         chdir: "{{ ansible_env.HOME }}/crowdsec"
         creates: "{{ ansible_env.HOME }}/crowdsec/tests/local-init/init-config-data.tar"
@@ -38,18 +48,27 @@
         # pidof -> /usr/sbin
         # bash -> /opt/bash/bin
         PATH: "/opt/bash/bin:{{ ansible_env.PATH }}:/usr/sbin:/usr/local/sbin"
+  rescue:
+    - name: "read crowdsec.log"
+      ansible.builtin.slurp:
+        path: "/var/log/crowdsec.log"
+      register: crowdsec_log
+    - name: "show crowdsec.log"
+      ansible.builtin.fail:
+        msg: "{{ crowdsec_log['content'] | b64decode }}"
   when: (package_testing is defined) and (package_testing not in ['', 'false', 'False'])
 
-- name: "read .environment.sh"
+- name: "debug - show environment"
+  become: false
+  block:
+    - name: "look for .environment.sh"
       ansible.builtin.slurp:
         src: "{{ ansible_env.HOME }}/crowdsec/tests/.environment.sh"
       changed_when: false
       register: envfile
-
-- name: "show .environment.sh"
+    - name: "cat .environment.sh"
       ansible.builtin.debug:
         msg: "{{ envfile['content'] | b64decode }}"
-
     - name: "show environment variables"
       ansible.builtin.debug:
         msg: "{{ ansible_env | to_nice_yaml }}"

tests/ansible/roles/run_func_tests/tasks/main.yml

@@ -1,18 +1,15 @@
 ---
-- name: "create /lib/systemd/system/crowdsec.service.d"
+- name: "tweak systemd configuration for tests"
   become: true
+  block:
+    - name: "create /lib/systemd/system/crowdsec.service.d"
       ansible.builtin.file:
         owner: root
         group: root
         mode: 0o755
         path: /lib/systemd/system/crowdsec.service.d
         state: directory
-  when:
-    - (package_testing is defined) and (package_testing not in ['', 'false', 'False'])
-    - ansible_facts.os_family in ["RedHat", "Debian"]
-
     - name: "override StartLimitBurst"
-  become: true
       ansible.builtin.ini_file:
         dest: /lib/systemd/system/crowdsec.service.d/startlimitburst.conf
         owner: root
@@ -21,18 +18,25 @@
         section: Service
         option: StartLimitBurst
         value: 100
-  when:
-    - (package_testing is defined) and (package_testing not in ['', 'false', 'False'])
-    - ansible_facts.os_family in ["RedHat", "Debian"]
-
     - name: "systemctl daemon-reload"
-  become: true
       ansible.builtin.systemd:
         daemon_reload: true
   when:
     - (package_testing is defined) and (package_testing not in ['', 'false', 'False'])
     - ansible_facts.os_family in ["RedHat", "Debian"]
 
+- name: "debug - show environment.sh"
+  become: false
+  block:
+    - name: "look for .environment.sh"
+      ansible.builtin.slurp:
+        src: "{{ ansible_env.HOME }}/crowdsec/tests/.environment.sh"
+      changed_when: false
+      register: envfile
+    - name: "cat .environment.sh"
+      ansible.builtin.debug:
+        msg: "{{ envfile['content'] | b64decode }}"
+
 - name: "search for test scripts"
   become: false
   ansible.builtin.find:
@@ -40,19 +44,10 @@
     pattern: "*.bats"
   register: testfiles
 
-- name: "read .environment.sh"
-  ansible.builtin.slurp:
-    src: "{{ ansible_env.HOME }}/crowdsec/tests/.environment.sh"
-  changed_when: false
-  register: envfile
-
-- name: "show .environment.sh"
-  ansible.builtin.debug:
-    msg: "{{ envfile['content'] | b64decode }}"
-
 - name: "run BATS tests for source build"
-  ignore_errors: false
   become: false
+  block:
+    - name: "run test scripts"
       ansible.builtin.command:
         cmd: tests/run-tests {{ item.path }}
         chdir: "{{ ansible_env.HOME }}/crowdsec"
@@ -65,13 +60,23 @@
         # bash -> /opt/bash/bin
         PATH: "/opt/bash/bin:{{ ansible_env.PATH }}:/usr/sbin:/usr/local/sbin"
       changed_when: false
+      when:
+        - (item.path | basename) not in skip_tests.split(',')
+  rescue:
+    - name: "read crowdsec.log"
+      ansible.builtin.slurp:
+        path: "{{ ansible_env.HOME }}/crowdsec/tests/local/var/log/crowdsec.log"
+      register: crowdsec_log
+    - name: "show crowdsec.log"
+      ansible.builtin.fail:
+        msg: "{{ crowdsec_log['content'] | b64decode }}"
   when:
     - (package_testing is not defined) or (package_testing in ['', 'false', 'False'])
-    - (item.path | basename) not in skip_tests.split(',')
 
 - name: "run BATS tests for binary package"
-  ignore_errors: false
   become: true
+  block:
+    - name: "run test scripts"
       ansible.builtin.command:
         cmd: tests/run-tests {{ item.path }}
         chdir: "{{ ansible_env.HOME }}/crowdsec"
@@ -85,5 +90,14 @@
         PATH: "/opt/bash/bin:{{ ansible_env.PATH }}:/usr/sbin:/usr/local/sbin"
       changed_when: false
       when:
-    - (package_testing is defined) and (package_testing not in ['', 'false', 'False'])
         - (item.path | basename) not in skip_tests.split(',')
+  rescue:
+    - name: "read crowdsec.log"
+      ansible.builtin.slurp:
+        path: "/var/log/crowdsec.log"
+      register: crowdsec_log
+    - name: "show crowdsec.log"
+      ansible.builtin.fail:
+        msg: "{{ crowdsec_log['content'] | b64decode }}"
+  when:
+    - (package_testing is defined) and (package_testing not in ['', 'false', 'False'])

tests/ansible/vagrant/Vagrantfile.common

@@ -1,7 +1,7 @@
 Vagrant.configure("2") do |config|
   config.vm.provider :libvirt do |libvirt|
     libvirt.cpus = 1
-    libvirt.memory = 1024
+    libvirt.memory = 1536
   end
 
   config.vm.synced_folder '.', '/vagrant', disabled: true

tests/ansible/vagrant/alma-8/Vagrantfile

@@ -1,6 +1,6 @@
+Vagrant.configure("2") do |config|
+  config.vm.box = "generic/alma8"
+end
+
 common = '../Vagrantfile.common'
 load common if File.exists?(common)
-
-Vagrant.configure("2") do |config|
-  config.vm.box = "cloudnatives/amazon-linux-2"
-end

tests/ansible/vagrant/alma-9/Vagrantfile

@@ -1,6 +1,6 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   config.vm.box = "generic/alma9"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/centos-7/Vagrantfile

@@ -1,6 +1,6 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   config.vm.box = "centos/7"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/centos-7/skip

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+die() {
+    echo "$@" >&2
+    exit 1
+}
+
+# postgres is too old on this distribution
+[ "${DB_BACKEND}" = "postgres" ] && die "skipping: postgres too old"
+[ "${DB_BACKEND}" = "pgx" ] && die "skipping: postgres too old"
+exit 0

tests/ansible/vagrant/centos-8/Vagrantfile

@@ -1,6 +1,6 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   config.vm.box = "centos/stream8"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/centos-9/Vagrantfile

@@ -1,6 +1,6 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   config.vm.box = "generic/centos9s"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/debian-10-buster/Vagrantfile

@@ -1,6 +1,6 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   config.vm.box = "debian/buster64"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/debian-11-bullseye/Vagrantfile

@@ -1,6 +1,6 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   config.vm.box = "debian/bullseye64"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/debian-9-stretch/Vagrantfile

@@ -1,6 +1,6 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   config.vm.box = "debian/stretch64"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/debian-9-stretch/skip

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+die() {
+    echo "$@" >&2
+    exit 1
+}
+
+# postgres is too old on this distribution
+[ "${DB_BACKEND}" = "postgres" ] && die "skipping: postgres too old"
+[ "${DB_BACKEND}" = "pgx" ] && die "skipping: postgres too old"
+exit 0

tests/ansible/vagrant/debian-testing/Vagrantfile

@@ -1,6 +1,6 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   config.vm.box = "debian/testing64"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/experimental/alpine-3.16/Vagrantfile

@@ -0,0 +1,18 @@
+Vagrant.configure("2") do |config|
+  config.vm.box = "generic/alpine316"
+  config.vm.provision "shell", path: "bootstrap"
+
+  config.vm.provider :libvirt do |libvirt|
+    libvirt.cpus = 1
+    libvirt.memory = 1536
+  end
+
+  config.vm.synced_folder '.', '/vagrant', disabled: true
+
+  config.vm.provision "ansible" do |ansible|
+    # ansible.verbose = "vvvv"
+    ansible.config_file = "../../../ansible.cfg"
+    ansible.playbook = "../../../run_all.yml"
+  end
+
+end

tests/ansible/vagrant/experimental/alpine-3.16/bootstrap

@@ -0,0 +1,5 @@
+#!/bin/sh
+unset IFS
+set -euf
+
+sudo apk add python3 go tar procps netcat-openbsd

tests/ansible/vagrant/experimental/alpine-3.16/skip

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+die() {
+    echo "$@" >&2
+    exit 1
+}
+
+[ "${PACKAGE_TESTING}" = "true" ] && die "no package available for this distribution"
+exit 0

tests/ansible/vagrant/experimental/amazon-linux-2/Vagrantfile

@@ -0,0 +1,17 @@
+Vagrant.configure("2") do |config|
+  config.vm.box = "cloudnatives/amazon-linux-2"
+
+  config.vm.provider :libvirt do |libvirt|
+    libvirt.cpus = 1
+    libvirt.memory = 1536
+  end
+
+  config.vm.synced_folder '.', '/vagrant', disabled: true
+
+  config.vm.provision "ansible" do |ansible|
+    # ansible.verbose = "vvvv"
+    ansible.config_file = "../../../ansible.cfg"
+    ansible.playbook = "../../../run_all.yml"
+  end
+
+end

tests/ansible/vagrant/experimental/amazon-linux-2/issues.txt

@@ -0,0 +1,3 @@
+
+The file 70_http_plugin.bats hangs forever when run from ansible on amzn2, but all tests pass when run from ssh.
+

tests/ansible/vagrant/experimental/arch/Vagrantfile

@@ -0,0 +1,17 @@
+Vagrant.configure("2") do |config|
+  config.vm.box = "generic/arch"
+
+  config.vm.provider :libvirt do |libvirt|
+    libvirt.cpus = 1
+    libvirt.memory = 1536
+  end
+
+  config.vm.synced_folder '.', '/vagrant', disabled: true
+
+  config.vm.provision "ansible" do |ansible|
+    # ansible.verbose = "vvvv"
+    ansible.config_file = "../../../ansible.cfg"
+    ansible.playbook = "../../../run_all.yml"
+  end
+
+end

tests/ansible/vagrant/experimental/devuan-3/Vagrantfile

@@ -0,0 +1,17 @@
+Vagrant.configure("2") do |config|
+  config.vm.box = "generic/devuan3"
+
+  config.vm.provider :libvirt do |libvirt|
+    libvirt.cpus = 1
+    libvirt.memory = 1536
+  end
+
+  config.vm.synced_folder '.', '/vagrant', disabled: true
+
+  config.vm.provision "ansible" do |ansible|
+    # ansible.verbose = "vvvv"
+    ansible.config_file = "../../../ansible.cfg"
+    ansible.playbook = "../../../run_all.yml"
+  end
+
+end

tests/ansible/vagrant/experimental/devuan-3/skip

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+die() {
+    echo "$@" >&2
+    exit 1
+}
+
+[ "${PACKAGE_TESTING}" = "true" ] && die "no package available for this distribution"
+exit 0

tests/ansible/vagrant/experimental/dragonflybsd-6/Vagrantfile

@@ -0,0 +1,17 @@
+Vagrant.configure("2") do |config|
+  config.vm.box = "generic/dragonflybsd6"
+
+  config.vm.provider :libvirt do |libvirt|
+    libvirt.cpus = 1
+    libvirt.memory = 1536
+  end
+
+  config.vm.synced_folder '.', '/vagrant', disabled: true
+
+  config.vm.provision "ansible" do |ansible|
+    # ansible.verbose = "vvvv"
+    ansible.config_file = "../../../ansible.cfg"
+    ansible.playbook = "../../../run_all.yml"
+  end
+
+end

tests/ansible/vagrant/experimental/gentoo/Vagrantfile

@@ -0,0 +1,18 @@
+Vagrant.configure("2") do |config|
+  config.vm.box = "generic/gentoo"
+  config.vm.provision "shell", path: "bootstrap"
+
+  config.vm.provider :libvirt do |libvirt|
+    libvirt.cpus = 1
+    libvirt.memory = 1536
+  end
+
+  config.vm.synced_folder '.', '/vagrant', disabled: true
+
+  config.vm.provision "ansible" do |ansible|
+    # ansible.verbose = "vvvv"
+    ansible.config_file = "../../../ansible.cfg"
+    ansible.playbook = "../../../run_all.yml"
+  end
+
+end

tests/ansible/vagrant/experimental/gentoo/bootstrap

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+sudo emerge --quiet app-portage/gentoolkit dev-vcs/git net-misc/curl app-misc/jq net-analyzer/openbsd-netcat

tests/ansible/vagrant/experimental/hardenedbsd-13/Vagrantfile

@@ -0,0 +1,18 @@
+Vagrant.configure("2") do |config|
+  config.vm.box = "generic/hardenedbsd13"
+  config.vm.provision "shell", path: "bootstrap"
+
+  config.vm.provider :libvirt do |libvirt|
+    libvirt.cpus = 1
+    libvirt.memory = 1536
+  end
+
+  config.vm.synced_folder '.', '/vagrant', disabled: true
+
+  config.vm.provision "ansible" do |ansible|
+    # ansible.verbose = "vvvv"
+    ansible.config_file = "../../../ansible.cfg"
+    ansible.playbook = "../../../run_all.yml"
+  end
+
+end

tests/ansible/vagrant/experimental/hardenedbsd-13/bootstrap

@@ -0,0 +1,5 @@
+#!/bin/sh
+unset IFS
+set -euf
+
+sudo pkg install python3

tests/ansible/vagrant/experimental/hardenedbsd-13/skip

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+die() {
+    echo "$@" >&2
+    exit 1
+}
+
+[ "${PACKAGE_TESTING}" = "true" ] && die "no package available for this distribution"
+exit 0

tests/ansible/vagrant/experimental/netbsd-9/Vagrantfile

@@ -0,0 +1,18 @@
+Vagrant.configure("2") do |config|
+  config.vm.box = "generic/netbsd9"
+  # config.vm.provision "shell", path: "bootstrap"
+
+  config.vm.provider :libvirt do |libvirt|
+    libvirt.cpus = 1
+    libvirt.memory = 1536
+  end
+
+  config.vm.synced_folder '.', '/vagrant', disabled: true
+
+  config.vm.provision "ansible" do |ansible|
+    # ansible.verbose = "vvvv"
+    ansible.config_file = "../../../ansible.cfg"
+    ansible.playbook = "../../../run_all.yml"
+  end
+
+end

tests/ansible/vagrant/experimental/openbsd-7/Vagrantfile

@@ -0,0 +1,18 @@
+Vagrant.configure("2") do |config|
+  config.vm.box = "generic/openbsd7"
+  config.vm.provision "shell", path: "bootstrap"
+
+  config.vm.provider :libvirt do |libvirt|
+    libvirt.cpus = 1
+    libvirt.memory = 1536
+  end
+
+  config.vm.synced_folder '.', '/vagrant', disabled: true
+
+  config.vm.provision "ansible" do |ansible|
+    # ansible.verbose = "vvvv"
+    ansible.config_file = "../../../ansible.cfg"
+    ansible.playbook = "../../../run_all.yml"
+  end
+
+end

tests/ansible/vagrant/experimental/openbsd-7/bootstrap

@@ -0,0 +1,6 @@
+#!/bin/sh
+unset IFS
+set -euf
+
+sudo pkg_add -u
+sudo pkg_add python-3.9.13 py3-pip gcc-11.2.0p2 openssl-3.0.3p0 gtar-1.34 truncate-5.2.1

tests/ansible/vagrant/experimental/openbsd-7/skip

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+die() {
+    echo "$@" >&2
+    exit 1
+}
+
+[ "${PACKAGE_TESTING}" = "true" ] && die "no package available for this distribution"
+exit 0

tests/ansible/vagrant/experimental/opensuse-15.4/Vagrantfile

@@ -0,0 +1,18 @@
+Vagrant.configure("2") do |config|
+  config.vm.box = "opensuse/Leap-15.4.x86_64"
+  config.vm.provision "shell", path: "bootstrap"
+
+  config.vm.provider :libvirt do |libvirt|
+    libvirt.cpus = 1
+    libvirt.memory = 1536
+  end
+
+  config.vm.synced_folder '.', '/vagrant', disabled: true
+
+  config.vm.provision "ansible" do |ansible|
+    # ansible.verbose = "vvvv"
+    ansible.config_file = "../../../ansible.cfg"
+    ansible.playbook = "../../../run_all.yml"
+  end
+
+end

tests/ansible/vagrant/experimental/ubuntu-14.04-trusty/Vagrantfile

@@ -0,0 +1,18 @@
+Vagrant.configure("2") do |config|
+  config.vm.box = "peru/ubuntu-14.04-server-amd64"
+  config.vm.box_version = "20190901.01"
+
+  config.vm.provider :libvirt do |libvirt|
+    libvirt.cpus = 1
+    libvirt.memory = 1536
+  end
+
+  config.vm.synced_folder '.', '/vagrant', disabled: true
+
+  config.vm.provision "ansible" do |ansible|
+    # ansible.verbose = "vvvv"
+    ansible.config_file = "../../../ansible.cfg"
+    ansible.playbook = "../../../run_all.yml"
+  end
+
+end

tests/ansible/vagrant/fedora-33/Vagrantfile

@@ -1,8 +1,8 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   # the official boxes seem to have issues with journalctl
   # config.vm.box = "fedora/33-cloud-base"
   config.vm.box = "generic/fedora33"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/fedora-33/skip

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+die() {
+    echo "$@" >&2
+    exit 1
+}
+
+[ "${DB_BACKEND}" = "mysql" ] && die "mysql role does not support this distribution"
+exit 0

tests/ansible/vagrant/fedora-34/Vagrantfile

@@ -1,8 +1,8 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   # the official boxes seem to have issues with journalctl
   # config.vm.box = "fedora/34-cloud-base"
   config.vm.box = "generic/fedora34"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/fedora-34/skip

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+die() {
+    echo "$@" >&2
+    exit 1
+}
+
+[ "${DB_BACKEND}" = "mysql" ] && die "mysql role does not support this distribution"
+exit 0

tests/ansible/vagrant/fedora-35/Vagrantfile

@@ -1,8 +1,8 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   # the official boxes seem to have issues with journalctl
   # config.vm.box = "fedora/35-cloud-base"
   config.vm.box = "generic/fedora35"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/fedora-35/skip

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+die() {
+    echo "$@" >&2
+    exit 1
+}
+
+[ "${DB_BACKEND}" = "mysql" ] && die "mysql role does not support this distribution"
+exit 0

tests/ansible/vagrant/fedora-36/Vagrantfile

@@ -1,8 +1,8 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   # the official boxes seem to have issues with journalctl
   # config.vm.box = "fedora/36-cloud-base"
   config.vm.box = "generic/fedora36"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/fedora-36/skip

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+die() {
+    echo "$@" >&2
+    exit 1
+}
+
+[ "${DB_BACKEND}" = "mysql" ] && die "mysql role does not support this distribution"
+exit 0

tests/ansible/vagrant/freebsd-12/Vagrantfile

@@ -1,6 +1,6 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   config.vm.box = "generic/freebsd12"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/freebsd-12/skip

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+die() {
+    echo "$@" >&2
+    exit 1
+}
+
+[ "${PACKAGE_TESTING}" = "true" ] && die "no package available for this distribution"
+[ "${DB_BACKEND}" = "mysql" ] && die "mysql role does not support freebsd"
+exit 0

tests/ansible/vagrant/freebsd-13/Vagrantfile

@@ -1,6 +1,6 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   config.vm.box = "generic/freebsd13"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/freebsd-13/skip

@@ -0,0 +1,10 @@
+#!/bin/sh
+
+die() {
+    echo "$@" >&2
+    exit 1
+}
+
+[ "${PACKAGE_TESTING}" = "true" ] && die "no package available for this distribution"
+[ "${DB_BACKEND}" = "mysql" ] && die "mysql role does not support freebsd"
+exit 0

tests/ansible/vagrant/oracle-7/Vagrantfile

@@ -1,6 +1,6 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   config.vm.box = "generic/oracle7"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/oracle-7/skip

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+die() {
+    echo "$@" >&2
+    exit 1
+}
+
+# postgres is too old on this distribution
+[ "${DB_BACKEND}" = "postgres" ] && die "skipping: postgres too old"
+[ "${DB_BACKEND}" = "pgx" ] && die "skipping: postgres too old"
+exit 0

tests/ansible/vagrant/oracle-8/Vagrantfile

@@ -1,6 +1,6 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   config.vm.box = "generic/oracle8"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/oracle-9/Vagrantfile

@@ -0,0 +1,6 @@
+Vagrant.configure("2") do |config|
+  config.vm.box = "generic/oracle9"
+end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/rocky-8/Vagrantfile

@@ -0,0 +1,6 @@
+Vagrant.configure("2") do |config|
+  config.vm.box = "generic/rocky8"
+end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/rocky-9/Vagrantfile

@@ -1,6 +1,6 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   config.vm.box = "generic/rocky9"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/ubuntu-16.04-xenial/Vagrantfile

@@ -1,7 +1,7 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   # the official boxes only supports virtualbox
   config.vm.box = "generic/ubuntu1604"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/ubuntu-16.04-xenial/skip

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+die() {
+    echo "$@" >&2
+    exit 1
+}
+
+# postgres is too old on this distribution
+[ "${DB_BACKEND}" = "postgres" ] && die "skipping: postgres too old"
+[ "${DB_BACKEND}" = "pgx" ] && die "skipping: postgres too old"
+exit 0

tests/ansible/vagrant/ubuntu-18.04-bionic/Vagrantfile

@@ -1,7 +1,7 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   # the official boxes only supports virtualbox
   config.vm.box = "generic/ubuntu1804"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/ubuntu-20.04-focal/Vagrantfile

@@ -1,7 +1,7 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   # the official boxes only supports virtualbox
   config.vm.box = "generic/ubuntu2004"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/ansible/vagrant/ubuntu-22.04-jammy/Vagrantfile

@@ -1,7 +1,7 @@
-common = '../Vagrantfile.common'
-load common if File.exists?(common)
-
 Vagrant.configure("2") do |config|
   # the official boxes only supports virtualbox
   config.vm.box = "generic/ubuntu2204"
 end
+
+common = '../Vagrantfile.common'
+load common if File.exists?(common)

tests/assert-crowdsec-not-running

@@ -4,14 +4,12 @@ is_crowdsec_running() {
     PIDS=$(pgrep -x 'crowdsec|crowdsec.test|crowdsec.cover')
 }
 
-is_crowdsec_running || exit 0
-
 # The process can be slow, especially on CI and during test coverage.
 # Give it some time, maybe it's quitting soon.
-sleep 2
-is_crowdsec_running || exit 0
-sleep 2
+for _i in {1..10}; do
     is_crowdsec_running || exit 0
+    sleep .5
+done
 
 PIDS=$(echo "${PIDS}" | tr '\n' ' ')
 msg="CrowdSec is already running (PID ${PIDS}). Please terminate it and run the tests again."

tests/bats/01_base.bats

@@ -71,9 +71,10 @@ declare stderr
 }
 
 @test "cscli alerts list: at startup returns at least one entry: community pull" {
-    is_db_postgres && skip
-    # it should have been received while preparing the fixture
-    run -0 cscli alerts list -a -o json
+    run cscli alerts list -a -o json
+    if [[ "${status}" -ne 0 ]]; then
+        run cscli alerts list -o json
+    fi
     run -0 jq -r '. | length' <(output)
     refute_output 0
 

tests/bats/05_config_yaml_local.bats

@@ -128,11 +128,12 @@ teardown() {
     echo -e "---\nfilename: ${tmpfile}\nlabels:\n  type: syslog\n" >>"${ACQUIS_YAML}"
 
     ./instance-crowdsec start
+    sleep .5
     fake_log >>"${tmpfile}"
 
     # this could be simplified, but some systems are slow and we don't want to
     # wait more than required
-    for ((idx = 0; idx < 20; idx++)); do
+    for ((i=0;i<30;i++)); do
         sleep .5
         run -0 --separate-stderr cscli decisions list -o json
         run -0 jq --exit-status '.[].decisions[0] | [.value,.type] == ["1.1.1.172","captcha"]' <(output) && break

tests/check-requirements

@@ -32,7 +32,6 @@ check_python3() {
 }
 
 check_jq() {
-    # shellcheck disable=SC2016
     if ! command -v jq >/dev/null; then
         die "Missing required program 'jq'"
     fi
@@ -62,22 +61,9 @@ check_yq() {
 }
 
 check_daemonizer() {
-    SYSTEM=$(uname -s)
-    case "${SYSTEM,,}" in
-        linux)
     if ! command -v daemonize >/dev/null; then
-                die "missing required program 'daemonize' (package 'daemonize')"
+        die "missing required program 'daemonize' (package 'daemonize' or 'https://github.com/bmc/daemonize')"
     fi
-            ;;
-        freebsd)
-            if ! command -v daemon >/dev/null; then
-                die "missing required program 'daemon'"
-            fi
-            ;;
-        *)
-            die "unsupported system: ${SYSTEM}"
-            ;;
-    esac
 }
 
 check_cfssl() {

tests/instance-mock-http

@@ -31,9 +31,11 @@ DAEMON_PID=${PID_DIR}/mock-http.pid
 
 start_instance() {
     [ $# -lt 1 ] && about
-    OUT_FILE="${LOG_DIR}/mock-http.out" \
-        DAEMON_PID="${DAEMON_PID}" \
-        "${TEST_DIR}/run-as-daemon" /usr/bin/env python3 -u "${THIS_DIR}/mock-http.py" "$1"
+    daemonize \
+        -p "${DAEMON_PID}" \
+        -e "${LOG_DIR}/mock-http.err" \
+        -o "${LOG_DIR}/mock-http.out" \
+        /usr/bin/env python3 -u "${THIS_DIR}/mock-http.py" "$1"
     ./lib/util/wait-for-port "$1"
 #    echo "mock http started on port $1"
 }

tests/lib/config/config-global

@@ -39,14 +39,21 @@ export DATA_DIR
 CONFIG_DIR="${LOCAL_DIR}/${REL_CONFIG_DIR}"
 export CONFIG_DIR
 
+if [[ $(uname) == "OpenBSD" ]]; then
+    TAR=gtar
+else
+    TAR=tar
+fi
+
 remove_init_data() {
+    ./assert-crowdsec-not-running
     rm -rf -- "${LOCAL_DIR:?}/${REL_CONFIG_DIR}"/* "${LOCAL_DIR:?}/${REL_DATA_DIR:?}"/*
 }
 
 # we need a separate function for initializing config when testing package
 # because we want to test the configuration as well
 make_init_data() {
-    "${TEST_DIR}/instance-crowdsec" stop
+    ./assert-crowdsec-not-running
 
     ./instance-db config-yaml
     ./instance-db setup
@@ -57,15 +64,17 @@ make_init_data() {
     [[ "${DB_BACKEND}" == "sqlite" ]] || ${CSCLI} machines add --auto
 
     "${TEST_DIR}/instance-crowdsec" start
+    [[ "${DB_BACKEND}" =~ ^postgres|pgx$ ]] && sleep 4
+    "${CSCLI}" lapi status
 
-    for ((i=0; i<10; i++)); do
-        sleep .5
-        "${CSCLI}" decisions delete --all && break
-    done
+    # a restart is required to receive community pull
+    "${TEST_DIR}/instance-crowdsec" stop
+    sleep 2
+    "${TEST_DIR}/instance-crowdsec" start
 
     for ((i=0; i<15; i++)); do
         sleep 2
-        [[ $("${CSCLI}" alerts list -a -o json) != "null" ]] && break
+        [[ $("${CSCLI}" alerts list -a -o json 2>/dev/null || "${CSCLI}" alerts list -o json) != "null" ]] && break
     done
     # shellcheck disable=SC2181
     [[ "$?" -ne "0" ]] && die "could not get community data"
@@ -81,14 +90,14 @@ make_init_data() {
     # disable CAPI by default
     yq e 'del(.api.server.online_client)' -i "${CONFIG_DIR}/config.yaml"
 
-    tar -C "${LOCAL_DIR}" --create \
+    "${TAR}" -C "${LOCAL_DIR}" --create \
         --exclude "${REL_DATA_DIR}"/crowdsec.db \
         --file "${LOCAL_INIT_DIR}/init-config-data.tar" "${REL_CONFIG_DIR}" "${REL_DATA_DIR}"
-
-    ./instance-db setup
 }
 
 load_init_data() {
+    ./assert-crowdsec-not-running
+
     if [[ ! -f "${LOCAL_INIT_DIR}/init-config-data.tar" ]]; then
         die "Initial data not found; did you run '${script_name} make' ?"
     fi
@@ -100,7 +109,7 @@ load_init_data() {
 
     remove_init_data
 
-    tar -C "${LOCAL_DIR}" --extract --file "${LOCAL_INIT_DIR}/init-config-data.tar"
+    "${TAR}" -C "${LOCAL_DIR}" --extract --file "${LOCAL_INIT_DIR}/init-config-data.tar"
 
     ./instance-db restore "${LOCAL_INIT_DIR}/database"
 }
@@ -112,14 +121,13 @@ load_init_data() {
 
 case "$1" in
     make)
+        "${TEST_DIR}/instance-crowdsec" stop
         make_init_data
         ;;
     load)
-        ./assert-crowdsec-not-running
         load_init_data
         ;;
     clean)
-        ./assert-crowdsec-not-running
         remove_init_data
         ;;
     *)

tests/lib/config/config-local

@@ -39,7 +39,14 @@ export DATA_DIR
 CONFIG_DIR="${LOCAL_DIR}/${REL_CONFIG_DIR}"
 export CONFIG_DIR
 
+if [[ $(uname) == "OpenBSD" ]]; then
+    TAR=gtar
+else
+    TAR=tar
+fi
+
 remove_init_data() {
+    ./assert-crowdsec-not-running
     rm -rf -- "${LOCAL_DIR:?}/${REL_CONFIG_DIR}"/* "${LOCAL_DIR:?}/${REL_DATA_DIR:?}"/*
 }
 
@@ -83,13 +90,14 @@ config_generate() {
 
 
 make_init_data() {
-    remove_init_data
+    ./assert-crowdsec-not-running
 
+    remove_init_data
     mkdir -p "${DATA_DIR}"
     mkdir -p "${CONFIG_DIR}/notifications"
     mkdir -p "${CONFIG_DIR}/hub"
     mkdir -p "${CONFIG_DIR}/patterns"
-    cp -ax "../config/patterns" "${CONFIG_DIR}/"
+    cp -a "../config/patterns" "${CONFIG_DIR}/"
     config_generate
     # XXX errors from instance-db should be reported...
     ./instance-db config-yaml
@@ -109,24 +117,25 @@ make_init_data() {
     sleep 2
     "${TEST_DIR}/instance-crowdsec" start
 
-    loop_max=15
-    for ((i = 0; i <= loop_max; i++)); do
+    for ((i=0; i<15; i++)); do
         sleep 2
-        [[ $("${CSCLI}" alerts list -o json) != "null" ]] && break
+        [[ $("${CSCLI}" alerts list -a -o json 2>/dev/null || "${CSCLI}" alerts list -o json) != "null" ]] && break
     done
-    [[ $("${CSCLI}" alerts list -a -o json) != "null" ]] || die "could not get community data"
+    # shellcheck disable=SC2181
+    [[ "$?" -ne "0" ]] && die "could not get community data"
 
     "${TEST_DIR}/instance-crowdsec" stop
 
     mkdir -p "${LOCAL_INIT_DIR}"
 
     ./instance-db dump "${LOCAL_INIT_DIR}/database"
+
     echo "${DB_BACKEND}" > "${LOCAL_INIT_DIR}/.backend"
 
     # disable CAPI by default
     yq e 'del(.api.server.online_client)' -i "${CONFIG_DIR}/config.yaml"
 
-    tar -C "${LOCAL_DIR}" --create \
+    "${TAR}" -C "${LOCAL_DIR}" --create \
         --exclude "${REL_DATA_DIR}"/crowdsec.db \
         --file "${LOCAL_INIT_DIR}/init-config-data.tar" "${REL_CONFIG_DIR}" "${REL_DATA_DIR}"
 
@@ -134,6 +143,8 @@ make_init_data() {
 }
 
 load_init_data() {
+    ./assert-crowdsec-not-running
+
     if [[ ! -f "${LOCAL_INIT_DIR}/init-config-data.tar" ]]; then
         die "Initial data not found; did you run '${script_name} make' ?"
     fi
@@ -145,7 +156,8 @@ load_init_data() {
 
     remove_init_data
 
-    tar -C "${LOCAL_DIR}" --extract --file "${LOCAL_INIT_DIR}/init-config-data.tar"
+    "${TAR}" -C "${LOCAL_DIR}" --extract --file "${LOCAL_INIT_DIR}/init-config-data.tar"
+
     ./instance-db restore "${LOCAL_INIT_DIR}/database"
 }
 
@@ -156,15 +168,12 @@ load_init_data() {
 
 case "$1" in
     make)
-        ./assert-crowdsec-not-running
         make_init_data
         ;;
     load)
-        ./assert-crowdsec-not-running
         load_init_data
         ;;
     clean)
-        ./assert-crowdsec-not-running
         remove_init_data
         ;;
     *)

tests/lib/init/crowdsec-daemon

@@ -31,9 +31,11 @@ fi
 DAEMON_PID=${PID_DIR}/crowdsec.pid
 
 start() {
-    OUT_FILE="${LOG_DIR}/crowdsec.out" \
-            DAEMON_PID="${DAEMON_PID}" \
-            "${TEST_DIR}/run-as-daemon" "${CROWDSEC}"
+    daemonize \
+        -p "${DAEMON_PID}" \
+        -e "${LOG_DIR}/crowdsec.err" \
+        -o "${LOG_DIR}/crowdsec.out" \
+        "${CROWDSEC}"
     ./lib/util/wait-for-port 6060
 }
 

tests/run-as-daemon

@@ -1,27 +0,0 @@
-#!/usr/bin/env bash
-
-SYSTEM=$(uname -s)
-
-die() {
-    echo >&2 "$@"
-    exit 1
-}
-
-[[ -n "${DAEMON_PID}" ]] || die "\$DAEMON_PID is required and must be the path of the pid file"
-[[ -n "${OUT_FILE}" ]] || die "\$OUT_FILE is required and must be the path of the resulting stdout"
-
-# Simplified dudeist daemonizer. Don't care about lock files, separate
-# stdout/stderr and fancy stuff. #YOLO
-
-case "${SYSTEM,,}" in
-    linux)
-        daemonize -p "${DAEMON_PID}" -e "${OUT_FILE}" -o "${OUT_FILE}" "$@"
-        ;;
-    freebsd)
-        daemon -p "${DAEMON_PID}" -o "${OUT_FILE}" "$@"
-        ;;
-    *)
-        die "unsupported system: ${SYSTEM}"
-        ;;
-esac
-