use ansible roles from crowdsecurity.testing collection (#1743)

debian/templates

@@ -4,7 +4,7 @@ Default: true
 Description: Do you want to run the local API server ?
  A local API is required to run crowdsec, but another installation can be used. 
  .
- If you don't know what to do, consider answer yes.
+ If you don't know what to do, answer yes.
 
 Template: crowdsec/lapi_host
 Type: string
@@ -20,4 +20,4 @@ Default: true
 Description: Do you want to the centralized remote API server ?
  To share information with other crowdsec you can register to the centralized remote API server.
  .
- If you don't know what to do, consider answer yes. 
+ If you don't know what to do, answer yes.

tests/ansible/README.md

@@ -11,7 +11,7 @@ operating systems, or architectures.
 The ansible hosts should be expendable machines with at least 1GB RAM, do not
 expect them to be stable if you use them for anything else after the tests.
 
-Install the requirements with `ansible-galaxy install -r requiements.yml`.
+Install (or update) the requirements with `ansible-galaxy install -r requirements.yml --force`.
 
 There are several Ansible playbooks. You can use `run-all.yml` to configure the
 installation and run the tests, or run the playbooks separately to iterate while developing.
@@ -20,9 +20,11 @@ installation and run the tests, or run the playbooks separately to iterate while
 
 - provision-dependencies.yml: install the bats requirements (bash, netcat, cfssl, etc.), compilers, and database.
 
-- provision-test-suite.yml: install the tests scripts and bats environment.
+- provision-test-suite.yml: install the tests scripts and bats environment, and the crowdsec sources if we want to build the `crowdsec under test`.
 
-- prepare-tests.yml: install the package under test, and create the test fixture data.
+- install_binary_package.yml: install the `crowdsec under test` from a binary package (already released or not).
+
+- prepare-tests.yml: create the test fixture data.
 
 - run-tests.yml: run the functional tests. This is not idempotent and can be run multiple times.
 
@@ -67,7 +69,7 @@ ansible won't be able to see them.
 - `TEST_PACKAGE_DIR`: optional (but conflicts with `TEST_PACKAGE_FILE`), the path
   to a directory containing packages with the following layout:
 
-  For DEB: `{{ package_dir }}/{{ ansible_distribution_release }}/{{ ansible_architecture.replace('x86_64', 'amd64' }}/crowdsec_*.deb`
+  For DEB: `{{ package_dir }}/{{ ansible_distribution_release }}/crowdsec_*_{{ ansible_architecture.replace('x86_64', 'amd64') }}.deb`
   For RPM: `{{ package_dir }}/{{ releasever }}/RPMS/{{ ansible_architecture }}/crowdsec-*.{{ releasever }}.{{ ansible_architecture }}.rpm`
 
 - `TEST_SKIP`: optional, comma-separated list of scripts that won't be executed.
@@ -130,6 +132,7 @@ The data was created with crowdsec v1.4.1.
 | AmazonLinux 2             | ✓ (1)         | ✓ (1)      | old-db          | old-db     | wip              |
 | CentOS 7                  | ✓             | ✓          | old-db          | old-db     | ✓                |
 | CentOS 8                  | ✓             | ✓          | ✓               | ✓          | ✓                |
+| CentOS 9                  | ✓             | ✓          | ✓               | ✓          | ✓                |
 | Debian 9 (stretch)        | ✓             | ✓          | old-db          | old-db     | wip              |
 | Debian 10 (buster)        | ✓             | ✓          | ✓               | ✓          | ✓                |
 | Debian 11 (bullseye)      | ✓             | ✓          | ✓               | ✓          | ✓                |
@@ -150,9 +153,11 @@ The data was created with crowdsec v1.4.1.
 
 Note: all tests with `local/<database>` are expected to pass for `pkg/<database>` as well.
 
-wip - missing ansible or bats parts, working on it
+wip - missing ansible or bats parts, could be fixed in a future release
 
-old-db - the database that ships with the distribution is not supported (Postgres < 10)
+old-db - the database that ships with the distribution is not supported
+(Postgres < 10). Won't fix, feel free to install the DB from an unofficial
+repository.
 
 0 - MySQL or MariaDB, depending on distribution defaults
 

tests/ansible/install_binary_package.yml

@@ -0,0 +1,112 @@
+# vim: set ft=yaml.ansible:
+---
+
+- name: "Install and set up binary crowdsec package..."
+  hosts: all
+  gather_facts: true
+  tasks:
+
+    - name: "Hardcode master branch for the hub, temporary override before install (config.yaml.local)"
+      become: true
+      block:
+        - name: "Create /etc/crowdsec"
+          ansible.builtin.file:
+            path: "/etc/crowdsec"
+            state: directory
+            mode: 0o0755
+        - name: "Create /etc/crowdsec/config.yaml.local"
+          ansible.builtin.copy:
+            dest: "/etc/crowdsec/config.yaml.local"
+            content: "{{ config_yaml_local | to_nice_yaml }}"
+            mode: 0o600
+          vars:
+            config_yaml_local:
+              cscli:
+                hub_branch: master
+      when:
+        - (package_version_deb | length > 0) or
+          (package_version_rpm | length > 0) or
+          (package_file | length > 0) or
+          (package_dir | length > 0)
+
+    - name: "Install crowdsec binaries from a binary repository"
+      ansible.builtin.include_role:
+        name: crowdsecurity.testing.install_package_from_repo
+      when: (package_version_deb | length > 0) or
+            (package_version_rpm | length > 0)
+
+    - name: "Install crowdsec binaries from a package file"
+      ansible.builtin.include_role:
+        name: crowdsecurity.testing.install_package_from_file
+      when: package_file | length > 0
+
+    - name: "Install crowdsec binaries from a package directory"
+      ansible.builtin.include_role:
+        name: crowdsecurity.testing.install_package_from_pkgdir
+      when: package_dir | length > 0
+
+    - name: "Hardcode master branch for the hub, for real this time"
+      become: true
+      block:
+        - name: "Read config.yaml"
+          ansible.builtin.slurp:
+            path: "/etc/crowdsec/config.yaml"
+          register: config_yaml
+        - name: "Create fact from config.yaml"
+          ansible.builtin.set_fact:
+            config_data: "{{ config_yaml['content'] | b64decode | from_yaml }}"
+        - name: "Patch dictionary"
+          ansible.builtin.set_fact:
+            config_data: "{{ config_data | combine(config_patch, recursive=True) }}"
+          vars:
+            config_patch:
+              cscli:
+                hub_branch: master
+        - name: "Write patched config.yaml"
+          ansible.builtin.copy:
+            content: '{{ config_data | to_nice_yaml }}'
+            dest: "/etc/crowdsec/config.yaml"
+            # preserve mode to be able to test permissions from package
+            mode: preserve
+        - name: "Remove config.yaml.local"
+          ansible.builtin.file:
+            path: "/etc/crowdsec/config.yaml.local"
+            state: absent
+      when:
+        - (package_version_deb | length > 0) or
+          (package_version_rpm | length > 0) or
+          (package_file | length > 0) or
+          (package_dir | length > 0)
+
+    # this is required to avoid fatal errors in case systemctl is not working
+    # (which happens on some aws instances)
+    - name: "Override acquis.yaml for package testing"
+      become: true
+      ansible.builtin.copy:
+        dest: "/etc/crowdsec/acquis.yaml"
+        content: "{{ acquis_yaml | to_nice_yaml }}"
+        mode: preserve
+      vars:
+        acquis_yaml:
+          filenames:
+            - /tmp/should-not-exist.log
+          labels:
+            type: syslog
+            force_inotify: true
+      when:
+        - (package_version_deb | length > 0) or
+          (package_version_rpm | length > 0) or
+          (package_file | length > 0) or
+          (package_dir | length > 0)
+
+  vars:
+    package_version_deb: >-
+      {{ lookup('ansible.builtin.env', 'TEST_PACKAGE_VERSION_DEB') }}
+    package_version_rpm: >-
+      {{ lookup('ansible.builtin.env', 'TEST_PACKAGE_VERSION_RPM') }}
+    package_file: >-
+      {{ lookup('ansible.builtin.env', 'TEST_PACKAGE_FILE') }}
+    package_dir: >-
+      {{ lookup('ansible.builtin.env', 'TEST_PACKAGE_DIR') }}
+    binary_package_name: >-
+      crowdsec

tests/ansible/prepare_tests.yml

@@ -1,21 +1,21 @@
 # vim: set ft=yaml.ansible:
 ---
-- name: "prepare functional tests"
+
+- name: "Prepare fixture for the functional tests"
   hosts: all
   gather_facts: true
   vars_files:
     - vars/go.yml
     - vars/mysql.yml
     - vars/postgres.yml
-  environment:
-    PGHOST: 127.0.0.1
-    PGPORT: 5432
-    PGPASSWORD: "{{ postgresql_users[0].password }}"
-    PGUSER: postgres
-    MYSQL_HOST: localhost
-    MYSQL_PORT: 3306
-    MYSQL_PASSWORD: "{{ mysql_root_password }}"
-    MYSQL_USER: "root"
   roles:
-    - role: install_crowdsec_package
-    - role: make_fixture
+    - name: make_fixture
+      environment:
+        PGHOST: 127.0.0.1
+        PGPORT: 5432
+        PGPASSWORD: "{{ postgresql_users[0].password }}"
+        PGUSER: postgres
+        MYSQL_HOST: localhost
+        MYSQL_PORT: 3306
+        MYSQL_PASSWORD: "{{ mysql_root_password }}"
+        MYSQL_USER: "root"

tests/ansible/provision_dependencies.yml

@@ -1,51 +1,21 @@
 # vim: set ft=yaml.ansible:
 ---
-- name: "install Go (!bsd)"
-  hosts: all
-  gather_facts: true
-  vars_files:
-    - vars/go.yml
-  tasks:
-    - ansible.builtin.include_role:
-        name: gantsign.golang
-      when:
-        - ansible_facts.system == 'Linux'
-        - ansible_facts.distribution != 'Alpine'
 
-- name: "install Go (bsd, alpine)"
-  hosts: all
-  gather_facts: true
-  become: true
-  tasks:
-    - ansible.builtin.package:
-        name: go
-        state: present
-      when:
-        - ansible_facts.system in ['FreeBSD', 'OpenBSD'] or ansible_facts.distribution == 'Alpine'
-
-# required for jq and basic tools on centos and rhel
-- name: "enable EPEL repository (RedHat)"
-  hosts: all
-  become: true
-  tasks:
-    - ansible.builtin.include_role:
-        name: geerlingguy.repo-epel
-      vars:
-        epel_repo_disable: false
-      when:
-        - ansible_facts.os_family == 'RedHat'
-        - (ansible_facts.distribution != 'Fedora') and (ansible_facts.distribution != 'Amazon')
-
-- name: "apply common configuration to all nodes"
+- name: "Install required packages"
   hosts: all
   vars_files:
     - vars/go.yml
   roles:
-    - common
-    - machine_id
-    - bats_requirements
+    - crowdsecurity.testing.apt_update
+    - crowdsecurity.testing.go
+    - crowdsecurity.testing.machine_id
+    - crowdsecurity.testing.epel
+    - crowdsecurity.testing.git
+    - crowdsecurity.testing.gcc
+    - crowdsecurity.testing.gnu_make
+    - crowdsecurity.testing.bats_requirements
 
-- name: "install Postgres"
+- name: "Install Postgres"
   hosts: all
   become: true
   vars_files:
@@ -53,13 +23,13 @@
   tasks:
     - ansible.builtin.include_role:
         name: geerlingguy.postgresql
-      # enable this for debugging
-      #vars:
+      ## enable this for debugging
+      # vars:
       #  postgres_users_no_log: false
       when:
         - lookup('ansible.builtin.env', 'DB_BACKEND') in ['pgx', 'postgres']
 
-- name: "install MySQL"
+- name: "Install MySQL"
   hosts: all
   become: true
   vars_files:

tests/ansible/provision_test_suite.yml

@@ -1,6 +1,34 @@
 # vim: set ft=yaml.ansible:
 ---
-- name: "install the test scripts"
+
+- name: "Fetch the test scripts"
   hosts: all
-  roles:
-    - install_crowdsec_tests
+  tasks:
+
+    - name: "Fetch the sources from a git repository"
+      ansible.builtin.include_role:
+        name: crowdsecurity.testing.download_sources_from_git
+      when: sources_zip | length == 0
+
+    - name: "Extract the sources from a zip archive"
+      ansible.builtin.include_role:
+        name: crowdsecurity.testing.extract_sources_from_zip
+      when: sources_zip | length > 0
+
+    - name: "Create crowdsec tests/local dir"
+      become: false
+      ansible.builtin.file:
+        path: "{{ ansible_env.HOME }}/crowdsec/tests/local"
+        state: directory
+        mode: 0o755
+
+  vars:
+    sources_dest_dir: "{{ ansible_env.HOME }}/crowdsec"
+    sources_git_repo: >-
+      {{ lookup('ansible.builtin.env', 'TEST_SUITE_GIT')
+      | default('https://github.com/crowdsecurity/crowdsec', True) }}
+    sources_git_version: >-
+      {{ lookup('ansible.builtin.env', 'TEST_SUITE_VERSION')
+      | default('master', True) }}
+    sources_zip: >-
+      {{ lookup('ansible.builtin.env', 'TEST_SUITE_ZIP') }}

tests/ansible/requirements.yml

@@ -1,18 +1,18 @@
 # vim: set ft=yaml.ansible:
 ---
+
 roles:
   - src: geerlingguy.mysql
-  - src: geerlingguy.repo-epel
-  - src: gantsign.golang
   - src: https://github.com/crowdsecurity/ansible-role-postgresql
     version: crowdsec
     name: geerlingguy.postgresql
 
-#collections:
-#  - name: crowdsecurity.testing
-#    source: ~/src/crowdsecurity.testing/crowdsecurity/testing
-#    type: dir
+collections:
+  - name: https://github.com/crowdsecurity/ansible-collection-crowdsecurity.testing.git
+    type: git
+    version: main
 
 #  - name: crowdsecurity.testing
-#    src: https://github.com/crowdsecurity/crowdsecurity.testing
-#    version: main
+#    source: ../../../crowdsecurity.testing
+#    type: dir
+

tests/ansible/roles/bats_requirements/defaults/main.yml

@@ -1,6 +0,0 @@
-# vim: set ft=yaml.ansible:
----
-build_bash: false
-bash_version: 5.1.16
-
-build_daemonize: (ansible_facts.distribution == "Ubuntu" and ansible_facts.distribution == '16.04') or ansible_facts.distribution == 'Amazon'

tests/ansible/roles/bats_requirements/tasks/bash.yml

@@ -1,43 +0,0 @@
-# vim: set ft=yaml.ansible:
----
-- name: "check if we need to build bash"
-  become: false
-  block:
-    - name: "look up bash version"
-      ansible.builtin.package_facts:
-    - name: "bash version found"
-      ansible.builtin.debug:
-        var: ansible_facts.packages['bash'][0].version
-    - name: "check if bash is too old (<4.4)"
-      ansible.builtin.set_fact:
-        build_bash: "{{ ansible_facts.packages['bash'][0].version is version('4.4', '<') }}"
-
-- name: "build bash"
-  block:
-    - name: "build bash: download"
-      become: false
-      ansible.builtin.unarchive:
-        src: http://ftp.gnu.org/gnu/bash/bash-{{ bash_version }}.tar.gz
-        dest: "{{ ansible_env.HOME }}"
-        remote_src: true
-        creates: "{{ ansible_env.HOME }}/bash-{{ bash_version }}"
-    - name: "build bash: configure"
-      become: false
-      ansible.builtin.command:
-        cmd: "./configure --prefix=/opt/bash"
-        creates: ./Makefile
-        chdir: "{{ ansible_env.HOME }}/bash-{{ bash_version }}"
-    - name: "build bash: create /opt/bash"
-      become: true
-      ansible.builtin.file:
-        path: /opt/bash
-        state: directory
-        mode: 0o755
-    - name: "build bash: make install"
-      become: true
-      ansible.builtin.command:
-        cmd: "make install"
-        creates: /opt/bash/bin/bash
-        chdir: "{{ ansible_env.HOME }}/bash-{{ bash_version }}"
-  when:
-    - build_bash

tests/ansible/roles/bats_requirements/tasks/daemonize.yml

@@ -1,43 +0,0 @@
-# vim: set ft=yaml.ansible:
----
-- name: "install daemonize package"
-  become: true
-  ansible.builtin.package:
-    name:
-      - daemonize
-  when:
-    - not build_daemonize
-
-- name: "build daemonize"
-  block:
-    - name: "build daemonize: git checkout"
-      become: false
-      ansible.builtin.git:
-        repo: https://github.com/bmc/daemonize
-        dest: "{{ ansible_env.HOME }}/daemonize"
-        version: release-1.7.8
-    - name: "build daemonize: configure"
-      become: false
-      ansible.builtin.command:
-        cmd: "./configure --prefix=/usr/local"
-        creates: ./Makefile
-        chdir: "{{ ansible_env.HOME }}/daemonize"
-    - name: "build daemonize: make install (linux)"
-      become: true
-      ansible.builtin.command:
-        cmd: "make all install"
-        creates: /usr/local/sbin/daemonize
-        chdir: "{{ ansible_env.HOME }}/daemonize"
-      when:
-        - ansible_facts.system == 'Linux'
-    - name: "build daemonize: make install (bsd)"
-      become: true
-      ansible.builtin.command:
-        cmd: "gmake all install"
-        creates: /usr/local/sbin/daemonize
-        chdir: "{{ ansible_env.HOME }}/daemonize"
-      when:
-        - ansible_facts.system in ['FreeBSD', 'OpenBSD']
-  when:
-    - build_daemonize
-

tests/ansible/roles/bats_requirements/tasks/main.yml

@@ -1,79 +0,0 @@
-# vim: set ft=yaml.ansible:
----
-- name: "install bash"
-  ansible.builtin.import_tasks: bash.yml
-  when:
-    # openbsd is not supported by the package_facts module, let's assume bash is ok
-    - ansible_facts.system != 'OpenBSD'
-
-- name: "Install daemonize"
-  ansible.builtin.import_tasks: daemonize.yml
-
-- name: "install netcat"
-  ansible.builtin.import_tasks: netcat.yml
-
-- name: "Install curl, jq, openssl, python3"
-  become: true
-  ansible.builtin.package:
-    name:
-      - curl
-      - jq
-      - openssl
-      - python3
-  when:
-    - ansible_facts.distribution != 'Gentoo'
-
-- name: "install bc (!freebsd)"
-  become: true
-  ansible.builtin.package:
-    name:
-      - bc
-  when:
-    - ansible_facts.system == 'Linux'
-
-# base64 for linux is in coreutils
-- name: "install base64 (bsd)"
-  become: true
-  ansible.builtin.package:
-    name:
-      - base64
-  when:
-    - ansible_facts.system in ['FreeBSD', 'OpenBSD']
-
-- name: "install pidof (Amazon)"
-  become: true
-  ansible.builtin.package:
-    name:
-      - procps-ng
-  when:
-    - ansible_facts.distribution == 'Amazon'
-
-- name: "install cfssl"
-  become: true
-  ansible.builtin.command:
-    cmd: "go install github.com/cloudflare/cfssl/cmd/cfssl@latest"
-    creates: /usr/bin/cfssl
-  environment:
-    GOBIN: /usr/bin
-    # make sure we use the built version of go, if there is one
-    PATH: "{{ golang_install_dir }}/bin:{{ ansible_env.PATH }}"
-
-- name: "install cfssljson"
-  become: true
-  ansible.builtin.command:
-    cmd: "go install github.com/cloudflare/cfssl/cmd/cfssljson@latest"
-    creates: /usr/bin/cfssljson
-  environment:
-    GOBIN: /usr/bin
-    # make sure we use the built version of go, if there is one
-    PATH: "{{ golang_install_dir }}/bin:{{ ansible_env.PATH }}"
-
-- name: "install yq"
-  become: true
-  ansible.builtin.command:
-    cmd: "go install github.com/mikefarah/yq/v4@latest"
-    creates: /usr/bin/yq
-  environment:
-    GOBIN: /usr/bin
-    # make sure we use the built version of go, if there is one
-    PATH: "{{ golang_install_dir }}/bin:{{ ansible_env.PATH }}"

tests/ansible/roles/bats_requirements/tasks/netcat.yml

@@ -1,26 +0,0 @@
-# vim: set ft=yaml.ansible:
----
-- name: "install netcat (Amazon, Fedora, CentOS, Oracle)"
-  become: true
-  ansible.builtin.package:
-    name:
-      - nmap-ncat
-  when:
-    - ansible_facts.distribution in ['Amazon',  'Fedora', 'CentOS', 'OracleLinux']
-
-- name: "install netcat (RedHat)"
-  become: true
-  ansible.builtin.package:
-    name:
-      - netcat
-  when:
-    - (ansible_facts.distribution == 'RedHat') or (ansible_facts.distribution == 'AlmaLinux') or (ansible_facts.distribution == 'Rocky')
-
-# "netcat" does not exist in some Debian versions (only -traditional or -openbsd)
-- name: "install netcat (Suse)"
-  become: true
-  ansible.builtin.package:
-    name:
-      - netcat-openbsd
-  when:
-    - ansible_facts.os_family in ["Debian", "Suse"]

tests/ansible/roles/common/tasks/main.yml

@@ -1,32 +0,0 @@
-# vim: set ft=yaml.ansible:
----
-- name: "update package cache (Debian)"
-  become: true
-  ansible.builtin.apt:
-    upgrade: false
-    update_cache: true
-  when:
-    - ansible_facts.os_family == "Debian"
-
-- name: "install gcc, git"
-  become: true
-  ansible.builtin.package:
-    name:
-      - gcc
-      - git
-
-- name: "install make (Linux)"
-  become: true
-  ansible.builtin.package:
-    name:
-      - make
-  when:
-    - ansible_facts.system == "Linux"
-
-- name: "install gmake (bsd)"
-  become: true
-  ansible.builtin.package:
-    name:
-      - gmake
-  when:
-    - ansible_facts.system in ['FreeBSD', 'OpenBSD']

tests/ansible/roles/install_crowdsec_package/tasks/install_from_deb.yml

@@ -1,35 +0,0 @@
-# vim: set ft=yaml.ansible:
----
-- name: "set package_file from package_dir"
-  ansible.builtin.set_fact:
-    package_file: "{{ package_dir }}/{{ ansible_facts.distribution_release }}/crowdsec_*{{ ansible_facts.architecture.replace('x86_64', 'amd64') }}.deb"
-  when:
-    - (package_dir is defined) and (package_dir | length > 0)
-
-- name: "install crowdsec from package_file"
-  become: true
-  block:
-    - name: "look for file matching package_file"
-      ansible.builtin.set_fact:
-        found_file: "{{ item }}"
-      with_fileglob:
-        - "{{ package_file }}"
-
-    - name: "check found_file"
-      ansible.builtin.fail:
-        msg: "No file found matching {{ package_file }}"
-      when:
-        - found_file is not defined
-
-    - name: "copy {{ found_file }}"
-      ansible.builtin.copy:
-        src: "{{ found_file }}"
-        dest: "/root/crowdsec.deb"
-        mode: 0o644
-
-    - name: "install crowdsec"
-      ansible.builtin.apt:
-        deb: "/root/crowdsec.deb"
-        allow_downgrade: true
-  when:
-    - (package_file is defined) and (package_file | length > 0)

tests/ansible/roles/install_crowdsec_package/tasks/install_from_deb_repo.yml

@@ -1,28 +0,0 @@
-# vim: set ft=yaml.ansible:
----
-- name: "install stuff"
-  become: true
-  ansible.builtin.package:
-    name:
-      - apt-transport-https
-      - gnupg
-
-- name: "install crowdsec from the package repository"
-  become: true
-  block:
-    - name: "stable apt repo key"
-      ansible.builtin.apt_key:
-        url: https://packagecloud.io/crowdsec/crowdsec/gpgkey
-    - name: "stable apt repo"
-      ansible.builtin.apt_repository:
-        repo: deb https://packagecloud.io/crowdsec/crowdsec/{{ ansible_facts.distribution | lower }}/ {{ ansible_facts.distribution_release }} main
-    - name: "testing apt repo key"
-      ansible.builtin.apt_key:
-        url: https://packagecloud.io/crowdsec/crowdsec-testing/gpgkey
-    - name: "testing apt repo"
-      ansible.builtin.apt_repository:
-        repo: deb https://packagecloud.io/crowdsec/crowdsec-testing/{{ ansible_facts.distribution | lower }}/ {{ ansible_facts.distribution_release }} main
-    - name: "install crowdsec {{ package_vesion_deb }} with apt"
-      ansible.builtin.package:
-        name:
-          - crowdsec={{ package_version_deb }}

tests/ansible/roles/install_crowdsec_package/tasks/install_from_rpm.yml

@@ -1,36 +0,0 @@
-# vim: set ft=yaml.ansible:
----
-- name: "set package_file from package_dir"
-  ansible.builtin.set_fact:
-    package_file: "{{ package_dir }}/{{ releasever }}/RPMS/{{ ansible_facts.architecture }}/crowdsec-*.{{ releasever }}.{{ ansible_facts.architecture }}.rpm"
-  when:
-    - (package_dir is defined) and (package_dir | length > 0)
-
-- name: "install crowdsec from package_file"
-  become: true
-  block:
-    - name: "look for file matching package_file"
-      ansible.builtin.set_fact:
-        found_file: "{{ item }}"
-      with_fileglob:
-        - "{{ package_file }}"
-
-    - name: "check found_file"
-      ansible.builtin.fail:
-        msg: "No file found matching {{ package_file }}"
-      when:
-        - found_file is not defined
-
-    - name: "copy {{ found_file }}"
-      ansible.builtin.copy:
-        src: "{{ found_file }}"
-        dest: "/root/crowdsec.rpm"
-        mode: 0o644
-
-    - name: "install crowdsec"
-      ansible.builtin.yum:
-        name: "/root/crowdsec.rpm"
-        disable_gpg_check: true
-        allow_downgrade: true
-  when:
-    - (package_file is defined) and (package_file | length > 0)

tests/ansible/roles/install_crowdsec_package/tasks/install_from_rpm_repo.yml

@@ -1,29 +0,0 @@
-# vim: set ft=yaml.ansible:
----
-- name: "install crowdsec from the package repository"
-  become: true
-  block:
-    - name: "stable rpm repo script"
-      ansible.builtin.get_url:
-        url: https://packagecloud.io/install/repositories/crowdsec/crowdsec/script.rpm.sh
-        dest: "/root/rpm.sh"
-        mode: 0o775
-    - name: "stable rpm repo"
-      ansible.builtin.command:
-        cmd: "/root/rpm.sh"
-      changed_when: true
-
-    - name: "testing rpm repo script"
-      ansible.builtin.get_url:
-        url: https://packagecloud.io/install/repositories/crowdsec/crowdsec-testing/script.rpm.sh
-        dest: "/root/rpm-testing.sh"
-        mode: 0o775
-    - name: "testing rpm repo"
-      ansible.builtin.command:
-        cmd: "/root/rpm-testing.sh"
-      changed_when: true
-
-    - name: "install crowdsec"
-      ansible.builtin.package:
-        name:
-          - crowdsec-{{ package_version_rpm }}.{{ releasever.replace('amzn2', 'el7').replace('ol7', 'el7').replace('ol8', 'el8') }}

tests/ansible/roles/install_crowdsec_package/tasks/main.yml

@@ -1,113 +0,0 @@
-# vim: set ft=yaml.ansible:
----
-- name: "set releasever for RedHat family"
-  ansible.builtin.set_fact:
-    releasever: "{{ release[ansible_facts.distribution] + ansible_facts.distribution_major_version }}"
-  vars:
-    release:
-      AlmaLinux: el
-      Amazon: amzn
-      CentOS: el
-      Fedora: fc
-      OracleLinux: ol
-      Rocky: el
-  when:
-    - ansible_facts.os_family == "RedHat"
-
-- name: "system details"
-  ansible.builtin.debug:
-    msg: |
-      Distribution: {{ ansible_facts.distribution }}
-      Version: {{ ansible_facts.distribution_version }}
-      Major: {{ ansible_facts.distribution_major_version | default('n/a') }}
-      Release: {{ ansible_facts.distribution_release }}
-      Releasever: {{ releasever | default('n/a') }}
-
-- name: "hardcode master branch for the hub, temporary override before install (config.yaml.local)"
-  become: true
-  block:
-    - name: "create /etc/crowdsec"
-      ansible.builtin.file:
-        path: "/etc/crowdsec"
-        state: directory
-        mode: 0o0755
-    - name: "create /etc/crowdsec/config.yaml.local"
-      ansible.builtin.copy:
-        dest: "/etc/crowdsec/config.yaml.local"
-        content: "{{ config_yaml_local | to_nice_yaml }}"
-        mode: 0o600
-      vars:
-        config_yaml_local:
-          cscli:
-            hub_branch: master
-  when:
-    - (package_testing is defined) and (package_testing not in ['', 'false', 'False'])
-
-- name: "install from binary repository (RedHat)"
-  ansible.builtin.import_tasks: install_from_rpm_repo.yml
-  when:
-    - (package_version_rpm is defined) and (package_version_rpm|length > 0)
-    - ansible_facts.os_family == "RedHat"
-
-- name: "install from binary repository (Debian)"
-  ansible.builtin.import_tasks: install_from_deb_repo.yml
-  when:
-    - (package_version_deb is defined) and (package_version_deb|length > 0)
-    - ansible_facts.os_family == "Debian"
-
-- name: "install from *.rpm package file"
-  ansible.builtin.import_tasks: install_from_rpm.yml
-  when:
-    - ansible_facts.os_family == "RedHat"
-
-- name: "install from *.deb package file"
-  ansible.builtin.import_tasks: install_from_deb.yml
-  when:
-    - ansible_facts.os_family == "Debian"
-
-- name: "hardcode master branch for the hub, for real this time"
-  become: true
-  block:
-    - name: "read config.yaml"
-      ansible.builtin.slurp:
-        path: "/etc/crowdsec/config.yaml"
-      register: config_yaml
-    - name: "create fact from config.yaml"
-      ansible.builtin.set_fact:
-        config_data: "{{ config_yaml['content'] | b64decode | from_yaml }}"
-    - name: "patch dictionary"
-      ansible.builtin.set_fact:
-        config_data: "{{ config_data | combine(config_patch, recursive=True) }}"
-      vars: 
-        config_patch:
-          cscli:
-            hub_branch: master
-    - name: "write patched config.yaml"
-      ansible.builtin.copy:
-        content: '{{ config_data | to_nice_yaml }}'
-        dest: "/etc/crowdsec/config.yaml"
-        # preserve mode to be able to test permissions from package
-        mode: preserve
-    - name: "remove config.yaml.local"
-      ansible.builtin.file:
-        path: "/etc/crowdsec/config.yaml.local"
-        state: absent
-  when:
-    - (package_testing is defined) and (package_testing not in ['', 'false', 'False'])
-
-# this is required to avoid fatal errors in case systemctl is not working (which happens on some aws instances)
-- name: "override acquis.yaml for package testing"
-  become: true
-  ansible.builtin.copy:
-    dest: "/etc/crowdsec/acquis.yaml"
-    content: "{{ acquis_yaml | to_nice_yaml }}"
-    mode: preserve
-  vars:
-    acquis_yaml:
-      filenames:
-        - /tmp/should-not-exist.log
-      labels:
-        type: syslog
-        force_inotify: true
-  when:
-    - (package_testing is defined) and (package_testing not in ['', 'false', 'False'])

tests/ansible/roles/install_crowdsec_package/vars/main.yml

@@ -1,9 +0,0 @@
-# vim: set ft=yaml.ansible:
----
-
-package_version_deb: "{{ lookup('ansible.builtin.env', 'TEST_PACKAGE_VERSION_DEB') }}"
-package_version_rpm: "{{ lookup('ansible.builtin.env', 'TEST_PACKAGE_VERSION_RPM') }}"
-package_file: "{{ lookup('ansible.builtin.env', 'TEST_PACKAGE_FILE') }}"
-package_dir: "{{ lookup('ansible.builtin.env', 'TEST_PACKAGE_DIR') }}"
-package_testing: "{{ lookup('ansible.builtin.env', 'PACKAGE_TESTING') }}"
-

tests/ansible/roles/install_crowdsec_tests/defaults/main.yml

@@ -1,4 +0,0 @@
-# vim: set ft=yaml.ansible:
----
-suite_git: "https://github.com/crowdsecurity/crowdsec"
-suite_version: "master"

tests/ansible/roles/install_crowdsec_tests/tasks/main.yml

@@ -1,75 +0,0 @@
-# vim: set ft=yaml.ansible:
----
-- name: "lookup $TEST_SUITE_GIT"
-  ansible.builtin.set_fact:
-    suite_git: "{{ lookup('ansible.builtin.env', 'TEST_SUITE_GIT') }}"
-  when: lookup('ansible.builtin.env', 'TEST_SUITE_GIT') | length>0
-
-- name: "lookup $TEST_SUITE_VERSION"
-  ansible.builtin.set_fact:
-    suite_version: "{{ lookup('ansible.builtin.env', 'TEST_SUITE_VERSION') }}"
-  when: lookup('ansible.builtin.env', 'TEST_SUITE_VERSION') | length>0
-
-- name: "lookup $TEST_SUITE_ZIP"
-  ansible.builtin.set_fact:
-    suite_zip: "{{ lookup('ansible.builtin.env', 'TEST_SUITE_ZIP') }}"
-  when: lookup('ansible.builtin.env', 'TEST_SUITE_ZIP') | length>0
-
-- name: "install tests from zip file"
-  block:
-    - name: "install unzip"
-      become: true
-      ansible.builtin.package:
-        name:
-          - unzip
-
-    - name: "install tests: create source dir"
-      become: false
-      ansible.builtin.file:
-        path: "{{ ansible_env.HOME }}/crowdsec"
-        state: directory
-        mode: 0o700
-
-    - name: "install tests: extract crowdsec"
-      become: false
-      ansible.builtin.unarchive:
-        src: "{{ suite_zip }}"
-        dest: "{{ ansible_env.HOME }}/crowdsec"
-
-    - name: "install tests: git submodules for bats"
-      become: false
-      ansible.builtin.command:
-        cmd: "{{ item }}"
-        chdir: "{{ ansible_env.HOME }}/crowdsec"
-      with_items:
-        - git submodule init
-        - git submodule update
-  when:
-    - (suite_zip is defined) and (suite_zip|length > 0)
-
-
-- name: "install tests from repository"
-  block:
-    - name: "install tests: checkout crowdsec"
-      become: false
-      ansible.builtin.git:
-        repo: "{{ suite_git }}"
-        dest: "{{ ansible_env.HOME }}/crowdsec"
-        version: "{{ suite_version }}"
-
-    # trust the dir if we need to test as root
-    - name: set safe.directory on crowdsec dir"
-      become: true
-      community.general.git_config:
-        scope: global
-        name: safe.directory
-        value: "{{ ansible_env.HOME }}/crowdsec"
-  when:
-    - (suite_zip is not defined) or (suite_zip|length == 0)
-
-- name: "install tests: create crowdsec tests/local dir"
-  become: false
-  ansible.builtin.file:
-    path: "{{ ansible_env.HOME }}/crowdsec/tests/local"
-    state: directory
-    mode: 0o755

tests/ansible/roles/machine_id/defaults/main.yml

@@ -1,3 +0,0 @@
-# vim: set ft=yaml.ansible:
----
-machine_id: "githubciXXXXXXXXXXXXXXXXXXXXXXXX\n"

tests/ansible/roles/machine_id/tasks/main.yml

@@ -1,24 +0,0 @@
-# vim: set ft=yaml.ansible:
----
-- name: "set /etc/machine-id"
-  become: true
-  ansible.builtin.copy:
-    dest: /etc/machine-id
-    content: "{{ machine_id }}"
-    mode: 0o444
-
-- name: "see if /var/lib/dbus exists"
-  ansible.builtin.stat:
-    path: "/var/lib/dbus"
-  register: dbus
-
-# some distributions use var/lib/dbus,
-# but the directory doesn't exists on fedora
-- name: "set /var/lib/dbus/machine-id"
-  become: true
-  ansible.builtin.copy:
-    dest: /var/lib/dbus/machine-id
-    content: "{{ machine_id }}"
-    mode: 0o444
-  when:
-    - dbus.stat.exists

tests/ansible/roles/make_fixture/tasks/main.yml

@@ -1,21 +1,21 @@
 # vim: set ft=yaml.ansible:
 ---
-- name: "set make_cmd = make (!bsd)"
+- name: "Set make_cmd = make (!bsd)"
   ansible.builtin.set_fact:
     make_cmd: make
   when:
     - ansible_facts.system not in ['FreeBSD', 'OpenBSD']
 
-- name: "set make_cmd = gmake (bsd)"
+- name: "Set make_cmd = gmake (bsd)"
   ansible.builtin.set_fact:
     make_cmd: gmake
   when:
     - ansible_facts.system in ['FreeBSD', 'OpenBSD']
 
-- name: "build crowdsec from sources, prepare test environment and fixture"
+- name: "Build crowdsec from sources, prepare test environment and fixture"
   become: false
   block:
-    - name: "make bats-build bats-fixture"
+    - name: "Make bats-build bats-fixture"
       ansible.builtin.command:
         cmd: "{{ make_cmd }} bats-build bats-fixture"
         chdir: "{{ ansible_env.HOME }}/crowdsec"
@@ -27,19 +27,19 @@
         # bash -> /opt/bash/bin
         PATH: "/opt/bash/bin:{{ ansible_env.PATH }}:{{ golang_install_dir }}/bin/:/usr/sbin:/usr/local/sbin"
   rescue:
-    - name: "read crowdsec.log"
+    - name: "Read crowdsec.log"
       ansible.builtin.slurp:
         path: "{{ ansible_env.HOME }}/crowdsec/tests/local/var/log/crowdsec.log"
       register: crowdsec_log
-    - name: "show crowdsec.log"
+    - name: "Show crowdsec.log"
       ansible.builtin.fail:
         msg: "{{ crowdsec_log['content'] | b64decode }}"
   when: (package_testing is not defined) or (package_testing in ['', 'false', 'False'])
 
-- name: "prepare test environment and fixture for binary package"
+- name: "Prepare test environment and fixture for binary package"
   become: true
   block:
-    - name: "make bats-environment bats-check-requirements bats-fixture"
+    - name: "Make bats-environment bats-check-requirements bats-fixture"
       ansible.builtin.command:
         cmd: "{{ make_cmd }} bats-environment bats-check-requirements bats-fixture"
         chdir: "{{ ansible_env.HOME }}/crowdsec"
@@ -52,26 +52,26 @@
         # bash -> /opt/bash/bin
         PATH: "/opt/bash/bin:{{ ansible_env.PATH }}:/usr/sbin:/usr/local/sbin"
   rescue:
-    - name: "read crowdsec.log"
+    - name: "Read crowdsec.log"
       ansible.builtin.slurp:
         path: "/var/log/crowdsec.log"
       register: crowdsec_log
-    - name: "show crowdsec.log"
+    - name: "Show crowdsec.log"
       ansible.builtin.fail:
         msg: "{{ crowdsec_log['content'] | b64decode }}"
   when: (package_testing is defined) and (package_testing not in ['', 'false', 'False'])
 
-- name: "debug - show environment"
+- name: "Debug - show environment"
   become: false
   block:
-    - name: "look for .environment.sh"
+    - name: "Look for .environment.sh"
       ansible.builtin.slurp:
         src: "{{ ansible_env.HOME }}/crowdsec/tests/.environment.sh"
       changed_when: true
       register: envfile
-    - name: "cat .environment.sh"
+    - name: "Show .environment.sh"
       ansible.builtin.debug:
         msg: "{{ envfile['content'] | b64decode }}"
-    - name: "show environment variables"
+    - name: "Show environment variables"
       ansible.builtin.debug:
         msg: "{{ ansible_env | to_nice_yaml }}"

tests/ansible/roles/run_func_tests/tasks/main.yml

@@ -1,16 +1,16 @@
 # vim: set ft=yaml.ansible:
 ---
-- name: "tweak systemd configuration for tests"
+- name: "Tweak systemd configuration for tests"
   become: true
   block:
-    - name: "create /lib/systemd/system/crowdsec.service.d"
+    - name: "Create /lib/systemd/system/crowdsec.service.d"
       ansible.builtin.file:
         owner: root
         group: root
         mode: 0o755
         path: /lib/systemd/system/crowdsec.service.d
         state: directory
-    - name: "override StartLimitBurst"
+    - name: "Override StartLimitBurst"
       ansible.builtin.ini_file:
         dest: /lib/systemd/system/crowdsec.service.d/startlimitburst.conf
         owner: root
@@ -19,36 +19,36 @@
         section: Service
         option: StartLimitBurst
         value: 100
-    - name: "systemctl daemon-reload"
+    - name: "Systemctl daemon-reload"
       ansible.builtin.systemd:
         daemon_reload: true
   when:
     - (package_testing is defined) and (package_testing not in ['', 'false', 'False'])
     - ansible_facts.os_family in ["RedHat", "Debian"]
 
-- name: "debug - show environment.sh"
+- name: "Debug - show environment.sh"
   become: false
   block:
-    - name: "look for .environment.sh"
+    - name: "Look for .environment.sh"
       ansible.builtin.slurp:
         src: "{{ ansible_env.HOME }}/crowdsec/tests/.environment.sh"
       changed_when: true
       register: envfile
-    - name: "cat .environment.sh"
+    - name: "Show .environment.sh"
       ansible.builtin.debug:
         msg: "{{ envfile['content'] | b64decode }}"
 
-- name: "search for test scripts"
+- name: "Search for test scripts"
   become: false
   ansible.builtin.find:
     paths: "{{ ansible_env.HOME }}/crowdsec/tests/bats"
     pattern: "*.bats"
   register: testfiles
 
-- name: "run BATS tests for source build"
+- name: "Run BATS tests for source build"
   become: false
   block:
-    - name: "run test scripts"
+    - name: "Run test scripts"
       ansible.builtin.command:
         cmd: tests/run-tests {{ item.path }}
         chdir: "{{ ansible_env.HOME }}/crowdsec"
@@ -64,20 +64,20 @@
       when:
         - (item.path | basename) not in skip_tests.split(',')
   rescue:
-    - name: "read crowdsec.log"
+    - name: "Read crowdsec.log"
       ansible.builtin.slurp:
         path: "{{ ansible_env.HOME }}/crowdsec/tests/local/var/log/crowdsec.log"
       register: crowdsec_log
-    - name: "show crowdsec.log"
+    - name: "Show crowdsec.log"
       ansible.builtin.fail:
         msg: "{{ crowdsec_log['content'] | b64decode }}"
   when:
     - (package_testing is not defined) or (package_testing in ['', 'false', 'False'])
 
-- name: "run BATS tests for binary package"
+- name: "Run BATS tests for binary package"
   become: true
   block:
-    - name: "run test scripts"
+    - name: "Run test scripts"
       ansible.builtin.command:
         cmd: tests/run-tests {{ item.path }}
         chdir: "{{ ansible_env.HOME }}/crowdsec"
@@ -93,11 +93,11 @@
       when:
         - (item.path | basename) not in skip_tests.split(',')
   rescue:
-    - name: "read crowdsec.log"
+    - name: "Read crowdsec.log"
       ansible.builtin.slurp:
         path: "/var/log/crowdsec.log"
       register: crowdsec_log
-    - name: "show crowdsec.log"
+    - name: "Show crowdsec.log"
       ansible.builtin.fail:
         msg: "{{ crowdsec_log['content'] | b64decode }}"
   when:

tests/ansible/run_all.yml

@@ -1,6 +1,8 @@
 # vim: set ft=yaml.ansible:
 ---
+
 - import_playbook: provision_dependencies.yml
 - import_playbook: provision_test_suite.yml
+- import_playbook: install_binary_package.yml
 - import_playbook: prepare_tests.yml
 - import_playbook: run_tests.yml

tests/ansible/run_tests.yml

@@ -1,19 +1,20 @@
 # vim: set ft=yaml.ansible:
 ---
-- name: "run functional tests"
+
+- name: "Run functional tests"
   hosts: all
   gather_facts: true
   vars_files:
     - vars/mysql.yml
     - vars/postgres.yml
-  environment:
-    PGHOST: 127.0.0.1
-    PGPORT: 5432
-    PGPASSWORD: "{{ postgresql_users[0].password }}"
-    PGUSER: postgres
-    MYSQL_HOST: localhost
-    MYSQL_PORT: 3306
-    MYSQL_PASSWORD: "{{ mysql_root_password }}"
-    MYSQL_USER: "root"
   roles:
-    - role: run_func_tests
+    - name: run_func_tests
+      environment:
+        PGHOST: 127.0.0.1
+        PGPORT: 5432
+        PGPASSWORD: "{{ postgresql_users[0].password }}"
+        PGUSER: postgres
+        MYSQL_HOST: localhost
+        MYSQL_PORT: 3306
+        MYSQL_PASSWORD: "{{ mysql_root_password }}"
+        MYSQL_USER: "root"

tests/ansible/vagrant/common

@@ -20,22 +20,27 @@ Vagrant.configure('2') do |config|
   # same as above, to run the steps separately
 
   # config.vm.provision 'ansible' do |provdep|
-  #   provdep.config_file = '../../ansible-common.cfg'
+  #   provdep.config_file = '../../ansible.cfg'
   #   provdep.playbook = '../../provision_dependencies.yml'
   # end
 
   # config.vm.provision 'ansible' do |provtest|
-  #   provtest.config_file = '../../ansible-common.cfg'
+  #   provtest.config_file = '../../ansible.cfg'
   #   provtest.playbook = '../../provision_test_suite.yml'
   # end
 
   # config.vm.provision 'ansible' do |preptest|
-  #   preptest.config_file = '../../ansible-common.cfg'
+  #   preptest.config_file = '../../ansible.cfg'
+  #   preptest.playbook = '../../install_binary_package.yml'
+  # end
+
+  # config.vm.provision 'ansible' do |preptest|
+  #   preptest.config_file = '../../ansible.cfg'
   #   preptest.playbook = '../../prepare_tests.yml'
   # end
 
   # config.vm.provision 'ansible' do |runtests|
-  #   runtests.config_file = '../../ansible-common.cfg'
+  #   runtests.config_file = '../../ansible.cfg'
   #   runtests.playbook = '../../run_tests.yml'
   # end
 end

tests/ansible/vars/go.yml

@@ -1,3 +1,5 @@
 # vim: set ft=yaml.ansible:
-golang_version: "1.18.5"
+---
+
+golang_version: "1.19.1"
 golang_install_dir: "/opt/go/{{ golang_version }}"