Parallel hubtests (#2667)

* generate hub tests in python
* run hub tests in 3 batches at the same time (hardcoded)

.github/workflows/bats-hub.yml

@@ -15,7 +15,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.21.5"]
+        test-file: ["hub-1.bats", "hub-2.bats", "hub-3.bats"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest
@@ -33,10 +33,10 @@ jobs:
         fetch-depth: 0
         submodules: true
 
-    - name: "Set up Go ${{ matrix.go-version }}"
+    - name: "Set up Go"
       uses: actions/setup-go@v4
       with:
-        go-version: ${{ matrix.go-version }}
+        go-version: "1.21.5"
 
     - name: "Install bats dependencies"
       env:
@@ -48,7 +48,9 @@ jobs:
       run: make bats-clean bats-build bats-fixture BUILD_STATIC=1
 
     - name: "Run hub tests"
-      run: make bats-test-hub
+      run: |
+          ./test/bin/generate-hub-tests
+          ./test/run-tests test/dyn-bats/${{ matrix.test-file }}
 
     - name: "Collect hub coverage"
       run: ./test/bin/collect-hub-coverage >> $GITHUB_ENV

.github/workflows/go-tests.yml

@@ -32,10 +32,6 @@ env:
 
 jobs:
   build:
-    strategy:
-      matrix:
-        go-version: ["1.21.5"]
-
     name: "Build + tests"
     runs-on: ubuntu-latest
     services:
@@ -128,10 +124,10 @@ jobs:
         fetch-depth: 0
         submodules: false
 
-    - name: "Set up Go ${{ matrix.go-version }}"
+    - name: "Set up Go"
       uses: actions/setup-go@v4
       with:
-        go-version: ${{ matrix.go-version }}
+        go-version: "1.21.5"
 
     - name: Build and run tests, static
       run: |

test/bin/generate-hub-tests

@@ -7,44 +7,13 @@ THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
 # shellcheck disable=SC1091
 . "${THIS_DIR}/../.environment.sh"
 
-cscli() {
-    "${CSCLI}" "$@"
-}
-
 "${TEST_DIR}/instance-data" load
 
 hubdir="${LOCAL_DIR}/hub-tests"
 git clone --depth 1 https://github.com/crowdsecurity/hub.git "${hubdir}" >/dev/null 2>&1 || (cd "${hubdir}"; git pull)
 
-HUBTESTS_BATS="${TEST_DIR}/dyn-bats/hub.bats"
-
-cat << EOT > "${HUBTESTS_BATS}"
-set -u
-
-setup_file() {
-    load "../lib/setup_file.sh"
-}
-
-teardown_file() {
-    load "../lib/teardown_file.sh"
-}
-
-setup() {
-    load "../lib/setup.sh"
-}
-
-EOT
-
 echo "Generating hub tests..."
 
-for testname in $("${CSCLI}" --crowdsec "${CROWDSEC}" --cscli "${CSCLI}" hubtest --hub "${hubdir}" list -o json | jq -r '.[] | .Name'); do
+python3 "$THIS_DIR/generate-hub-tests.py" \
-    cat << EOT >> "${HUBTESTS_BATS}"
+    <("${CSCLI}" --crowdsec "${CROWDSEC}" --cscli "${CSCLI}" hubtest --hub "${hubdir}" list -o json) \
-
+    "${TEST_DIR}/dyn-bats/"
-@test "${testname}" {
-    run "\${CSCLI}" --crowdsec "\${CROWDSEC}" --cscli "\${CSCLI}" --hub "${hubdir}" hubtest run "${testname}" --clean
-    # in case of error, need to see what went wrong
-    echo "\$output"
-    assert_success
-}
-EOT
-done

test/bin/generate-hub-tests.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+
+import json
+import pathlib
+import os
+import sys
+import textwrap
+
+test_header = """
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh"
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh"
+}
+
+setup() {
+    load "../lib/setup.sh"
+}
+"""
+
+
+def write_chunk(target_dir, n, chunk):
+    with open(target_dir / f"hub-{n}.bats", "w") as f:
+        f.write(test_header)
+        for test in chunk:
+            cscli = os.environ['CSCLI']
+            crowdsec = os.environ['CROWDSEC']
+            testname = test['Name']
+            hubdir = os.environ['LOCAL_DIR'] + '/hub-tests'
+            f.write(textwrap.dedent(f"""
+                @test "{testname}" {{
+                    run "{cscli}" \\
+                        --crowdsec "{crowdsec}" \\
+                        --cscli "{cscli}" \\
+                        --hub "{hubdir}" \\
+                        hubtest run "{testname}" \\
+                        --clean
+                    echo "$output"
+                    assert_success
+                }}
+            """))
+
+
+def main():
+    hubtests_json = sys.argv[1]
+    target_dir = sys.argv[2]
+
+    with open(hubtests_json) as f:
+        j = json.load(f)
+        chunk_size = len(j) // 3 + 1
+        n = 1
+        for i in range(0, len(j), chunk_size):
+            chunk = j[i:i + chunk_size]
+            write_chunk(pathlib.Path(target_dir), n, chunk)
+            n += 1
+
+
+if __name__ == "__main__":
+    main()