From c2c173ac7e7eb6689ce076e40c624264a9ad73a3 Mon Sep 17 00:00:00 2001
From: mmetc <92726601+mmetc@users.noreply.github.com>
Date: Fri, 15 Dec 2023 18:30:20 +0100
Subject: [PATCH] Parallel hubtests (#2667)

* generate hub tests in python
* run hub tests in 3 batches at the same time (hardcoded)
---
 .github/workflows/bats-hub.yml | 10 +++---
 .github/workflows/go-tests.yml |  8 ++---
 test/bin/generate-hub-tests    | 37 ++------------------
 test/bin/generate-hub-tests.py | 63 ++++++++++++++++++++++++++++++++++
 4 files changed, 74 insertions(+), 44 deletions(-)
 create mode 100644 test/bin/generate-hub-tests.py

diff --git a/.github/workflows/bats-hub.yml b/.github/workflows/bats-hub.yml
index b9326b462..5520b0dac 100644
--- a/.github/workflows/bats-hub.yml
+++ b/.github/workflows/bats-hub.yml
@@ -15,7 +15,7 @@ jobs:
   build:
     strategy:
       matrix:
-        go-version: ["1.21.5"]
+        test-file: ["hub-1.bats", "hub-2.bats", "hub-3.bats"]
 
     name: "Build + tests"
     runs-on: ubuntu-latest
@@ -33,10 +33,10 @@ jobs:
         fetch-depth: 0
         submodules: true
 
-    - name: "Set up Go ${{ matrix.go-version }}"
+    - name: "Set up Go"
       uses: actions/setup-go@v4
       with:
-        go-version: ${{ matrix.go-version }}
+        go-version: "1.21.5"
 
     - name: "Install bats dependencies"
       env:
@@ -48,7 +48,9 @@ jobs:
       run: make bats-clean bats-build bats-fixture BUILD_STATIC=1
 
     - name: "Run hub tests"
-      run: make bats-test-hub
+      run: |
+          ./test/bin/generate-hub-tests
+          ./test/run-tests test/dyn-bats/${{ matrix.test-file }}
 
     - name: "Collect hub coverage"
       run: ./test/bin/collect-hub-coverage >> $GITHUB_ENV
diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml
index 78bed2a40..4cc118e66 100644
--- a/.github/workflows/go-tests.yml
+++ b/.github/workflows/go-tests.yml
@@ -32,10 +32,6 @@ env:
 
 jobs:
   build:
-    strategy:
-      matrix:
-        go-version: ["1.21.5"]
-
     name: "Build + tests"
     runs-on: ubuntu-latest
     services:
@@ -128,10 +124,10 @@ jobs:
         fetch-depth: 0
         submodules: false
 
-    - name: "Set up Go ${{ matrix.go-version }}"
+    - name: "Set up Go"
       uses: actions/setup-go@v4
       with:
-        go-version: ${{ matrix.go-version }}
+        go-version: "1.21.5"
 
     - name: Build and run tests, static
       run: |
diff --git a/test/bin/generate-hub-tests b/test/bin/generate-hub-tests
index 21031285c..658cc33a7 100755
--- a/test/bin/generate-hub-tests
+++ b/test/bin/generate-hub-tests
@@ -7,44 +7,13 @@ THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
 # shellcheck disable=SC1091
 . "${THIS_DIR}/../.environment.sh"
 
-cscli() {
-    "${CSCLI}" "$@"
-}
-
 "${TEST_DIR}/instance-data" load
 
 hubdir="${LOCAL_DIR}/hub-tests"
 git clone --depth 1 https://github.com/crowdsecurity/hub.git "${hubdir}" >/dev/null 2>&1 || (cd "${hubdir}"; git pull)
 
-HUBTESTS_BATS="${TEST_DIR}/dyn-bats/hub.bats"
-
-cat << EOT > "${HUBTESTS_BATS}"
-set -u
-
-setup_file() {
-    load "../lib/setup_file.sh"
-}
-
-teardown_file() {
-    load "../lib/teardown_file.sh"
-}
-
-setup() {
-    load "../lib/setup.sh"
-}
-
-EOT
-
 echo "Generating hub tests..."
 
-for testname in $("${CSCLI}" --crowdsec "${CROWDSEC}" --cscli "${CSCLI}" hubtest --hub "${hubdir}" list -o json | jq -r '.[] | .Name'); do
-    cat << EOT >> "${HUBTESTS_BATS}"
-
-@test "${testname}" {
-    run "\${CSCLI}" --crowdsec "\${CROWDSEC}" --cscli "\${CSCLI}" --hub "${hubdir}" hubtest run "${testname}" --clean
-    # in case of error, need to see what went wrong
-    echo "\$output"
-    assert_success
-}
-EOT
-done
+python3 "$THIS_DIR/generate-hub-tests.py" \
+    <("${CSCLI}" --crowdsec "${CROWDSEC}" --cscli "${CSCLI}" hubtest --hub "${hubdir}" list -o json) \
+    "${TEST_DIR}/dyn-bats/"
diff --git a/test/bin/generate-hub-tests.py b/test/bin/generate-hub-tests.py
new file mode 100644
index 000000000..48f296776
--- /dev/null
+++ b/test/bin/generate-hub-tests.py
@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+
+import json
+import pathlib
+import os
+import sys
+import textwrap
+
+test_header = """
+set -u
+
+setup_file() {
+    load "../lib/setup_file.sh"
+}
+
+teardown_file() {
+    load "../lib/teardown_file.sh"
+}
+
+setup() {
+    load "../lib/setup.sh"
+}
+"""
+
+
+def write_chunk(target_dir, n, chunk):
+    with open(target_dir / f"hub-{n}.bats", "w") as f:
+        f.write(test_header)
+        for test in chunk:
+            cscli = os.environ['CSCLI']
+            crowdsec = os.environ['CROWDSEC']
+            testname = test['Name']
+            hubdir = os.environ['LOCAL_DIR'] + '/hub-tests'
+            f.write(textwrap.dedent(f"""
+                @test "{testname}" {{
+                    run "{cscli}" \\
+                        --crowdsec "{crowdsec}" \\
+                        --cscli "{cscli}" \\
+                        --hub "{hubdir}" \\
+                        hubtest run "{testname}" \\
+                        --clean
+                    echo "$output"
+                    assert_success
+                }}
+            """))
+
+
+def main():
+    hubtests_json = sys.argv[1]
+    target_dir = sys.argv[2]
+
+    with open(hubtests_json) as f:
+        j = json.load(f)
+        chunk_size = len(j) // 3 + 1
+        n = 1
+        for i in range(0, len(j), chunk_size):
+            chunk = j[i:i + chunk_size]
+            write_chunk(pathlib.Path(target_dir), n, chunk)
+            n += 1
+
+
+if __name__ == "__main__":
+    main()