CI: Static builds by default; replace bincover with go -cover from 1.20 (#2150)

* Makefile: build static binaries only
* Replace bincover with go -cover from 1.20
* CI: Fix timing issue between lapi and agent containers

.github/workflows/bats-sqlite-coverage.yml

@@ -55,7 +55,6 @@ jobs:
         go install github.com/mikefarah/yq/v4@latest
         go install github.com/cloudflare/cfssl/cmd/cfssl@master
         go install github.com/cloudflare/cfssl/cmd/cfssljson@master
-        go install github.com/wadey/gocovmerge@latest
 
     - name: "Build crowdsec and fixture"
       run: |
@@ -64,6 +63,20 @@ jobs:
     - name: "Run tests"
       run: make bats-test
 
+    - name: "Collect coverage data"
+      run: |
+        go tool covdata textfmt -i test/coverage -o coverage-bats-raw.out
+        # filter out unwanted packages, should match the argument to "go-acc --ignore"
+        grep -v \
+          -e '/pkg/database' \
+          -e '/plugins/notifications' \
+          -e '/pkg/protobufs' \
+          -e '/pkg/cwversions' \
+          -e '/pkg/cstest' \
+          -e '/pkg/models' \
+          < coverage-bats-raw.out \
+          > coverage-bats.out
+
     #
     # In case you need to inspect the database status after the failure of a given test
     #
@@ -87,11 +100,5 @@ jobs:
     - name: Upload crowdsec coverage to codecov
       uses: codecov/codecov-action@v3
       with:
-        files: ./test/local/var/lib/coverage/coverage-crowdsec.out
-        flags: func-crowdsec
-
-    - name: Upload cscli coverage to codecov
-      uses: codecov/codecov-action@v3
-      with:
-        files: ./test/local/var/lib/coverage/coverage-cscli.out
-        flags: func-cscli
+        files: ./coverage-bats.out
+        flags: bats

.github/workflows/go-tests.yml

@@ -141,12 +141,6 @@ jobs:
         make build
         make go-acc | richgo testfilter
 
-    - name: Build and run tests (static)
-      run: |
-        make clean build BUILD_STATIC=yes
-        make test \
-          | richgo testfilter
-
     - name: Upload unit coverage to Codecov
       uses: codecov/codecov-action@v3
       with:

.github/workflows/release_publish-package.yml

@@ -41,53 +41,10 @@ jobs:
 
       - name: Build the binaries
         run: make release
-      - name: Upload to release
-        uses: JasonEtco/upload-to-release@master
-        with:
-          args: crowdsec-release.tgz application/x-gzip
-        env:
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
-  build_static:
-    strategy:
-      matrix:
-        go-version: ["1.20.1"]
-
-    name: Build and upload binary package
-    runs-on: ubuntu-latest
-    steps:
-      - name: "Set up Go ${{ matrix.go-version }}"
-        uses: actions/setup-go@v3
-        with:
-          go-version: ${{ matrix.go-version }}
-        id: go
-
-      - name: Check out code into the Go module directory
-        uses: actions/checkout@v3
-        with:
-          fetch-depth: 0
-          submodules: false
-
-      - name: Cache Go modules
-        uses: actions/cache@v3
-        with:
-          path: |
-            ~/go/pkg/mod
-            ~/.cache/go-build
-            ~/Library/Caches/go-build
-            %LocalAppData%\go-build
-          key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-${{ matrix.go-version }}-go-
-
-      - name: Build the binaries
-        run: |
-          make release BUILD_STATIC=yes
-          mv crowdsec-release.tgz crowdsec-release-static.tgz
 
       - name: Upload to release
         uses: JasonEtco/upload-to-release@master
         with:
-          args: crowdsec-release-static.tgz application/x-gzip
+          args: crowdsec-release.tgz application/x-gzip
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -15,8 +15,9 @@
 # Test binaries, built from *_test.go
 pkg/csplugin/tests/cs_plugin_test*
 
-# Output of go-acc
+# Output of go-acc, go -cover
 *.out
+test/coverage/*
 
 # Development artifacts, backups, etc
 *.swp

Makefile

@@ -27,8 +27,6 @@ BUILD_CMD = build
 GO_MODULE_NAME = github.com/crowdsecurity/crowdsec
 
 LD_OPTS_VARS= \
--X '$(GO_MODULE_NAME)/cmd/crowdsec.bincoverTesting=$(BINCOVER_TESTING)' \
--X '$(GO_MODULE_NAME)/cmd/crowdsec-cli.bincoverTesting=$(BINCOVER_TESTING)' \
 -X '$(GO_MODULE_NAME)/pkg/cwversion.Version=$(BUILD_VERSION)' \
 -X '$(GO_MODULE_NAME)/pkg/cwversion.BuildDate=$(BUILD_TIMESTAMP)' \
 -X '$(GO_MODULE_NAME)/pkg/cwversion.Codename=$(BUILD_CODENAME)' \
@@ -41,9 +39,14 @@ LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.System=docker'
 endif
 
 ifdef BUILD_STATIC
-	export LD_OPTS=-ldflags "-s -w $(LD_OPTS_VARS) -extldflags '-static'" -tags netgo,osusergo,sqlite_omit_load_extension
-else
-	export LD_OPTS=-ldflags "-s -w $(LD_OPTS_VARS)"
+$(warning WARNING: The BUILD_STATIC variable is deprecated and has no effect. Builds are static by default since v1.5.0.)
+endif
+
+export LD_OPTS=-ldflags "-s -w -extldflags '-static' $(LD_OPTS_VARS)" \
+	-trimpath -tags netgo,osusergo,sqlite_omit_load_extension
+
+ifneq (,$(TEST_COVERAGE))
+LD_OPTS += -cover
 endif
 
 GOCMD = go
@@ -68,7 +71,6 @@ clean: testclean
 	@$(RM) $(CSCLI_BIN) $(WIN_IGNORE_ERR)
 	@$(RM) *.log $(WIN_IGNORE_ERR)
 	@$(RM) crowdsec-release.tgz $(WIN_IGNORE_ERR)
-	@$(RM) crowdsec-release-static.tgz $(WIN_IGNORE_ERR)
 	@$(RM) $(HTTP_PLUGIN_FOLDER)/$(HTTP_PLUGIN_BIN) $(WIN_IGNORE_ERR)
 	@$(RM) $(SLACK_PLUGIN_FOLDER)/$(SLACK_PLUGIN_BIN) $(WIN_IGNORE_ERR)
 	@$(RM) $(SPLUNK_PLUGIN_FOLDER)/$(SPLUNK_PLUGIN_BIN) $(WIN_IGNORE_ERR)
@@ -79,15 +81,9 @@ clean: testclean
 cscli: goversion
 	@$(MAKE) -C $(CSCLI_FOLDER) build --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
 
-cscli-bincover: goversion
-	@GOARCH=$(GOARCH) GOOS=$(GOOS) $(MAKE) -C $(CSCLI_FOLDER) build-bincover --no-print-directory
-
 crowdsec: goversion
 	@$(MAKE) -C $(CROWDSEC_FOLDER) build --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
 
-crowdsec-bincover: goversion
-	@GOARCH=$(GOARCH) GOOS=$(GOOS) $(MAKE) -C $(CROWDSEC_FOLDER) build-bincover --no-print-directory
-
 http-plugin: goversion
 	@$(MAKE) -C $(HTTP_PLUGIN_FOLDER) build --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
 

cmd/crowdsec-cli/Makefile

@@ -12,8 +12,6 @@ GOTEST = $(GOCMD) test
 GOGET = $(GOCMD) get
 
 BINARY_NAME = cscli$(EXT)
-# names longer than 15 chars break 'pgrep'
-BINARY_NAME_COVER = $(BINARY_NAME).cover
 PREFIX ?= "/"
 BIN_PREFIX = $(PREFIX)"/usr/local/bin/"
 
@@ -23,9 +21,6 @@ all: clean build
 build: clean
 	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
 
-build-bincover: clean
-	$(GOTEST) . -tags testrunmain -coverpkg=$(go list github.com/crowdsecurity/crowdsec/... | grep -v -e 'pkg/database' -e 'plugins/notifications' -e 'pkg/protobufs' -e 'pkg/cwversions' -e 'pkg/cstest' -e 'pkg/models') -covermode=atomic $(LD_OPTS) -c -o $(BINARY_NAME_COVER)
-
 .PHONY: install
 install: install-conf install-bin
 
@@ -39,4 +34,4 @@ uninstall:
 	@$(RM) $(BIN_PREFIX)$(BINARY_NAME) $(WIN_IGNORE_ERR)
 
 clean:
-	@$(RM) $(BINARY_NAME) $(BINARY_NAME_COVER) $(WIN_IGNORE_ERR)
+	@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)

cmd/crowdsec-cli/main.go

@@ -7,7 +7,6 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/confluentinc/bincover"
 	"github.com/fatih/color"
 	cc "github.com/ivanpirog/coloredcobra"
 	log "github.com/sirupsen/logrus"
@@ -21,8 +20,6 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/fflag"
 )
 
-var bincoverTesting = ""
-
 var trace_lvl, dbg_lvl, nfo_lvl, wrn_lvl, err_lvl bool
 
 var ConfigFilePath string
@@ -252,15 +249,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 	}
 
 	if err := rootCmd.Execute(); err != nil {
-		if bincoverTesting != "" {
-			log.Debug("coverage report is enabled")
-		}
-
-		exitCode := 1
 		log.NewEntry(log.StandardLogger()).Log(log.FatalLevel, err)
-		if bincoverTesting == "" {
-			os.Exit(exitCode)
-		}
-		bincover.ExitCode = exitCode
+		os.Exit(1)
 	}
 }

cmd/crowdsec-cli/main_test.go

@@ -1,13 +0,0 @@
-//go:build testrunmain
-
-package main
-
-import (
-	"testing"
-
-	"github.com/confluentinc/bincover"
-)
-
-func TestBincoverRunMain(t *testing.T) {
-	bincover.RunTest(main)
-}

cmd/crowdsec/Makefile

@@ -13,7 +13,6 @@ GOGET = $(GOCMD) get
 
 CROWDSEC_BIN = crowdsec$(EXT)
 # names longer than 15 chars break 'pgrep'
-CROWDSEC_BIN_COVER = $(CROWDSEC_BIN).cover
 PREFIX ?= "/"
 CFG_PREFIX = $(PREFIX)"/etc/crowdsec/config/"
 BIN_PREFIX = $(PREFIX)"/usr/local/bin/"
@@ -28,14 +27,11 @@ all: clean test build
 build: clean
 	$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(CROWDSEC_BIN)
 
-build-bincover: clean
-	$(GOTEST) . -tags testrunmain -coverpkg=$(go list github.com/crowdsecurity/crowdsec/... | grep -v -e 'pkg/database' -e 'plugins/notifications' -e 'pkg/protobufs' -e 'pkg/cwversions' -e 'pkg/cstest' -e 'pkg/models') -covermode=atomic $(LD_OPTS) -c -o $(CROWDSEC_BIN_COVER)
-
 test:
 	$(GOTEST) $(LD_OPTS) -v ./...
 
 clean:
-	@$(RM) $(CROWDSEC_BIN) $(CROWDSEC_BIN).test $(CROWDSEC_BIN_COVER) $(WIN_IGNORE_ERR)
+	@$(RM) $(CROWDSEC_BIN) $(CROWDSEC_BIN).test $(WIN_IGNORE_ERR)
 
 .PHONY: install
 install: install-conf install-bin

cmd/crowdsec/main.go

@@ -9,7 +9,6 @@ import (
 	"strings"
 	"time"
 
-	"github.com/confluentinc/bincover"
 	"github.com/pkg/errors"
 	log "github.com/sirupsen/logrus"
 	"gopkg.in/tomb.v2"
@@ -51,8 +50,6 @@ var (
 	pluginBroker      csplugin.PluginBroker
 )
 
-var bincoverTesting = ""
-
 type Flags struct {
 	ConfigFile     string
 	TraceLevel     bool
@@ -296,25 +293,6 @@ func LoadConfig(cConfig *csconfig.Config) error {
 	return nil
 }
 
-// exitWithCode must be called right before the program termination,
-// to allow measuring functional test coverage in case of abnormal exit.
-//
-// without bincover: log error and exit with code
-// with bincover: log error and tell bincover the exit code, then return
-func exitWithCode(exitCode int, err error) {
-	if err != nil {
-		// this method of logging a fatal error does not
-		// trigger a program exit (as stated by the authors, it
-		// is not going to change in logrus to keep backward
-		// compatibility), and allows us to report coverage.
-		log.NewEntry(log.StandardLogger()).Log(log.FatalLevel, err)
-	}
-	if bincoverTesting == "" {
-		os.Exit(exitCode)
-	}
-	bincover.ExitCode = exitCode
-}
-
 // crowdsecT0 can be used to measure start time of services,
 // or uptime of the application
 var crowdsecT0 time.Time
@@ -332,8 +310,6 @@ func main() {
 
 	crowdsecT0 = time.Now()
 
-	defer types.CatchPanic("crowdsec/main")
-
 	log.Debugf("os.Args: %v", os.Args)
 
 	// Handle command line arguments
@@ -344,20 +320,17 @@ func main() {
 		fmt.Fprintf(os.Stderr, "argument provided but not defined: %s\n", flag.Args()[0])
 		flag.Usage()
 		// the flag package exits with 2 in case of unknown flag
-		exitWithCode(2, nil)
-		return
+		os.Exit(2)
 	}
 
 	if flags.PrintVersion {
 		cwversion.Show()
-		exitWithCode(0, nil)
-		return
+		os.Exit(0)
 	}
 
-	exitCode := 0
 	err := StartRunSvc()
 	if err != nil {
-		exitCode = 1
+		log.Fatal(err)
 	}
-	exitWithCode(exitCode, err)
+	os.Exit(0)
 }

cmd/crowdsec/main_test.go

@@ -1,13 +0,0 @@
-//go:build testrunmain
-
-package main
-
-import (
-	"github.com/confluentinc/bincover"
-
-	"testing"
-)
-
-func TestBincoverRunMain(t *testing.T) {
-	bincover.RunTest(main)
-}

cmd/crowdsec/run_in_svc.go

@@ -4,6 +4,7 @@
 package main
 
 import (
+	"fmt"
 	"os"
 
 	log "github.com/sirupsen/logrus"
@@ -12,6 +13,7 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
 func StartRunSvc() error {
@@ -20,6 +22,8 @@ func StartRunSvc() error {
 		err     error
 	)
 
+	defer types.CatchPanic("crowdsec/StartRunSvc")
+
 	// Set a default logger with level=fatal on stderr,
 	// in addition to the one we configure afterwards
 	log.AddHook(&writer.Hook{
@@ -40,10 +44,6 @@ func StartRunSvc() error {
 
 	log.Infof("Crowdsec %s", cwversion.VersionStr())
 
-	if bincoverTesting != "" {
-		log.Debug("coverage report is enabled")
-	}
-
 	apiReady := make(chan bool, 1)
 	agentReady := make(chan bool, 1)
 
@@ -56,7 +56,7 @@ func StartRunSvc() error {
 			dbClient, err = database.NewClient(cConfig.DbConfig)
 
 			if err != nil {
-				log.Fatalf("unable to create database client: %s", err)
+				return fmt.Errorf("unable to create database client: %s", err)
 			}
 		}
 		registerPrometheus(cConfig.Prometheus)

cmd/crowdsec/run_in_svc_windows.go

@@ -10,12 +10,15 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/cwversion"
 	"github.com/crowdsecurity/crowdsec/pkg/database"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
 func StartRunSvc() error {
 	const svcName = "CrowdSec"
 	const svcDescription = "Crowdsec IPS/IDS"
 
+	defer types.CatchPanic("crowdsec/StartRunSvc")
+
 	isRunninginService, err := svc.IsWindowsService()
 	if err != nil {
 		return errors.Wrap(err, "failed to determine if we are running in windows service mode")
@@ -68,10 +71,6 @@ func WindowsRun() error {
 	// Configure logging
 	log.Infof("Crowdsec %s", cwversion.VersionStr())
 
-	if bincoverTesting != "" {
-		log.Debug("coverage report is enabled")
-	}
-
 	apiReady := make(chan bool, 1)
 	agentReady := make(chan bool, 1)
 
@@ -84,7 +83,7 @@ func WindowsRun() error {
 			dbClient, err = database.NewClient(cConfig.DbConfig)
 
 			if err != nil {
-				log.Fatalf("unable to create database client: %s", err)
+				return fmt.Errorf("unable to create database client: %s", err)
 			}
 		}
 		registerPrometheus(cConfig.Prometheus)

docker/test/tests/test_agent_only.py

@@ -28,11 +28,12 @@ def test_split_lapi_agent(crowdsec, flavor):
     cs_lapi = crowdsec(name=lapiname, environment=lapi_env, flavor=flavor)
     cs_agent = crowdsec(name=agentname, environment=agent_env, flavor=flavor)
 
-    with cs_lapi as lapi, cs_agent as agent:
+    with cs_lapi as lapi:
         lapi.wait_for_log("*CrowdSec Local API listening on 0.0.0.0:8080*")
-        agent.wait_for_log("*Starting processing data*")
         lapi.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
-        res = agent.cont.exec_run('cscli lapi status')
-        assert res.exit_code == 0
-        stdout = res.output.decode()
-        assert "You can successfully interact with Local API (LAPI)" in stdout
+        with cs_agent as agent:
+            agent.wait_for_log("*Starting processing data*")
+            res = agent.cont.exec_run('cscli lapi status')
+            assert res.exit_code == 0
+            stdout = res.output.decode()
+            assert "You can successfully interact with Local API (LAPI)" in stdout

docker/test/tests/test_tls.py

@@ -168,22 +168,26 @@ def test_tls_split_lapi_agent(crowdsec, flavor, certs_dir):
         certs_dir(lapi_hostname=lapiname): {'bind': '/etc/ssl/crowdsec', 'mode': 'ro'},
     }
 
-    with crowdsec(flavor=flavor, name=lapiname, environment=lapi_env, volumes=volumes) as lapi, crowdsec(flavor=flavor, name=agentname, environment=agent_env, volumes=volumes) as agent:
+    cs_lapi = crowdsec(flavor=flavor, name=lapiname, environment=lapi_env, volumes=volumes)
+    cs_agent = crowdsec(flavor=flavor, name=agentname, environment=agent_env, volumes=volumes)
+
+    with cs_lapi as lapi:
         lapi.wait_for_log([
             "*(tls) Client Auth Type set to VerifyClientCertIfGiven*",
             "*CrowdSec Local API listening on 0.0.0.0:8080*"
         ])
         # TODO: wait_for_https
         lapi.wait_for_http(8080, '/health', want_status=None)
-        agent.wait_for_log("*Starting processing data*")
-        res = agent.cont.exec_run('cscli lapi status')
-        assert res.exit_code == 0
-        stdout = res.output.decode()
-        assert "You can successfully interact with Local API (LAPI)" in stdout
-        res = lapi.cont.exec_run('cscli lapi status')
-        assert res.exit_code == 0
-        stdout = res.output.decode()
-        assert "You can successfully interact with Local API (LAPI)" in stdout
+        with cs_agent as agent:
+            agent.wait_for_log("*Starting processing data*")
+            res = agent.cont.exec_run('cscli lapi status')
+            assert res.exit_code == 0
+            stdout = res.output.decode()
+            assert "You can successfully interact with Local API (LAPI)" in stdout
+            res = lapi.cont.exec_run('cscli lapi status')
+            assert res.exit_code == 0
+            stdout = res.output.decode()
+            assert "You can successfully interact with Local API (LAPI)" in stdout
 
 
 def test_tls_mutual_split_lapi_agent(crowdsec, flavor, certs_dir):
@@ -215,22 +219,26 @@ def test_tls_mutual_split_lapi_agent(crowdsec, flavor, certs_dir):
         certs_dir(lapi_hostname=lapiname): {'bind': '/etc/ssl/crowdsec', 'mode': 'ro'},
     }
 
-    with crowdsec(flavor=flavor, name=lapiname, environment=lapi_env, volumes=volumes) as lapi, crowdsec(flavor=flavor, name=agentname, environment=agent_env, volumes=volumes) as agent:
+    cs_lapi = crowdsec(flavor=flavor, name=lapiname, environment=lapi_env, volumes=volumes)
+    cs_agent = crowdsec(flavor=flavor, name=agentname, environment=agent_env, volumes=volumes)
+
+    with cs_lapi as lapi:
         lapi.wait_for_log([
             "*(tls) Client Auth Type set to VerifyClientCertIfGiven*",
             "*CrowdSec Local API listening on 0.0.0.0:8080*"
         ])
         # TODO: wait_for_https
         lapi.wait_for_http(8080, '/health', want_status=None)
-        agent.wait_for_log("*Starting processing data*")
-        res = agent.cont.exec_run('cscli lapi status')
-        assert res.exit_code == 0
-        stdout = res.output.decode()
-        assert "You can successfully interact with Local API (LAPI)" in stdout
-        res = lapi.cont.exec_run('cscli lapi status')
-        assert res.exit_code == 0
-        stdout = res.output.decode()
-        assert "You can successfully interact with Local API (LAPI)" in stdout
+        with cs_agent as agent:
+            agent.wait_for_log("*Starting processing data*")
+            res = agent.cont.exec_run('cscli lapi status')
+            assert res.exit_code == 0
+            stdout = res.output.decode()
+            assert "You can successfully interact with Local API (LAPI)" in stdout
+            res = lapi.cont.exec_run('cscli lapi status')
+            assert res.exit_code == 0
+            stdout = res.output.decode()
+            assert "You can successfully interact with Local API (LAPI)" in stdout
 
 
 def test_tls_client_ou(crowdsec, certs_dir):
@@ -262,26 +270,40 @@ def test_tls_client_ou(crowdsec, certs_dir):
         certs_dir(lapi_hostname=lapiname, agent_ou='custom-client-ou'): {'bind': '/etc/ssl/crowdsec', 'mode': 'ro'},
     }
 
-    with crowdsec(name=lapiname, environment=lapi_env, volumes=volumes) as lapi, crowdsec(name=agentname, environment=agent_env, volumes=volumes) as agent:
+    cs_lapi = crowdsec(name=lapiname, environment=lapi_env, volumes=volumes)
+    cs_agent = crowdsec(name=agentname, environment=agent_env, volumes=volumes)
+
+    with cs_lapi as lapi:
         lapi.wait_for_log([
-            "*client certificate OU (?custom-client-ou?) doesn't match expected OU (?agent-ou?)*",
+            "*(tls) Client Auth Type set to VerifyClientCertIfGiven*",
+            "*CrowdSec Local API listening on 0.0.0.0:8080*"
         ])
+        # TODO: wait_for_https
+        lapi.wait_for_http(8080, '/health', want_status=None)
+        with cs_agent as agent:
+            lapi.wait_for_log([
+                "*client certificate OU (?custom-client-ou?) doesn't match expected OU (?agent-ou?)*",
+            ])
 
     lapi_env['AGENTS_ALLOWED_OU'] = 'custom-client-ou'
 
-    with crowdsec(name=lapiname, environment=lapi_env, volumes=volumes) as lapi, crowdsec(name=agentname, environment=agent_env, volumes=volumes) as agent:
+    cs_lapi = crowdsec(name=lapiname, environment=lapi_env, volumes=volumes)
+    cs_agent = crowdsec(name=agentname, environment=agent_env, volumes=volumes)
+
+    with cs_lapi as lapi:
         lapi.wait_for_log([
             "*(tls) Client Auth Type set to VerifyClientCertIfGiven*",
             "*CrowdSec Local API listening on 0.0.0.0:8080*"
         ])
         # TODO: wait_for_https
         lapi.wait_for_http(8080, '/health', want_status=None)
-        agent.wait_for_log("*Starting processing data*")
-        res = agent.cont.exec_run('cscli lapi status')
-        assert res.exit_code == 0
-        stdout = res.output.decode()
-        assert "You can successfully interact with Local API (LAPI)" in stdout
-        res = lapi.cont.exec_run('cscli lapi status')
-        assert res.exit_code == 0
-        stdout = res.output.decode()
-        assert "You can successfully interact with Local API (LAPI)" in stdout
+        with cs_agent as agent:
+            agent.wait_for_log("*Starting processing data*")
+            res = agent.cont.exec_run('cscli lapi status')
+            assert res.exit_code == 0
+            stdout = res.output.decode()
+            assert "You can successfully interact with Local API (LAPI)" in stdout
+            res = lapi.cont.exec_run('cscli lapi status')
+            assert res.exit_code == 0
+            stdout = res.output.decode()
+            assert "You can successfully interact with Local API (LAPI)" in stdout

go.mod

@@ -12,7 +12,6 @@ require (
 	github.com/aws/aws-sdk-go v1.42.25
 	github.com/buger/jsonparser v1.1.1
 	github.com/c-robinson/iplib v1.0.3
-	github.com/confluentinc/bincover v0.2.0
 	github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
 	github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
 	github.com/crowdsecurity/grokky v0.2.1
@@ -84,6 +83,7 @@ require (
 	github.com/shirou/gopsutil/v3 v3.22.12
 	github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c
 	github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26
+	github.com/wasilibs/go-re2 v0.2.1
 	golang.org/x/sys v0.5.0
 	gopkg.in/yaml.v3 v3.0.1
 	k8s.io/apiserver v0.22.5
@@ -177,7 +177,6 @@ require (
 	github.com/tklauser/numcpus v0.6.0 // indirect
 	github.com/ugorji/go/codec v1.2.6 // indirect
 	github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect
-	github.com/wasilibs/go-re2 v0.2.1 // indirect
 	github.com/yusufpapurcu/wmi v1.2.2 // indirect
 	github.com/zclconf/go-cty v1.8.0 // indirect
 	go.mongodb.org/mongo-driver v1.9.0 // indirect

go.sum

@@ -88,10 +88,6 @@ github.com/alexliesenfeld/health v0.5.1 h1:cohQdtQbJdA6bj0aMD4gdXA9xQyvh9NxWO9XL
 github.com/alexliesenfeld/health v0.5.1/go.mod h1:N4NDIeQtlWumG+6z1ne1v62eQxktz5ylEgGgH9emdMw=
 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8=
 github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
-github.com/antonmedv/expr v1.12.2 h1:nlRcu4uHI6oSKCf6GHJTcT7hIf7dFAjgfvG0MWb7Cu0=
-github.com/antonmedv/expr v1.12.2/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU=
-github.com/antonmedv/expr v1.12.4 h1:YRkeF7r0cejMS47bDYe3Jyes7L9t1AhpunC+Duq+R9k=
-github.com/antonmedv/expr v1.12.4/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU=
 github.com/antonmedv/expr v1.12.5 h1:Fq4okale9swwL3OeLLs9WD9H6GbgBLJyN/NUHRv+n0E=
 github.com/antonmedv/expr v1.12.5/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU=
 github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
@@ -156,8 +152,6 @@ github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMe
 github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo=
 github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA=
 github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI=
-github.com/confluentinc/bincover v0.2.0 h1:WSS3MqzwJbosCLMOuF3tJ0pMpALzBfrm80Tb+/3gbQs=
-github.com/confluentinc/bincover v0.2.0/go.mod h1:qeI1wx0RxdGTZtrJY0HVlgJ4NqC/X2Z+fHbvy87tgHE=
 github.com/containerd/containerd v1.6.18 h1:qZbsLvmyu+Vlty0/Ex5xc0z2YtKpIsb5n45mAMI+2Ns=
 github.com/containerd/containerd v1.6.18/go.mod h1:1RdCUu95+gc2v9t3IL+zIlpClSmew7/0YS8O5eQZrOw=
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
@@ -181,8 +175,6 @@ github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw=
 github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk=
-github.com/crowdsecurity/grokky v0.2.0 h1:8u3dCU1kqSlOu2phP6S+HF2OTohZ/JBGcZCWu1dhnF0=
-github.com/crowdsecurity/grokky v0.2.0/go.mod h1:qdyzFHtnY6nmdYZJFHVbdms2HMFLiEXzmqq5tCUmiGE=
 github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4=
 github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM=
 github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc=
@@ -927,10 +919,9 @@ github.com/vmihailenco/msgpack v4.0.4+incompatible h1:dSLoQfGFAo3F6OoNhwUmLwVgaU
 github.com/vmihailenco/msgpack v4.0.4+incompatible/go.mod h1:fy3FlTQTDXWkZ7Bh6AcGMlsjHatGryHQYUTf1ShIgkk=
 github.com/vmihailenco/msgpack/v4 v4.3.12/go.mod h1:gborTTJjAo/GWTqqRjrLCn9pgNN+NXzzngzBKDPIqw4=
 github.com/vmihailenco/tagparser v0.1.1/go.mod h1:OeAg3pn3UbLjkWt+rN9oFYB6u/cQgqMEUPoW2WPyhdI=
-github.com/wasilibs/go-re2 v0.1.0 h1:+pcjlvge6E9WWU4eaMlULRikEUkhPDXSFfCF5p7htMI=
-github.com/wasilibs/go-re2 v0.1.0/go.mod h1:F91Yac+zPNDFrrd8fl4mSd7+TTu2tYiX56BEIEPQl2M=
 github.com/wasilibs/go-re2 v0.2.1 h1:bZVXxlqGHXnHlQt5ys6CsHyypsCsP952m1oKBB5O9Dc=
 github.com/wasilibs/go-re2 v0.2.1/go.mod h1:jrB0h+KSTmTwsL0lI2sEyu8TcRi1htLiR/TbSURMwMM=
+github.com/wasilibs/nottinygc v0.2.0 h1:cXz2Ac9bVMLkpuOlUlPQMWowjw0K2cOErXZOFdAj7yE=
 github.com/xdg-go/pbkdf2 v1.0.0/go.mod h1:jrpuAogTd400dnrH08LKmI/xc1MbPOebTwRqcT5RDeI=
 github.com/xdg-go/scram v1.0.2/go.mod h1:1WAq6h33pAW+iRreB34OORO2Nf7qel3VV3fjBj+hCSs=
 github.com/xdg-go/stringprep v1.0.2/go.mod h1:8F9zXuvzgwmyT5DUm4GUfZGDdT3W+LCvS6+da4O5kxM=
@@ -1426,7 +1417,6 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=

test/bats.mk

@@ -27,18 +27,8 @@ PID_DIR = $(LOCAL_DIR)/var/run
 PLUGIN_DIR = $(LOCAL_DIR)/lib/crowdsec/plugins
 DB_BACKEND ?= sqlite
 
-ifdef TEST_COVERAGE
-  CROWDSEC = $(TEST_DIR)/bin/crowdsec-wrapper
-  CSCLI = $(TEST_DIR)/bin/cscli-wrapper
-  BINCOVER_TESTING = true
-else
-  # the wrappers should work here too - it detects TEST_COVERAGE - but we allow
-  # overriding the path to the binaries
-  CROWDSEC ?= $(BIN_DIR)/crowdsec
-  CSCLI ?= $(BIN_DIR)/cscli
-  # any value is considered true
-  BINCOVER_TESTING =
-endif
+CROWDSEC ?= $(BIN_DIR)/crowdsec
+CSCLI ?= $(BIN_DIR)/cscli
 
 # If you change the name of the crowdsec executable, make sure the pgrep
 # parameters are correct in $(TEST_DIR)/assert-crowdsec-not-running
@@ -59,6 +49,7 @@ export INIT_BACKEND="$(INIT_BACKEND)"
 export CONFIG_BACKEND="$(CONFIG_BACKEND)"
 export PACKAGE_TESTING="$(PACKAGE_TESTING)"
 export TEST_COVERAGE="$(TEST_COVERAGE)"
+export GOCOVERDIR="$(TEST_DIR)/coverage"
 endef
 
 bats-all: bats-clean bats-build bats-fixture bats-test bats-test-hub
@@ -76,11 +67,9 @@ bats-check-requirements:
 # Build and installs crowdsec in a local directory. Rebuilds if already exists.
 bats-build: bats-environment bats-check-requirements
 	@mkdir -p $(BIN_DIR) $(LOG_DIR) $(PID_DIR) $(PLUGIN_DIR)
-	@BINCOVER_TESTING=$(BINCOVER_TESTING) DEFAULT_CONFIGDIR=$(CONFIG_DIR) DEFAULT_DATADIR=$(DATA_DIR) $(MAKE) goversion crowdsec cscli plugins
+	@TEST_COVERAGE=$(TEST_COVERAGE) DEFAULT_CONFIGDIR=$(CONFIG_DIR) DEFAULT_DATADIR=$(DATA_DIR) $(MAKE) goversion crowdsec cscli plugins
 	@install -m 0755 cmd/crowdsec/crowdsec cmd/crowdsec-cli/cscli $(BIN_DIR)/
 	@install -m 0755 plugins/notifications/*/notification-* $(PLUGIN_DIR)/
-	@BINCOVER_TESTING=$(BINCOVER_TESTING) DEFAULT_CONFIGDIR=$(CONFIG_DIR) DEFAULT_DATADIR=$(DATA_DIR) $(MAKE) goversion crowdsec-bincover cscli-bincover
-	@install -m 0755 cmd/crowdsec/crowdsec.cover cmd/crowdsec-cli/cscli.cover $(BIN_DIR)/
 
 # Create a reusable package with initial configuration + data
 bats-fixture:

test/bats/01_crowdsec.bats

@@ -77,8 +77,8 @@ teardown() {
     log_old="${logdir1}/crowdsec.log"
     config_set ".common.log_dir=\"${logdir1}\""
 
-    rune -0 ./instance-crowdsec start
-    # PID="$output"
+    rune -0 ./instance-crowdsec start-pid
+    PID="$output"
     assert_file_exist "$log_old"
     assert_file_contains "$log_old" "Starting processing data"
 
@@ -90,15 +90,7 @@ teardown() {
 
     sleep 5
 
-    # this won't work as crowdsec-wrapper does not relay the signal
-    # rune -0 kill -HUP "$PID"
-
-    # During functional tests, crowdsec is often run from a wrapper script,
-    # which captures its output (for coverage reports) and cannot relay signals
-    # at the same time. So instead of sending a SIGHUP to the wrapper, we send
-    # it to the crowdsec process by name - with or without coverage.
-    rune pkill -HUP -f "$BIN_DIR/crowdsec.cover"
-    rune pkill -HUP -f "$BIN_DIR/crowdsec"
+    rune -0 kill -HUP "$PID"
 
     for ((i=0; i<10; i++)); do
         sleep 1

test/bin/assert-crowdsec-not-running

@@ -2,7 +2,7 @@
 
 is_crowdsec_running() {
     # ignore processes in containers
-    PIDS=$(pgrep --ns $$ -x 'crowdsec|crowdsec.test|crowdsec.cover')
+    PIDS=$(pgrep --ns $$ -x 'crowdsec|crowdsec.test')
 }
 
 # The process can be slow, especially on CI and during test coverage.

test/bin/check-requirements

@@ -87,12 +87,6 @@ check_cfssljson() {
     fi
 }
 
-check_gocovmerge() {
-    if ! command -v gocovmerge >/dev/null; then
-        die "missing required program 'gocovmerge'. You can install it with \"go install github.com/wadey/gocovmerge@latest\""
-    fi
-}
-
 check_bats_core
 check_curl
 check_daemonizer
@@ -104,7 +98,3 @@ check_base64
 check_python3
 check_yq
 check_pkill
-if [[ -n "${TEST_COVERAGE}" ]]; then
-    check_gocovmerge
-fi
-

test/bin/crowdsec-wrapper

@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-
-set -eu
-
-#
-# Delegate operations to an instrumented binary and collects coverage data.
-#
-
-#shellcheck disable=SC1007
-THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
-# no need to change directory, and doing it here would break hub tests
-# shellcheck disable=SC1091
-. "${THIS_DIR}/../.environment.sh"
-
-set -o pipefail  # don't let sed hide the statuscode
-mkdir -p "${LOCAL_DIR}/var/lib/coverage"
-
-# this would be nice but doesn't work, since the binary is not running in background
-#_hup() {
-#    echo "pkill -1 crowdsec.cover"
-#    pkill -HUP crowdsec.cover
-#}
-#
-## relay the "configuration reload" signal
-#trap _hup SIGHUP
-
-# we collect rc and output by hand, because setting -o pipefail would trigger a
-# SIGPIPE.
-set +e
-
-# Arguments to crowdsec are passed through a temporary, newline-delimited
-# file courtesy of github.com/confluentinc/bincover. Coverage data will be
-# merged at the end of the test run.
-# The '=' between flags and values is required.
-output=$("${BIN_DIR}/crowdsec.cover" \
-    -test.run="^TestBincoverRunMain$" \
-    -test.coverprofile="${LOCAL_DIR}/var/lib/coverage/crowdsec-$(date +'%s')-$$-${RANDOM}.out" \
-    -args-file=<(for i; do echo "${i}"; done))
-rc=$?
-
-# If there is bincover metadata, we take the status code from there. Otherwise,
-# we keep the status from the above command.
-if [[ ${output} =~ (.*)(START_BINCOVER_METADATA[[:space:]]*)(.*)([[:space:]]END_BINCOVER_METADATA) ]]; then
-    echo -n "${BASH_REMATCH[1]}"
-    exit "$(jq '.exit_code' <<< "${BASH_REMATCH[3]}")"
-fi
-
-echo -n "${output}"
-exit "${rc}"

test/bin/cscli-wrapper

@@ -1,40 +0,0 @@
-#!/usr/bin/env bash
-
-set -eu
-
-#
-# Delegate operations to an instrumented binary and collects coverage data.
-#
-
-#shellcheck disable=SC1007
-THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
-# no need to change directory, and doing it here would break hub tests
-# shellcheck disable=SC1091
-. "${THIS_DIR}/../.environment.sh"
-
-set -o pipefail  # don't let sed hide the statuscode
-mkdir -p "${LOCAL_DIR}/var/lib/coverage"
-
-# we collect rc and output by hand, because setting -o pipefail would trigger a
-# SIGPIPE.
-set +e
-
-# Arguments to cscli are passed through a temporary, newline-delimited
-# file courtesy of github.com/confluentinc/bincover. Coverage data will be
-# merged at the end of the test run.
-# The '=' between flags and values is required.
-output=$("${BIN_DIR}/cscli.cover" \
-    -test.run="^TestBincoverRunMain$" \
-    -test.coverprofile="${LOCAL_DIR}/var/lib/coverage/cscli-$(date +'%s')-$$-${RANDOM}.out" \
-    -args-file=<(for i; do echo "${i}"; done))
-rc=$?
-
-# If there is bincover metadata, we take the status code from there. Otherwise,
-# we keep the status from the above command.
-if [[ ${output} =~ (.*)(START_BINCOVER_METADATA[[:space:]]*)(.*)([[:space:]]END_BINCOVER_METADATA) ]]; then
-    echo -n "${BASH_REMATCH[1]}"
-    exit "$(jq '.exit_code' <<< "${BASH_REMATCH[3]}")"
-fi
-
-echo -n "${output}"
-exit "${rc}"

test/lib/config/config-local

@@ -120,11 +120,11 @@ make_init_data() {
     ./instance-db config-yaml
     ./instance-db setup
 
-    "${CSCLI}" machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto
-    "${CSCLI}" hub update
-    "${CSCLI}" collections install crowdsecurity/linux
+    "$CSCLI" machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto
+    "$CSCLI" hub update
+    "$CSCLI" collections install crowdsecurity/linux
 
-    mkdir -p "${LOCAL_INIT_DIR}"
+    mkdir -p "$LOCAL_INIT_DIR"
 
     ./instance-db dump "${LOCAL_INIT_DIR}/database"
 

test/run-tests

@@ -20,8 +20,6 @@ if [[ -z "${TEST_COVERAGE}" ]]; then
     echo "Coverage report: no"
 else
     echo "Coverage report: yes"
-    rm -f "${LOCAL_DIR}/var/lib/coverage/*"
-    mkdir -p "${LOCAL_DIR}/var/lib/coverage"
 fi
 
 dump_backend="$(cat "${LOCAL_INIT_DIR}/.backend")"
@@ -44,11 +42,3 @@ else
         --print-output-on-failure \
         "${TEST_DIR}/bats" "${TEST_DIR}/dyn-bats"
 fi
-
-if [[ -n "${TEST_COVERAGE}" ]]; then
-    # empty files just to avoid merge errors
-    touch "${LOCAL_DIR}"/var/lib/coverage/crowdsec-
-    touch "${LOCAL_DIR}"/var/lib/coverage/cscli-
-    gocovmerge "${LOCAL_DIR}"/var/lib/coverage/crowdsec-* > "${LOCAL_DIR}/var/lib/coverage/coverage-crowdsec.out"
-    gocovmerge "${LOCAL_DIR}"/var/lib/coverage/cscli-* > "${LOCAL_DIR}/var/lib/coverage/coverage-cscli.out"
-fi