more func test coverage; capture exit code for bincover (#1425)

.github/workflows/bats-mysql.yml

@@ -11,6 +11,7 @@ on:
         required: true
 
 jobs:
+
   build:
     name: "Build + tests"
     runs-on: ubuntu-latest
@@ -22,6 +23,7 @@ jobs:
           MYSQL_ROOT_PASSWORD: ${{ secrets.DATABASE_PASSWORD }}
         ports:
           - 3306:3306
+
     steps:
 
     - name: "Force machineid"

.github/workflows/bats-postgres.yml

@@ -7,6 +7,7 @@ on:
         required: true
 
 jobs:
+
   build:
     name: "Build + tests"
     runs-on: ubuntu-latest
@@ -23,6 +24,7 @@ jobs:
           --health-interval 10s
           --health-timeout 5s
           --health-retries 5
+
     steps:
 
     - name: "Force machineid"

.github/workflows/bats-sqlite-coverage.yml

@@ -4,10 +4,12 @@ on:
   workflow_call:
 
 jobs:
+
   build:
     name: "Build + tests"
     runs-on: ubuntu-latest
-    timeout-minutes: 30
+    timeout-minutes: 20
+
     steps:
 
     - name: "Force machineid"
@@ -31,23 +33,25 @@ jobs:
       run: |
         sudo apt install -y -qq build-essential daemonize jq netcat-openbsd
         GO111MODULE=on go get github.com/mikefarah/yq/v4
+        sudo cp -u ~/go/bin/yq /usr/local/bin/
         go install github.com/wadey/gocovmerge@latest
-        sudo cp -u ~/go/bin/yq ~/go/bin/gocovmerge /usr/local/bin/
+        sudo cp -u ~/go/bin/gocovmerge /usr/local/bin/
 
     - name: "Build crowdsec and fixture"
       run: TEST_COVERAGE=true make bats-clean bats-build bats-fixture
 
-    - name: "Run tests (with coverage)"
-      run: TEST_COVERAGE=true make bats-test
+    - name: "Run tests"
+      run: |
+        TEST_COVERAGE=true make bats-test
+        bzip2 ./tests/local/var/lib/coverage/coverage-bats.out
 
-    - name: Upload coverage report
+    - name: "Coverage report artifact"
       uses: actions/upload-artifact@v2
       with:
-        name: coverage-bats.out
-        path: ./tests/local/var/lib/coverage/coverage-bats.out
+        name: coverage-bats.out.bz2
+        path: ./tests/local/var/lib/coverage/coverage-bats.out.bz2
 
     - name: "Show crowdsec logs"
       run:
         for file in $(find ./tests/local/var/log -type f); do echo ">>>>> $file"; cat $file; echo; done
       if: ${{ always() }}
-

.github/workflows/ci_tests.yml

@@ -71,17 +71,20 @@ jobs:
       - name: Download unit report
         uses: actions/download-artifact@v3
         with:
-          name: coverage.out
+          name: coverage.out.bz2
 
       - name: Download bats report
         uses: actions/download-artifact@v3
         with:
-          name: coverage-bats.out
+          name: coverage-bats.out.bz2
 
       - name: merge coverage reports
         run: |
           go get -u github.com/wadey/gocovmerge
+          bunzip2 coverage.out.bz2
+          bunzip2 coverage-bats.out.bz2
           ~/go/bin/gocovmerge coverage.out coverage-bats.out > coverage-all.out
+          bzip2 <coverage-all.out >coverage-all.out.bz2
 
       - name: gcov2lcov
         uses: jandelgado/gcov2lcov-action@v1.0.8
@@ -89,6 +92,12 @@ jobs:
           infile: coverage-all.out
           outfile: coverage-all.txt
 
+      - name: Coverage report artifact (merged)
+        uses: actions/upload-artifact@v2
+        with:
+          name: coverage-all.out.bz2
+          path: ./coverage-all.out.bz2
+
       - name: Coveralls
         uses: coverallsapp/github-action@master
         continue-on-error: true

.github/workflows/go-tests.yml

@@ -1,6 +1,6 @@
 name: Go tests
 
-#those env variables are for localstack, so we can emulate aws services
+# these env variables are for localstack, so we can emulate aws services
 env:
   AWS_HOST: localstack
   SERVICES: cloudwatch,logs,kinesis
@@ -42,22 +42,31 @@ jobs:
           --health-interval=10s
           --health-timeout=5s
           --health-retries=3
+
     steps:
-    - name: Set up Go 1.17
+
+    - name: "Set up Go 1.17"
       uses: actions/setup-go@v3
       with:
         go-version: 1.17
       id: go
-    - name: Check out code into the Go module directory
+
+    - name: "Clone CrowdSec"
       uses: actions/checkout@v3
+      with:
+        fetch-depth: 0
+        submodules: false
+
     - name: Build
       run: make build && go get -u github.com/jandelgado/gcov2lcov && go get -u github.com/ory/go-acc
-    - name: All tests
-      run: go run github.com/ory/go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models
 
-    - name: Upload coverage report
+    - name: "Run tests"
+      run: |
+        go run github.com/ory/go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models
+        bzip2 ./coverage.out
+
+    - name: "Coverage report artifact"
       uses: actions/upload-artifact@v2
       with:
-        name: coverage.out
-        path: ./coverage.out
-
+        name: coverage.out.bz2
+        path: ./coverage.out.bz2

Makefile

@@ -48,8 +48,10 @@ BUILD_TIMESTAMP = $(shell date +%F"_"%T)
 BUILD_TAG ?= "$(shell git rev-parse HEAD)"
 DEFAULT_CONFIGDIR ?= "/etc/crowdsec"
 DEFAULT_DATADIR ?= "/var/lib/crowdsec/data"
+BINCOVER_TESTING ?= false
 
 LD_OPTS_VARS= \
+-X github.com/crowdsecurity/crowdsec/cmd/crowdsec/main.bincoverTesting=$(BINCOVER_TESTING) \
 -X github.com/crowdsecurity/crowdsec/pkg/cwversion.Version=$(BUILD_VERSION) \
 -X github.com/crowdsecurity/crowdsec/pkg/cwversion.BuildDate=$(BUILD_TIMESTAMP) \
 -X github.com/crowdsecurity/crowdsec/pkg/cwversion.Codename=$(BUILD_CODENAME) \

cmd/crowdsec-cli/main.go

@@ -153,7 +153,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
 		log.Fatalf("failed to make branch hidden : %s", err)
 	}
 
-	if len(os.Args) > 1 && os.Args[1] != "completion" {
+	if len(os.Args) > 1 && os.Args[1] != "completion" && os.Args[1] != "version" && os.Args[1] != "help" {
 		cobra.OnInitialize(initConfig)
 	}
 

cmd/crowdsec-cli/utils.go

@@ -29,7 +29,7 @@ import (
 func printHelp(cmd *cobra.Command) {
 	err := cmd.Help()
 	if err != nil {
-		log.Fatalf("uname to print help(): %s", err)
+		log.Fatalf("unable to print help(): %s", err)
 	}
 }
 

cmd/crowdsec/main.go

@@ -10,6 +10,7 @@ import (
 	_ "net/http/pprof"
 	"time"
 
+	"github.com/confluentinc/bincover"
 	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/csplugin"
@@ -50,6 +51,8 @@ var (
 	pluginBroker      csplugin.PluginBroker
 )
 
+const bincoverTesting = false
+
 type Flags struct {
 	ConfigFile     string
 	TraceLevel     bool
@@ -303,10 +306,13 @@ func main() {
 		go registerPrometheus(cConfig.Prometheus)
 	}
 
-	if rc, err := Serve(cConfig); err != nil {
+	if exitCode, err := Serve(cConfig); err != nil {
 		if err != nil {
 			log.Errorf(err.Error())
-			os.Exit(rc)
+			if !bincoverTesting {
+				os.Exit(exitCode)
+			}
+			bincover.ExitCode = exitCode
 		}
 	}
 }

tests/bats.mk

@@ -30,11 +30,13 @@ DB_BACKEND ?= sqlite
 ifdef TEST_COVERAGE
   CROWDSEC = "$(TEST_DIR)/crowdsec-wrapper"
   CSCLI = "$(TEST_DIR)/cscli-wrapper"
+  BINCOVER_TESTING = true
 else
   # the wrappers should work here too - it detects TEST_COVERAGE - but we allow
   # overriding the path to the binaries
   CROWDSEC ?= "$(BIN_DIR)/crowdsec"
   CSCLI ?= "$(BIN_DIR)/cscli"
+  BINCOVER_TESTING = false
 endif
 
 # If you change the name of the crowdsec executable, make sure the pgrep
@@ -71,10 +73,10 @@ bats-check-requirements:
 # Build and installs crowdsec in a local directory. Rebuilds if already exists.
 bats-build: bats-environment bats-check-requirements
 	@mkdir -p $(BIN_DIR) $(LOG_DIR) $(PID_DIR) $(PLUGIN_DIR)
-	@DEFAULT_CONFIGDIR=$(CONFIG_DIR) DEFAULT_DATADIR=$(DATA_DIR) $(MAKE) goversion crowdsec cscli plugins
+	@BINCOVER_TESTING=$(BINCOVER_TESTING) DEFAULT_CONFIGDIR=$(CONFIG_DIR) DEFAULT_DATADIR=$(DATA_DIR) $(MAKE) goversion crowdsec cscli plugins
 	@install -m 0755 cmd/crowdsec/crowdsec cmd/crowdsec-cli/cscli $(BIN_DIR)/
 	@install -m 0755 plugins/notifications/*/notification-* $(PLUGIN_DIR)/
-	@DEFAULT_CONFIGDIR=$(CONFIG_DIR) DEFAULT_DATADIR=$(DATA_DIR) $(MAKE) goversion crowdsec-bincover cscli-bincover
+	@BINCOVER_TESTING=$(BINCOVER_TESTING) DEFAULT_CONFIGDIR=$(CONFIG_DIR) DEFAULT_DATADIR=$(DATA_DIR) $(MAKE) goversion crowdsec-bincover cscli-bincover
 	@install -m 0755 cmd/crowdsec/crowdsec.cover cmd/crowdsec-cli/cscli.cover $(BIN_DIR)/
 
 # Create a reusable package with initial configuration + data

tests/bats/01_base.bats

@@ -44,6 +44,22 @@ declare stderr
     assert_output --partial "Constraint_scenario:"
     assert_output --partial "Constraint_api:"
     assert_output --partial "Constraint_acquis:"
+
+    # should work without configuration file
+    rm "${CONFIG_YAML}"
+    run -0 cscli version
+    assert_output --partial "version:"
+}
+
+@test "$FILE cscli help" {
+    run -0 cscli help
+    assert_line "Available Commands:"
+    assert_line --regexp ".* help .* Help about any command"
+
+    # should work without configuration file
+    rm "${CONFIG_YAML}"
+    run -0 cscli help
+    assert_line "Available Commands:"
 }
 
 @test "$FILE cscli alerts list: at startup returns at least one entry: community pull" {
@@ -155,3 +171,29 @@ declare stderr
     run -0 cscli completion zsh
     assert_output --partial "# zsh completion for cscli"
 }
+
+@test "$FILE cscli hub list" {
+    run -0 cscli hub list -o human
+    assert_line --regexp '^ crowdsecurity/linux'
+    assert_line --regexp '^ crowdsecurity/sshd'
+    assert_line --regexp '^ crowdsecurity/dateparse-enrich'
+    assert_line --regexp '^ crowdsecurity/geoip-enrich'
+    assert_line --regexp '^ crowdsecurity/sshd-logs'
+    assert_line --regexp '^ crowdsecurity/syslog-logs'
+    assert_line --regexp '^ crowdsecurity/ssh-bf'
+    assert_line --regexp '^ crowdsecurity/ssh-slow-bf'
+
+    run -0 cscli hub list -o raw
+    assert_line --regexp '^crowdsecurity/linux,enabled,[0-9]+\.[0-9]+,core linux support : syslog\+geoip\+ssh,collections$'
+    assert_line --regexp '^crowdsecurity/sshd,enabled,[0-9]+\.[0-9]+,sshd support : parser and brute-force detection,collections$'
+    assert_line --regexp '^crowdsecurity/dateparse-enrich,enabled,[0-9]+\.[0-9]+,,parsers$'
+    assert_line --regexp '^crowdsecurity/geoip-enrich,enabled,[0-9]+\.[0-9]+,"Populate event with geoloc info : as, country, coords, source range.",parsers$'
+    assert_line --regexp '^crowdsecurity/sshd-logs,enabled,[0-9]+\.[0-9]+,Parse openSSH logs,parsers$'
+    assert_line --regexp '^crowdsecurity/syslog-logs,enabled,[0-9]+\.[0-9]+,,parsers$'
+    assert_line --regexp '^crowdsecurity/ssh-bf,enabled,[0-9]+\.[0-9]+,Detect ssh bruteforce,scenarios$'
+    assert_line --regexp '^crowdsecurity/ssh-slow-bf,enabled,[0-9]+\.[0-9]+,Detect slow ssh bruteforce,scenarios$'
+
+    run -0 cscli hub list -o json
+    run jq -c '[[.collections[].name], [.parsers[].name], [.scenarios[].name]]' <(output)
+    assert_output '[["crowdsecurity/linux","crowdsecurity/sshd"],["crowdsecurity/dateparse-enrich","crowdsecurity/geoip-enrich","crowdsecurity/sshd-logs","crowdsecurity/syslog-logs"],["crowdsecurity/ssh-bf","crowdsecurity/ssh-slow-bf"]]'
+}

tests/bats/80_alerts.bats

@@ -21,6 +21,8 @@ teardown() {
     ./instance-crowdsec stop
 }
 
+declare stderr
+
 #----------
 
 @test "$FILE cscli alerts list, with and without --machine" {
@@ -43,3 +45,99 @@ teardown() {
     assert_output --regexp "\| 'githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})?' \|"
     assert_output --regexp "\| githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})? \|"
 }
+
+@test "$FILE cscli alerts list, human/json/raw" {
+    run -0 cscli decisions add -i 10.20.30.40 -t ban
+
+    run -0 cscli alerts list -o human
+    assert_output --regexp ".* ID .* VALUE .* REASON .* COUNTRY .* AS .* DECISIONS .* CREATED AT .*"
+    assert_output --regexp ".*Ip:10.20.30.40.*manual 'ban' from.*ban:1.*"
+
+    run -0 cscli alerts list -o json
+    run -0 jq -c '.[].decisions[0] | [.origin, .scenario, .scope, .simulated, .type, .value]' <(output)
+    assert_line "[\"cscli\",\"manual 'ban' from 'githubciXXXXXXXXXXXXXXXXXXXXXXXX'\",\"Ip\",false,\"ban\",\"10.20.30.40\"]"
+
+    run -0 cscli alerts list -o raw
+    assert_line "id,scope,value,reason,country,as,decisions,created_at"
+    assert_line --regexp ".*,Ip,10.20.30.40,manual 'ban' from 'githubciXXXXXXXXXXXXXXXXXXXXXXXX',,\" \",ban:1,.*"
+
+    run -0 cscli alerts list -o raw --machine
+    assert_line "id,scope,value,reason,country,as,decisions,created_at,machine"
+    assert_line --regexp "^[0-9]+,Ip,10.20.30.40,manual 'ban' from 'githubciXXXXXXXXXXXXXXXXXXXXXXXX',,\" \",ban:1,.*,githubciXXXXXXXXXXXXXXXXXXXXXXXX$"
+}
+
+@test "$FILE cscli alerts inspect" {
+    run -0 cscli decisions add -i 10.20.30.40 -t ban
+    run -0 cscli alerts list -o raw <(output)
+    run -0 grep 10.20.30.40 <(output)
+    run -0 cut -d, -f1 <(output)
+    ALERT_ID="$output"
+
+    run -0 cscli alerts inspect "$ALERT_ID" -o human
+    assert_line --regexp '^#+$'
+    assert_line --regexp "^ - ID *: $ALERT_ID$"
+    assert_line --regexp "^ - Date *: .*$"
+    assert_line --regexp "^ - Machine *: githubciXXXXXXXXXXXXXXXXXXXXXXXX"
+    assert_line --regexp "^ - Simulation *: false$"
+    assert_line --regexp "^ - Reason *: manual 'ban' from 'githubciXXXXXXXXXXXXXXXXXXXXXXXX'$"
+    assert_line --regexp "^ - Events Count *: 1$"
+    assert_line --regexp "^ - Scope:Value *: Ip:10.20.30.40$"
+    assert_line --regexp "^ - Country *: *$"
+    assert_line --regexp "^ - AS *: *$"
+    assert_line --regexp "^ - Begin *: .*$"
+    assert_line --regexp "^ - End *: .*$"
+    assert_line --regexp "^ - Active Decisions *:$"
+    assert_line --regexp "^.* ID .* SCOPE:VALUE .* ACTION .* EXPIRATION .* CREATED AT .*$"
+    assert_line --regexp "^.* Ip:10.20.30.40 .* ban .*$"
+
+    run -0 cscli alerts inspect "$ALERT_ID" -o human --details
+    # XXX can we have something here?
+
+    run -0 cscli alerts inspect "$ALERT_ID" -o raw
+    assert_line --regexp "^ *capacity: 0$"
+    assert_line --regexp "^ *id: $ALERT_ID$"
+    assert_line --regexp "^ *origin: cscli$"
+    assert_line --regexp "^ *scenario: manual 'ban' from 'githubciXXXXXXXXXXXXXXXXXXXXXXXX'$"
+    assert_line --regexp "^ *scope: Ip$"
+    assert_line --regexp "^ *simulated: false$"
+    assert_line --regexp "^ *type: ban$"
+    assert_line --regexp "^ *value: 10.20.30.40$"
+
+    run -0 cscli alerts inspect "$ALERT_ID" -o json
+    alert=$output
+    run jq -c '.decisions[] | [.origin,.scenario,.scope,.simulated,.type,.value]' <<< "$alert"
+    assert_output "[\"cscli\",\"manual 'ban' from 'githubciXXXXXXXXXXXXXXXXXXXXXXXX'\",\"Ip\",false,\"ban\",\"10.20.30.40\"]"
+    run jq -c '.source' <<< "$alert"
+    assert_output '{"ip":"10.20.30.40","scope":"Ip","value":"10.20.30.40"}'
+}
+
+@test "$FILE no active alerts" {
+    run -0 cscli alerts list --until 200d -o human
+    assert_output "No active alerts"
+    run -0 cscli alerts list --until 200d -o json
+    assert_output "null"
+    run -0 cscli alerts list --until 200d -o raw
+    assert_output "id,scope,value,reason,country,as,decisions,created_at"
+    run -0 cscli alerts list --until 200d -o raw --machine
+    assert_output "id,scope,value,reason,country,as,decisions,created_at,machine"
+}
+
+@test "$FILE cscli alerts delete" {
+    run -0 --separate-stderr cscli alerts delete --all
+    run echo "$stderr"
+    assert_output --partial 'alert(s) deleted'
+
+    # XXX TODO: delete by scope, id, value, scenario, range..
+}
+
+@test "$FILE bad duration" {
+    skip 'TODO'
+    run -0 cscli decisions add -i 10.20.30.40 -t ban
+    run -9 cscli decisions list --ip 10.20.30.40 -o json
+    run -9 jq -r '.[].decisions[].id' <(output)
+    DECISION_ID="$output"
+
+    ./instance-crowdsec stop
+    run -0 ./instance-db exec_sql "UPDATE decisions SET ... WHERE id=$DECISION_ID"
+    ./instance-crowdsec start
+}

tests/bats/90_decisions.bats

@@ -57,3 +57,13 @@ declare stderr
     assert_output --regexp "\| 'githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})?' \|"
     assert_output --regexp "\| githubciXXXXXXXXXXXXXXXXXXXXXXXX([a-zA-Z0-9]{16})? \|"
 }
+
+@test "$FILE cscli decisions list, incorrect parameters" {
+    run -1 --separate-stderr cscli decisions list --until toto
+    run echo "$stderr"
+    assert_output --partial 'Unable to list decisions : performing request: API error: while parsing duration: time: invalid duration \"toto\"'
+    run -1 --separate-stderr cscli decisions list --until toto -o json
+    run echo "$stderr"
+    run -0 jq -c '[.level, .msg]' <(output)
+    assert_output '["fatal","Unable to list decisions : performing request: API error: while parsing duration: time: invalid duration \"toto\""]'
+}

tests/lib/db/instance-mysql

@@ -114,6 +114,10 @@ case "$1" in
         shift
         restore "$@"
         ;;
+    exec_sql)
+        shift
+        exec_sql "$@"
+        ;;
     *)
         about
         ;;

tests/lib/db/instance-postgres

@@ -89,6 +89,10 @@ case "$1" in
         shift
         restore "$@"
         ;;
+    exec_sql)
+        shift
+        exec_sql "$@"
+        ;;
     *)
         about
         ;;

tests/lib/db/instance-sqlite

@@ -20,6 +20,11 @@ cd "${THIS_DIR}"/../../
 #shellcheck disable=SC1091
 . ./.environment.sh
 
+exec_sql() {
+    cmd="${1?Missing required sql command}"
+    sqlite3 "${DB_FILE}" "$@"
+}
+
 # you have not removed set -u above, have you?
 
 [ -z "${CONFIG_YAML-}" ] && die "\$CONFIG_YAML must be defined."
@@ -56,6 +61,10 @@ case "$1" in
         [ -f "$backup_file" ] || die "missing file $backup_file"
         cp "$backup_file" "${DB_FILE}"
         ;;
+    exec_sql)
+        shift
+        exec_sql "$@"
+        ;;
     *)
         about
         ;;