beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

CI: colored test output, colored crowdsec and crowdsec-api logs, full final db dump for mysql and sqlite (#1596)

Browse source 
* github-ci: color unit test output and logs
* new config option: force_color_logs (useful in CI)
* bats: show sqlite/mysql dump at the end
* removed "-v" (print package names) from "go build"
* general workflow cleanup
This commit is contained in:
mmetc 2022-06-17 16:12:49 +02:00 committed by GitHub
parent a2d91119d4
commit c78c833400
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
42 changed files with 289 additions and 188 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
.github/workflows/.yamllint vendored Symbolic link
View file

@ -0,0 +1 @@
../../.yamllint

5
.github/workflows/bats-hub.yml vendored
View file

@ -8,6 +8,9 @@ on:
GIST_BADGES_ID:
required: true
env:
PREFIX_TEST_NAMES_WITH_FILE: true
jobs:
build:
name: "Build + tests"
@ -34,7 +37,7 @@ jobs:
- name: "Install bats dependencies"
run: |
sudo apt install -y -qq build-essential daemonize jq netcat-openbsd
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
go install github.com/mikefarah/yq/v4@latest
go install github.com/cloudflare/cfssl/cmd/cfssl@latest
go install github.com/cloudflare/cfssl/cmd/cfssljson@latest

33
.github/workflows/bats-mysql.yml vendored
View file

@ -1,4 +1,4 @@
name: Functional tests with MySQL
name: Functional tests (MySQL)
on:
workflow_call:
@ -10,6 +10,9 @@ on:
DATABASE_PASSWORD:
required: true
env:
PREFIX_TEST_NAMES_WITH_FILE: true
jobs:
build:
@ -37,7 +40,7 @@ jobs:
go-version: 1.17
id: go
- name: "Clone CrowdSec"
- name: "Check out CrowdSec repository"
uses: actions/checkout@v3
with:
fetch-depth: 0
@ -45,16 +48,15 @@ jobs:
- name: "Install bats dependencies"
run: |
sudo apt install -y -qq build-essential daemonize jq netcat-openbsd
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
go install github.com/mikefarah/yq/v4@latest
go install github.com/cloudflare/cfssl/cmd/cfssl@latest
go install github.com/cloudflare/cfssl/cmd/cfssljson@latest
sudo cp -u ~/go/bin/yq /usr/local/bin/
sudo cp -u ~/go/bin/cfssl /usr/local/bin
sudo cp -u ~/go/bin/cfssljson /usr/local/bin
sudo cp -u ~/go/bin/yq ~/go/bin/cfssl ~/go/bin/cfssljson /usr/local/bin/
- name: "Build crowdsec and fixture"
run: make bats-clean bats-build bats-fixture
run: |
make clean bats-build bats-fixture
env:
DB_BACKEND: mysql
MYSQL_HOST: 127.0.0.1
@ -71,6 +73,22 @@ jobs:
MYSQL_PASSWORD: ${{ secrets.DATABASE_PASSWORD }}
MYSQL_USER: root
#
# In case you need to inspect the database status after the failure of a given test
#
# - name: "Run specified tests"
# run: ./tests/run-tests tests/bats/<filename>.bats -f "<test name>"
- name: Show database dump
run: ./tests/instance-db dump /dev/fd/1
env:
DB_BACKEND: mysql
MYSQL_HOST: 127.0.0.1
MYSQL_PORT: 3306
MYSQL_PASSWORD: ${{ secrets.DATABASE_PASSWORD }}
MYSQL_USER: root
if: ${{ always() }}
- name: "Show crowdsec logs"
run:
for file in $(find ./tests/local/var/log -type f); do echo ">>>>> $file"; cat $file; echo; done
@ -79,4 +97,3 @@ jobs:
- name: "Show database logs"
run: docker logs "${{ job.services.database.id }}"
if: ${{ always() }}

17
.github/workflows/bats-postgres.yml vendored
View file

@ -1,4 +1,4 @@
name: Functional tests with PostgreSQL
name: Functional tests (Postgres)
on:
workflow_call:
@ -6,6 +6,9 @@ on:
DATABASE_PASSWORD:
required: true
env:
PREFIX_TEST_NAMES_WITH_FILE: true
jobs:
build:
@ -38,7 +41,7 @@ jobs:
go-version: 1.17
id: go
- name: "Clone CrowdSec"
- name: "Check out CrowdSec repository"
uses: actions/checkout@v3
with:
fetch-depth: 0
@ -46,16 +49,15 @@ jobs:
- name: "Install bats dependencies"
run: |
sudo apt install -y -qq build-essential daemonize jq netcat-openbsd
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
go install github.com/mikefarah/yq/v4@latest
go install github.com/cloudflare/cfssl/cmd/cfssl@latest
go install github.com/cloudflare/cfssl/cmd/cfssljson@latest
sudo cp -u ~/go/bin/yq /usr/local/bin/
sudo cp -u ~/go/bin/cfssl /usr/local/bin
sudo cp -u ~/go/bin/cfssljson /usr/local/bin
sudo cp -u ~/go/bin/yq ~/go/bin/cfssl ~/go/bin/cfssljson /usr/local/bin/
- name: "Build crowdsec and fixture (DB_BACKEND: pgx)"
run: make clean bats-build bats-fixture
run: |
make clean bats-build bats-fixture
env:
DB_BACKEND: pgx
PGHOST: 127.0.0.1
@ -98,4 +100,3 @@ jobs:
- name: "Show database logs"
run: docker logs "${{ job.services.database.id }}"
if: ${{ always() }}

30
.github/workflows/bats-sqlite-coverage.yml vendored
View file

@ -1,8 +1,12 @@
name: Functional tests with sqlite
name: Functional tests (sqlite)
on:
workflow_call:
env:
PREFIX_TEST_NAMES_WITH_FILE: true
TEST_COVERAGE: true
jobs:
build:
@ -23,7 +27,7 @@ jobs:
go-version: 1.17
id: go
- name: "Clone CrowdSec"
- name: "Check out CrowdSec repository"
uses: actions/checkout@v3
with:
fetch-depth: 0
@ -31,19 +35,32 @@ jobs:
- name: "Install bats dependencies"
run: |
sudo apt install -y -qq build-essential daemonize jq netcat-openbsd
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
go install github.com/mikefarah/yq/v4@latest
go install github.com/cloudflare/cfssl/cmd/cfssl@latest
go install github.com/cloudflare/cfssl/cmd/cfssljson@latest
sudo cp -u ~/go/bin/yq ~/go/bin/cfssl ~/go/bin/cfssljson /usr/local/bin/
go install github.com/wadey/gocovmerge@latest
sudo cp -u ~/go/bin/yq ~/go/bin/gocovmerge ~/go/bin/cfssl ~/go/bin/cfssljson /usr/local/bin/
sudo cp -u ~/go/bin/gocovmerge /usr/local/bin/
- name: "Build crowdsec and fixture"
run: TEST_COVERAGE=true make bats-clean bats-build bats-fixture
run: |
make clean bats-build bats-fixture
- name: "Run tests"
run: make bats-test
#
# In case you need to inspect the database status after the failure of a given test
#
# - name: "Run specified tests"
# run: ./tests/run-tests tests/bats/<filename>.bats -f "<test name>"
- name: "Show database dump"
run: |
TEST_COVERAGE=true make bats-test
./tests/instance-crowdsec stop
sqlite3 ./tests/local/var/lib/crowdsec/data/crowdsec.db '.dump'
if: ${{ always() }}
- name: "Show crowdsec logs"
run:
@ -61,4 +78,3 @@ jobs:
with:
files: ./tests/local/var/lib/coverage/coverage-cscli.out
flags: func-cscli

37
.github/workflows/ci_tests.yml → .github/workflows/bats.yml vendored
View file

@ -1,39 +1,31 @@
name: Tests
name: Bats
# Main workflow for tests, it calls all the others through parallel jobs.
#
# A final step collects and merges coverage output, then pushes it to
# coveralls.io
# Main workflow for functional tests, it calls all the others through parallel jobs.
#
# https://docs.github.com/en/actions/using-workflows/reusing-workflows
#
# There is no need to merge coverage output because codecov.io should take care of that.
on:
push:
branches:
- master
- testing*
branches: [master]
paths-ignore:
- 'README.md'
pull_request:
branches:
- master
- testing*
branches: [master]
paths-ignore:
- 'README.md'
jobs:
go-tests:
uses: ./.github/workflows/go-tests.yml
bats-sqlite:
sqlite:
uses: ./.github/workflows/bats-sqlite-coverage.yml
# Jobs for Postgres and MySQL can have failing tests on GitHub CI, but they
# pass when run on devs' machines or in the release checks. We disable them
# here by default. Remove the if..false to enable them.
# Jobs for Postgres (and sometimes MySQL) can have failing tests on GitHub
# CI, but they pass when run on devs' machines or in the release checks. We
# disable them here by default. Remove the if..false to enable them.
bats-mariadb:
mariadb:
if: ${{ false }}
uses: ./.github/workflows/bats-mysql.yml
with:
@ -41,7 +33,7 @@ jobs:
secrets:
DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD}}
bats-mysql:
mysql:
if: ${{ false }}
uses: ./.github/workflows/bats-mysql.yml
with:
@ -49,15 +41,14 @@ jobs:
secrets:
DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD}}
bats-postgres:
postgres:
if: ${{ false }}
uses: ./.github/workflows/bats-postgres.yml
secrets:
DATABASE_PASSWORD: ${{ secrets.DATABASE_PASSWORD}}
bats-hub:
hub:
uses: ./.github/workflows/bats-hub.yml
secrets:
GIST_BADGES_ID: ${{ secrets.GIST_BADGES_ID }}
GIST_BADGES_SECRET: ${{ secrets.GIST_BADGES_SECRET }}

40
.github/workflows/ci-go-test-windows.yml vendored
View file

@ -1,40 +0,0 @@
name: tests-windows
on:
push:
branches: [ master ]
paths-ignore:
- 'docs/**'
- 'mkdocs.yml'
- 'README.md'
pull_request:
branches: [ master ]
paths-ignore:
- 'docs/**'
- 'mkdocs.yml'
- 'README.md'
jobs:
build:
name: Build
runs-on: windows-2022
steps:
- name: Set up Go 1.17
uses: actions/setup-go@v1
with:
go-version: 1.17
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v2
- name: Build
run: make build
- name: All tests
run: go test -coverprofile coverage.out -covermode=atomic ./...
- name: Upload unit coverage to Codecov
uses: codecov/codecov-action@v2
with:
files: coverage.out
flags: unit-windows

2
.github/workflows/ci-windows-build-msi.yml vendored
View file

@ -1,4 +1,4 @@
name: build-msi
name: build-msi (windows)
on:
pull_request:

7
.github/workflows/ci_golangci-lint-windows.yml vendored
View file

@ -1,4 +1,5 @@
name: golangci-lint-windows
name: golangci-lint (windows)
on:
push:
tags:
@ -16,7 +17,7 @@ on:
- 'README.md'
jobs:
golangci:
name: lint-windows
name: lint
runs-on: windows-2022
steps:
- uses: actions/checkout@v3
@ -28,5 +29,3 @@ jobs:
# Optional: golangci-lint command line arguments.
args: --issues-exit-code=0 --timeout 10m
only-new-issues: true

4
.github/workflows/ci_golangci-lint.yml vendored
View file

@ -1,4 +1,5 @@
name: golangci-lint
name: golangci-lint (linux)
on:
push:
tags:
@ -36,4 +37,3 @@ jobs:
skip-pkg-cache: false
# Optional: if set to true then the action don't cache or restore ~/.cache/go-build.
skip-build-cache: false

1
.github/workflows/ci_release-drafter.yml vendored
View file

@ -18,4 +18,3 @@ jobs:
# config-name: my-config.yml
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

1
.github/workflows/codeql-analysis.yml vendored
View file

@ -1,3 +1,4 @@
# yamllint disable rule:comments
# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#

2
.github/workflows/dispatch_create_branch_hub.yaml vendored
View file

@ -1,6 +1,6 @@
name: Dispatch to hub when creating pre-release
on:
on:
release:
types: prereleased

2
.github/workflows/dispatch_delete_branch_hub.yaml vendored
View file

@ -1,6 +1,6 @@
name: Dispatch to hub when deleting pre-release
on:
on:
release:
types: deleted

49
.github/workflows/go-tests-windows.yml vendored Normal file
View file

@ -0,0 +1,49 @@
name: Go tests (windows)
on:
push:
branches: [master]
paths-ignore:
- 'README.md'
pull_request:
branches: [master]
paths-ignore:
- 'README.md'
env:
RICHGO_FORCE_COLOR: 1
jobs:
build:
name: "Build + tests"
runs-on: windows-2022
steps:
- name: "Set up Go 1.17"
uses: actions/setup-go@v3
with:
go-version: 1.17
id: go
- name: Check out CrowdSec repository
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: false
- name: Build
run: |
make build
- name: Run tests
run: |
go install github.com/kyoh86/richgo@v0.3.10
go test -coverprofile coverage.out -covermode=atomic ./... | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
- name: Upload unit coverage to Codecov
uses: codecov/codecov-action@v2
with:
files: coverage.out
flags: unit-windows

42
.github/workflows/go-tests.yml vendored
View file

@ -1,20 +1,28 @@
name: Go tests
name: Go tests (linux w/ localstack)
on:
push:
branches: [master]
paths-ignore:
- 'README.md'
pull_request:
branches: [master]
paths-ignore:
- 'README.md'
# these env variables are for localstack, so we can emulate aws services
env:
RICHGO_FORCE_COLOR: 1
AWS_HOST: localstack
SERVICES: cloudwatch,logs,kinesis
#those are to mimic aws config
# these are to mimic aws config
AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
AWS_REGION: us-east-1
#and to override our endpoint in aws sdk
# and to override our endpoint in aws sdk
AWS_ENDPOINT_FORCE: http://localhost:4566
KINESIS_INITIALIZE_STREAMS: "stream-1-shard:1,stream-2-shards:2"
on:
workflow_call:
jobs:
build:
@ -24,7 +32,7 @@ jobs:
localstack:
image: localstack/localstack:0.13.3
ports:
- 4566:4566 # Localstack exposes all services on same port
- 4566:4566 # Localstack exposes all services on the same port
env:
SERVICES: ${{ env.SERVICES }}
DEBUG: ""
@ -34,8 +42,8 @@ jobs:
DOCKER_HOST: unix:///var/run/docker.sock
HOST_TMP_FOLDER: "/tmp"
KINESIS_INITIALIZE_STREAMS: ${{ env.KINESIS_INITIALIZE_STREAMS }}
HOSTNAME_EXTERNAL: ${{ env.AWS_HOST }} # Required so that resource urls are provided properly
# e.g sqs url will get localhost if we don't set this env to map our service
HOSTNAME_EXTERNAL: ${{ env.AWS_HOST }} # Required so that resource urls are provided properly
# e.g sqs url will get localhost if we don't set this env to map our service
options: >-
--name=localstack
--health-cmd="curl -sS 127.0.0.1:4566 || exit 1"
@ -51,22 +59,26 @@ jobs:
go-version: 1.17
id: go
- name: "Clone CrowdSec"
- name: Check out CrowdSec repository
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: false
- name: Build
run: make build && go get -u github.com/ory/go-acc
- name: "Run tests"
run: |
go run github.com/ory/go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models
make build
- name: Run tests
run: |
go install github.com/ory/go-acc@v0.2.8
go install github.com/kyoh86/richgo@v0.3.10
go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models \
| sed 's/ *coverage:.*of statements in.*//' \
| richgo testfilter
- name: Upload unit coverage to Codecov
uses: codecov/codecov-action@v2
with:
files: coverage.out
flags: unit-linux

4
.github/workflows/release_publish-package.yml vendored
View file

@ -1,7 +1,7 @@
# .github/workflows/build-docker-image.yml
name: build
on:
on:
release:
types: prereleased
@ -86,7 +86,7 @@ jobs:
- name: "Test post-install macines"
run: |
cd scripts/func_tests/
./tests_post-install_3machines.sh
./tests_post-install_3machines.sh
- name: "Test post-install ip management"
run: |
cd scripts/func_tests/

2
.github/workflows/release_publish_docker-image-debian.yml vendored
View file

@ -1,10 +1,12 @@
name: Publish Docker Debian image
on:
release:
types:
- released
- prereleased
workflow_dispatch:
jobs:
push_to_registry:
name: Push Docker debian image to Docker Hub

2
.github/workflows/release_publish_docker-image.yml vendored
View file

@ -1,9 +1,11 @@
name: Publish Docker image
on:
release:
types:
- released
- prereleased
jobs:
push_to_registry:
name: Push Docker image to Docker Hub

43
.yamllint Normal file
View file

@ -0,0 +1,43 @@
---
rules:
braces:
min-spaces-inside: 0
max-spaces-inside: 1
brackets:
min-spaces-inside: 0
max-spaces-inside: 1
colons:
max-spaces-before: 0
max-spaces-after: 1
commas:
max-spaces-before: 0
min-spaces-after: 1
max-spaces-after: 1
comments:
level: warning
require-starting-space: true
min-spaces-from-content: 2
comments-indentation:
level: warning
document-end: disable
document-start: disable
empty-lines:
max: 2
max-start: 0
max-end: 0
hyphens:
max-spaces-after: 1
indentation:
spaces: consistent
indent-sequences: whatever
check-multi-line-strings: false
key-duplicates: enable
line-length:
max: 180
allow-non-breakable-words: true
allow-non-breakable-inline-mappings: false
new-line-at-end-of-file: enable
new-lines:
type: unix
trailing-spaces: enable
truthy: disable

4
cmd/crowdsec-cli/Makefile
View file

@ -21,13 +21,13 @@ BIN_PREFIX = $(PREFIX)"/usr/local/bin/"
all: clean build
build: clean
@$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME) -v
@$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
build-bincover: clean
$(GOTEST) . -tags testrunmain -coverpkg=$(go list github.com/crowdsecurity/crowdsec/... | grep -v -e 'pkg/database' -e 'plugins/notifications' -e 'pkg/protobufs' -e 'pkg/cwversions' -e 'pkg/cstest' -e 'pkg/models') -covermode=atomic $(LD_OPTS) -c -o $(BINARY_NAME_COVER)
static: clean
@$(GOBUILD) $(LD_OPTS_STATIC) -o $(BINARY_NAME) -v -a -tags netgo
@$(GOBUILD) $(LD_OPTS_STATIC) -o $(BINARY_NAME) -a -tags netgo
.PHONY: install
install: install-conf install-bin

4
cmd/crowdsec/Makefile
View file

@ -26,13 +26,13 @@ SYSTEMD_PATH_FILE="/etc/systemd/system/crowdsec.service"
all: clean test build
build: clean
$(GOBUILD) $(LD_OPTS) -o $(CROWDSEC_BIN) -v
$(GOBUILD) $(LD_OPTS) -o $(CROWDSEC_BIN)
build-bincover: clean
$(GOTEST) . -tags testrunmain -coverpkg=$(go list github.com/crowdsecurity/crowdsec/... | grep -v -e 'pkg/database' -e 'plugins/notifications' -e 'pkg/protobufs' -e 'pkg/cwversions' -e 'pkg/cstest' -e 'pkg/models') -covermode=atomic $(LD_OPTS) -c -o $(CROWDSEC_BIN_COVER)
static: clean
$(GOBUILD) $(LD_OPTS_STATIC) -o $(CROWDSEC_BIN) -v -a -tags netgo
$(GOBUILD) $(LD_OPTS_STATIC) -o $(CROWDSEC_BIN) -a -tags netgo
test:
$(GOTEST) $(LD_OPTS) -v ./...

2
cmd/crowdsec/run_in_svc.go
View file

@ -36,7 +36,7 @@ func StartRunSvc() error {
}
// Configure logging
if err = types.SetDefaultLoggerConfig(cConfig.Common.LogMedia, cConfig.Common.LogDir, *cConfig.Common.LogLevel,
cConfig.Common.LogMaxSize, cConfig.Common.LogMaxFiles, cConfig.Common.LogMaxAge, cConfig.Common.CompressLogs); err != nil {
cConfig.Common.LogMaxSize, cConfig.Common.LogMaxFiles, cConfig.Common.LogMaxAge, cConfig.Common.CompressLogs, cConfig.Common.ForceColorLogs); err != nil {
log.Fatal(err.Error())
}

2
cmd/crowdsec/run_in_svc_windows.go
View file

@ -76,7 +76,7 @@ func WindowsRun() error {
}
// Configure logging
if err = types.SetDefaultLoggerConfig(cConfig.Common.LogMedia, cConfig.Common.LogDir, *cConfig.Common.LogLevel,
cConfig.Common.LogMaxSize, cConfig.Common.LogMaxFiles, cConfig.Common.LogMaxAge, cConfig.Common.CompressLogs); err != nil {
cConfig.Common.LogMaxSize, cConfig.Common.LogMaxFiles, cConfig.Common.LogMaxAge, cConfig.Common.CompressLogs, cConfig.Common.ForceColorLogs); err != nil {
return err
}

2
cmd/crowdsec/serve.go
View file

@ -60,7 +60,7 @@ func reloadHandler(sig os.Signal, cConfig *csconfig.Config) error {
}
// Configure logging
if err = types.SetDefaultLoggerConfig(cConfig.Common.LogMedia, cConfig.Common.LogDir, *cConfig.Common.LogLevel,
cConfig.Common.LogMaxSize, cConfig.Common.LogMaxFiles, cConfig.Common.LogMaxAge, cConfig.Common.CompressLogs); err != nil {
cConfig.Common.LogMaxSize, cConfig.Common.LogMaxFiles, cConfig.Common.LogMaxAge, cConfig.Common.CompressLogs, cConfig.Common.ForceColorLogs); err != nil {
return err
}

2
cmd/crowdsec/win_service.go
View file

@ -74,7 +74,7 @@ func runService(name string) error {
// Configure logging
if err := types.SetDefaultLoggerConfig(cConfig.Common.LogMedia, cConfig.Common.LogDir, *cConfig.Common.LogLevel,
cConfig.Common.LogMaxSize, cConfig.Common.LogMaxFiles, cConfig.Common.LogMaxAge, cConfig.Common.CompressLogs); err != nil {
cConfig.Common.LogMaxSize, cConfig.Common.LogMaxFiles, cConfig.Common.LogMaxAge, cConfig.Common.CompressLogs, cConfig.Common.ForceColorLogs); err != nil {
return err
}

4
pkg/apiserver/apiserver_test.go
View file

@ -357,7 +357,7 @@ func TestLoggingDebugToFileConfig(t *testing.T) {
cfg.LogLevel = &lvl
// Configure logging
if err := types.SetDefaultLoggerConfig(cfg.LogMedia, cfg.LogDir, *cfg.LogLevel, cfg.LogMaxSize, cfg.LogMaxFiles, cfg.LogMaxAge, cfg.CompressLogs); err != nil {
if err := types.SetDefaultLoggerConfig(cfg.LogMedia, cfg.LogDir, *cfg.LogLevel, cfg.LogMaxSize, cfg.LogMaxFiles, cfg.LogMaxAge, cfg.CompressLogs, false); err != nil {
t.Fatal(err.Error())
}
api, err := NewServer(&cfg)
@ -414,7 +414,7 @@ func TestLoggingErrorToFileConfig(t *testing.T) {
cfg.LogLevel = &lvl
// Configure logging
if err := types.SetDefaultLoggerConfig(cfg.LogMedia, cfg.LogDir, *cfg.LogLevel, cfg.LogMaxSize, cfg.LogMaxFiles, cfg.LogMaxAge, cfg.CompressLogs); err != nil {
if err := types.SetDefaultLoggerConfig(cfg.LogMedia, cfg.LogDir, *cfg.LogLevel, cfg.LogMaxSize, cfg.LogMaxFiles, cfg.LogMaxAge, cfg.CompressLogs, false); err != nil {
t.Fatal(err.Error())
}
api, err := NewServer(&cfg)

21
pkg/csconfig/common.go
View file

@ -10,16 +10,17 @@ import (
/*daemonization/service related stuff*/
type CommonCfg struct {
Daemonize bool
PidDir string `yaml:"pid_dir,omitempty"` // TODO: This is just for backward compat. Remove this later
LogMedia string `yaml:"log_media"`
LogDir string `yaml:"log_dir,omitempty"` //if LogMedia = file
LogLevel *log.Level `yaml:"log_level"`
WorkingDir string `yaml:"working_dir,omitempty"` ///var/run
CompressLogs *bool `yaml:"compress_logs,omitempty"`
LogMaxSize int `yaml:"log_max_size,omitempty"`
LogMaxAge int `yaml:"log_max_age,omitempty"`
LogMaxFiles int `yaml:"log_max_files,omitempty"`
Daemonize bool
PidDir string `yaml:"pid_dir,omitempty"` // TODO: This is just for backward compat. Remove this later
LogMedia string `yaml:"log_media"`
LogDir string `yaml:"log_dir,omitempty"` //if LogMedia = file
LogLevel *log.Level `yaml:"log_level"`
WorkingDir string `yaml:"working_dir,omitempty"` ///var/run
CompressLogs *bool `yaml:"compress_logs,omitempty"`
LogMaxSize int `yaml:"log_max_size,omitempty"`
LogMaxAge int `yaml:"log_max_age,omitempty"`
LogMaxFiles int `yaml:"log_max_files,omitempty"`
ForceColorLogs bool `yaml:"force_color_logs,omitempty"`
}
func (c *Config) LoadCommon() error {

4
pkg/types/utils.go
View file

@ -23,7 +23,7 @@ var logFormatter log.Formatter
var LogOutput *lumberjack.Logger //io.Writer
var logLevel log.Level
func SetDefaultLoggerConfig(cfgMode string, cfgFolder string, cfgLevel log.Level, maxSize int, maxFiles int, maxAge int, compress *bool) error {
func SetDefaultLoggerConfig(cfgMode string, cfgFolder string, cfgLevel log.Level, maxSize int, maxFiles int, maxAge int, compress *bool, forceColors bool) error {
/*Configure logs*/
if cfgMode == "file" {
@ -67,7 +67,7 @@ func SetDefaultLoggerConfig(cfgMode string, cfgFolder string, cfgLevel log.Level
}
logLevel = cfgLevel
log.SetLevel(logLevel)
logFormatter = &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}
logFormatter = &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true, ForceColors: forceColors}
log.SetFormatter(logFormatter)
return nil
}

4
plugins/notifications/dummy/Makefile
View file

@ -17,7 +17,7 @@ clean:
@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)
build: clean
@$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME) -v
@$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
static: clean
$(GOBUILD) $(LD_OPTS_STATIC) -o $(BINARY_NAME) -v -a -tags netgo
$(GOBUILD) $(LD_OPTS_STATIC) -o $(BINARY_NAME) -a -tags netgo

4
plugins/notifications/email/Makefile
View file

@ -17,7 +17,7 @@ clean:
@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)
build: clean
@$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME) -v
@$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
static: clean
$(GOBUILD) $(LD_OPTS_STATIC) -o $(BINARY_NAME) -v -a -tags netgo
$(GOBUILD) $(LD_OPTS_STATIC) -o $(BINARY_NAME) -a -tags netgo

4
plugins/notifications/http/Makefile
View file

@ -17,7 +17,7 @@ clean:
@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)
build: clean
$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME) -v
$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
static: clean
$(GOBUILD) $(LD_OPTS_STATIC) -o $(BINARY_NAME) -v -a -tags netgo
$(GOBUILD) $(LD_OPTS_STATIC) -o $(BINARY_NAME) -a -tags netgo

4
plugins/notifications/slack/Makefile
View file

@ -13,11 +13,11 @@ GOGET=$(GOCMD) get
BINARY_NAME=notification-slack$(EXT)
build: clean
@$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME) -v
@$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
clean:
@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)
static: clean
$(GOBUILD) $(LD_OPTS_STATIC) -o $(BINARY_NAME) -v -a -tags netgo
$(GOBUILD) $(LD_OPTS_STATIC) -o $(BINARY_NAME) -a -tags netgo

4
plugins/notifications/splunk/Makefile
View file

@ -15,10 +15,10 @@ GOGET=$(GOCMD) get
BINARY_NAME=notification-splunk$(EXT)
build: clean
$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME) -v
$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
clean:
@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)
static: clean
$(GOBUILD) $(LD_OPTS_STATIC) -o $(BINARY_NAME) -v -a -tags netgo
$(GOBUILD) $(LD_OPTS_STATIC) -o $(BINARY_NAME) -a -tags netgo

36
tests/bats/01_base.bats
View file

@ -27,7 +27,7 @@ declare stderr
#----------
@test "${FILE} cscli - usage" {
@test "cscli - usage" {
run -0 cscli
assert_output --partial "Usage:"
assert_output --partial "cscli [command]"
@ -41,7 +41,7 @@ declare stderr
refute_output --partial 'unknown command "blahblah" for "cscli"'
}
@test "${FILE} cscli version" {
@test "cscli version" {
run -0 cscli version
assert_output --partial "version:"
assert_output --partial "Codename:"
@ -59,7 +59,7 @@ declare stderr
assert_output --partial "version:"
}
@test "${FILE} cscli help" {
@test "cscli help" {
run -0 cscli help
assert_line "Available Commands:"
assert_line --regexp ".* help .* Help about any command"
@ -70,7 +70,7 @@ declare stderr
assert_line "Available Commands:"
}
@test "${FILE} cscli alerts list: at startup returns at least one entry: community pull" {
@test "cscli alerts list: at startup returns at least one entry: community pull" {
is_db_postgres && skip
# it should have been received while preparing the fixture
run -0 cscli alerts list -o json
@ -88,7 +88,7 @@ declare stderr
# refute_output 0
}
@test "${FILE} cscli capi status" {
@test "cscli capi status" {
run -0 cscli capi status
assert_output --partial "Loaded credentials from"
assert_output --partial "Trying to authenticate with username"
@ -102,7 +102,7 @@ declare stderr
assert_output --partial "Local API is disabled, please run this command on the local API machine: loading online client credentials: failed to read api server credentials configuration file '${ONLINE_API_CREDENTIALS_YAML}': open ${ONLINE_API_CREDENTIALS_YAML}: no such file or directory"
}
@test "${FILE} cscli config show -o human" {
@test "cscli config show -o human" {
run -0 cscli config show -o human
assert_output --partial "Global:"
assert_output --partial "Crowdsec:"
@ -110,7 +110,7 @@ declare stderr
assert_output --partial "Local API Server:"
}
@test "${FILE} cscli config show -o json" {
@test "cscli config show -o json" {
run -0 cscli config show -o json
assert_output --partial '"API":'
assert_output --partial '"Common":'
@ -123,7 +123,7 @@ declare stderr
assert_output --partial '"Prometheus":'
}
@test "${FILE} cscli config show -o raw" {
@test "cscli config show -o raw" {
run -0 cscli config show -o raw
assert_line "api:"
assert_line "common:"
@ -135,12 +135,12 @@ declare stderr
assert_line "prometheus:"
}
@test "${FILE} cscli config show --key" {
@test "cscli config show --key" {
run -0 cscli config show --key Config.API.Server.ListenURI
assert_output "127.0.0.1:8080"
}
@test "${FILE} cscli config backup / restore" {
@test "cscli config backup / restore" {
# test that we need a valid path
# disabled because in CI, the empty string is not passed as a parameter
## run -1 --separate-stderr cscli config backup ""
@ -182,7 +182,7 @@ declare stderr
rm -rf -- "${backupdir:?}"
}
@test "${FILE} cscli lapi status" {
@test "cscli lapi status" {
if is_db_postgres; then sleep 4; fi
run -0 --separate-stderr cscli lapi status
@ -193,7 +193,7 @@ declare stderr
assert_output --partial "You can successfully interact with Local API (LAPI)"
}
@test "${FILE} cscli - missing LAPI credentials file" {
@test "cscli - missing LAPI credentials file" {
LOCAL_API_CREDENTIALS=$(config_yq '.api.client.credentials_path')
rm -f "${LOCAL_API_CREDENTIALS}"
run -1 --separate-stderr cscli lapi status
@ -209,7 +209,7 @@ declare stderr
assert_output --partial "loading api client: while reading yaml file: open ${LOCAL_API_CREDENTIALS}: no such file or directory"
}
@test "${FILE} cscli - empty LAPI credentials file" {
@test "cscli - empty LAPI credentials file" {
LOCAL_API_CREDENTIALS=$(config_yq '.api.client.credentials_path')
truncate -s 0 "${LOCAL_API_CREDENTIALS}"
run -1 --separate-stderr cscli lapi status
@ -225,7 +225,7 @@ declare stderr
assert_output --partial "no credentials or URL found in api client configuration '${LOCAL_API_CREDENTIALS}'"
}
@test "${FILE} cscli - missing LAPI client settings" {
@test "cscli - missing LAPI client settings" {
yq e 'del(.api.client)' -i "${CONFIG_YAML}"
run -1 --separate-stderr cscli lapi status
run -0 echo "${stderr}"
@ -240,7 +240,7 @@ declare stderr
assert_output --partial "loading api client: no API client section in configuration"
}
@test "${FILE} cscli - malformed LAPI url" {
@test "cscli - malformed LAPI url" {
LOCAL_API_CREDENTIALS=$(config_yq '.api.client.credentials_path')
yq e '.url="https://127.0.0.1:-80"' -i "${LOCAL_API_CREDENTIALS}"
@ -260,7 +260,7 @@ declare stderr
assert_output --partial 'invalid port \":-80\" after host'
}
@test "${FILE} cscli metrics" {
@test "cscli metrics" {
run -0 cscli lapi status
run -0 --separate-stderr cscli metrics
assert_output --partial "ROUTE"
@ -270,7 +270,7 @@ declare stderr
assert_output --partial "Local Api Metrics:"
}
@test "${FILE} 'cscli completion' with or without configuration file" {
@test "'cscli completion' with or without configuration file" {
run -0 cscli completion bash
assert_output --partial "# bash completion for cscli"
run -0 cscli completion zsh
@ -285,7 +285,7 @@ declare stderr
assert_output --partial "# bash completion for cscli"
}
@test "${FILE} cscli hub list" {
@test "cscli hub list" {
# we check for the presence of some objects. There may be others when we
# use $PACKAGE_TESTING, so the order is not important.

8
tests/bats/01_crowdsec.bats
View file

@ -25,7 +25,7 @@ declare stderr
#----------
@test "${FILE} crowdsec (usage)" {
@test "crowdsec (usage)" {
run -0 --separate-stderr timeout 2s "${CROWDSEC}" -h
run -0 echo "${stderr}"
assert_line --regexp "Usage of .*:"
@ -35,21 +35,21 @@ declare stderr
assert_line --regexp "Usage of .*:"
}
@test "${FILE} crowdsec (unknown flag)" {
@test "crowdsec (unknown flag)" {
run -2 --separate-stderr timeout 2s "${CROWDSEC}" --foobar
run -0 echo "${stderr}"
assert_line "flag provided but not defined: -foobar"
assert_line --regexp "Usage of .*"
}
@test "${FILE} crowdsec (unknown argument)" {
@test "crowdsec (unknown argument)" {
run -2 --separate-stderr timeout 2s "${CROWDSEC}" trololo
run -0 echo "${stderr}"
assert_line "argument provided but not defined: trololo"
assert_line --regexp "Usage of .*"
}
@test "${FILE} crowdsec (no api and no agent)" {
@test "crowdsec (no api and no agent)" {
run -1 --separate-stderr timeout 2s "${CROWDSEC}" -no-api -no-cs
run -0 echo "${stderr}"
assert_line --partial "You must run at least the API Server or crowdsec"

18
tests/bats/02_nolapi.bats
View file

@ -25,18 +25,16 @@ declare stderr
#----------
@test "$FILE test without -no-api flag" {
@test "test without -no-api flag" {
run -124 --separate-stderr timeout 2s "${CROWDSEC}"
# from `man timeout`: If the command times out, and --preserve-status is not set, then exit with status 124.
}
@test "$FILE crowdsec should not run without LAPI (-no-api flag)" {
skip
@test "crowdsec should not run without LAPI (-no-api flag)" {
run -1 --separate-stderr timeout 2s "${CROWDSEC}" -no-api
}
@test "$FILE crowdsec should not run without LAPI (no api.server in configuration file)" {
skip
@test "crowdsec should not run without LAPI (no api.server in configuration file)" {
yq e 'del(.api.server)' -i "${CONFIG_YAML}"
run -1 --separate-stderr timeout 2s "${CROWDSEC}"
@ -44,7 +42,7 @@ declare stderr
assert_output --partial "crowdsec local API is disabled"
}
@test "$FILE capi status shouldn't be ok without api.server" {
@test "capi status shouldn't be ok without api.server" {
yq e 'del(.api.server)' -i "${CONFIG_YAML}"
run -1 --separate-stderr cscli capi status
@ -53,7 +51,7 @@ declare stderr
assert_output --partial "There is no configuration on 'api.server:'"
}
@test "$FILE cscli config show -o human" {
@test "cscli config show -o human" {
yq e 'del(.api.server)' -i "${CONFIG_YAML}"
run -0 cscli config show -o human
assert_output --partial "Global:"
@ -62,7 +60,7 @@ declare stderr
refute_output --partial "Local API Server:"
}
@test "$FILE cscli config backup" {
@test "cscli config backup" {
yq e 'del(.api.server)' -i "${CONFIG_YAML}"
backupdir=$(TMPDIR="${BATS_TEST_TMPDIR}" mktemp -u)
run -0 cscli config backup "${backupdir}"
@ -75,7 +73,7 @@ declare stderr
assert_output --partial "file exists"
}
@test "$FILE lapi status shouldn't be ok without api.server" {
@test "lapi status shouldn't be ok without api.server" {
yq e 'del(.api.server)' -i "${CONFIG_YAML}"
./instance-crowdsec start || true
run -1 --separate-stderr cscli machines list
@ -83,7 +81,7 @@ declare stderr
assert_output --partial "Local API is disabled, please run this command on the local API machine"
}
@test "$FILE cscli metrics" {
@test "cscli metrics" {
skip 'need to trigger metrics with a live parse'
yq e 'del(.api.server)' -i "${CONFIG_YAML}"
./instance-crowdsec start

4
tests/generate-hub-tests
View file

@ -16,7 +16,7 @@ cscli() {
hubdir="${LOCAL_DIR}/hub-tests"
git clone --depth 1 https://github.com/crowdsecurity/hub.git "${hubdir}" >/dev/null 2>&1 || (cd "${hubdir}"; git pull)
HUBTESTS_BATS="${TEST_DIR}/dyn-bats/99_hub.bats"
HUBTESTS_BATS="${TEST_DIR}/dyn-bats/hub.bats"
cat << EOT > "${HUBTESTS_BATS}"
set -u
@ -40,7 +40,7 @@ echo "Generating hub tests..."
for testname in $("${CSCLI}" --crowdsec "${CROWDSEC}" --cscli "${CSCLI}" hubtest --hub "${hubdir}" list -o json | grep -v NAME | grep -v -- '-------' | awk '{print $1}'); do
cat << EOT >> "${HUBTESTS_BATS}"
@test "\$FILE ${testname}" {
@test "${testname}" {
run "\${CSCLI}" --crowdsec "\${CROWDSEC}" --cscli "\${CSCLI}" --hub "${hubdir}" hubtest run "${testname}" --clean
# in case of error, need to see what went wrong
echo "\$output"

3
tests/lib/config/config-global
View file

@ -81,7 +81,7 @@ load_init_data() {
dump_backend="$(cat "${LOCAL_INIT_DIR}/.backend")"
if [[ "${DB_BACKEND}" != "${dump_backend}" ]]; then
die "Can't run with backend '${DB_BACKEND}' because the test data was build with '${dump_backend}'"
die "Can't run with backend '${DB_BACKEND}' because the test data was built with '${dump_backend}'"
fi
remove_init_data
@ -97,7 +97,6 @@ load_init_data() {
[[ $# -lt 1 ]] && about
case "$1" in
make)
make_init_data

4
tests/lib/config/config-local
View file

@ -60,6 +60,7 @@ config_generate() {
.common.daemonize=true |
del(.common.pid_dir) |
.common.log_level="info" |
.common.force_color_logs=true |
.common.log_dir=strenv(LOG_DIR) |
.config_paths.config_dir=strenv(CONFIG_DIR) |
.config_paths.data_dir=strenv(DATA_DIR) |
@ -132,7 +133,7 @@ load_init_data() {
dump_backend="$(cat "${LOCAL_INIT_DIR}/.backend")"
if [[ "${DB_BACKEND}" != "${dump_backend}" ]]; then
die "Can't run with backend '${DB_BACKEND}' because the test data was build with '${dump_backend}'"
die "Can't run with backend '${DB_BACKEND}' because the test data was built with '${dump_backend}'"
fi
remove_init_data
@ -146,7 +147,6 @@ load_init_data() {
[[ $# -lt 1 ]] && about
case "$1" in
make)
./assert-crowdsec-not-running

10
tests/lib/setup_file.sh
View file

@ -24,8 +24,14 @@ cd "${TEST_DIR}"
# complain if there's a crowdsec running system-wide or leftover from a previous test
./assert-crowdsec-not-running
# we can use the filename in test descriptions
FILE="$(basename "${BATS_TEST_FILENAME}" .bats):"
# we can prepend the filename to the test descriptions (useful to feed a TAP consumer)
if [[ "${PREFIX_TEST_NAMES_WITH_FILE:-false}" == "true" ]]; then
BATS_TEST_NAME_PREFIX="$(basename "${BATS_TEST_FILENAME}" .bats): "
export BATS_TEST_NAME_PREFIX
fi
# before bats 1.7, we did that by hand
FILE=
export FILE
# the variables exported here can be seen in other setup/teardown/test functions

5
tests/run-tests
View file

@ -34,15 +34,16 @@ if [[ $# -ge 1 ]]; then
echo "test files: $*"
"${TEST_DIR}/lib/bats-core/bin/bats" \
--jobs 1 \
--print-output-on-failure \
--timing \
--print-output-on-failure \
"$@"
else
echo "test files: ${TEST_DIR}/bats ${TEST_DIR}/dyn-bats"
"${TEST_DIR}/lib/bats-core/bin/bats" \
--jobs 1 \
--timing \
--print-output-on-failure \
-T "${TEST_DIR}/bats" "${TEST_DIR}/dyn-bats"
"${TEST_DIR}/bats" "${TEST_DIR}/dyn-bats"
fi
if [[ -n "${TEST_COVERAGE}" ]]; then