diff --git a/.github/workflows/bats-hub.yml b/.github/workflows/bats-hub.yml index 026e8feaa..c62229cbe 100644 --- a/.github/workflows/bats-hub.yml +++ b/.github/workflows/bats-hub.yml @@ -15,7 +15,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.4"] + go-version: ["1.20.5"] name: "Build + tests" runs-on: ubuntu-latest @@ -27,40 +27,26 @@ jobs: sudo chmod +w /etc/machine-id echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go-version }} - - name: "Check out CrowdSec repository" uses: actions/checkout@v3 with: fetch-depth: 0 submodules: true - - name: Cache Go modules - uses: actions/cache@v3 + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v4 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - %LocalAppData%\go-build - key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.go-version }}-go- + go-version: ${{ matrix.go-version }} + cache-dependency-path: "**/go.sum" - name: "Install bats dependencies" env: GOBIN: /usr/local/bin run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd - go install github.com/mikefarah/yq/v4@latest - go install github.com/cloudflare/cfssl/cmd/cfssl@master - go install github.com/cloudflare/cfssl/cmd/cfssljson@master + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev - name: "Build crowdsec and fixture" - run: make bats-clean bats-build bats-fixture + run: make bats-clean bats-build bats-fixture BUILD_STATIC=1 - name: "Run hub tests" run: make bats-test-hub diff --git a/.github/workflows/bats-mysql.yml b/.github/workflows/bats-mysql.yml index 3c8825a47..529c3f521 100644 --- a/.github/workflows/bats-mysql.yml +++ b/.github/workflows/bats-mysql.yml @@ -14,7 +14,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.4"] + go-version: ["1.20.5"] name: "Build + tests" runs-on: ubuntu-latest @@ -34,41 +34,27 @@ jobs: sudo chmod +w /etc/machine-id echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go-version }} - - name: "Check out CrowdSec repository" uses: actions/checkout@v3 with: fetch-depth: 0 submodules: true - - name: Cache Go modules - uses: actions/cache@v3 + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v4 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - %LocalAppData%\go-build - key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.go-version }}-go- + go-version: ${{ matrix.go-version }} + cache-dependency-path: "**/go.sum" - name: "Install bats dependencies" env: GOBIN: /usr/local/bin run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd - go install github.com/mikefarah/yq/v4@latest - go install github.com/cloudflare/cfssl/cmd/cfssl@master - go install github.com/cloudflare/cfssl/cmd/cfssljson@master + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev - name: "Build crowdsec and fixture" run: | - make clean bats-build bats-fixture + make clean bats-build bats-fixture BUILD_STATIC=1 env: DB_BACKEND: mysql MYSQL_HOST: 127.0.0.1 diff --git a/.github/workflows/bats-postgres.yml b/.github/workflows/bats-postgres.yml index de99a8e22..91e7ac103 100644 --- a/.github/workflows/bats-postgres.yml +++ b/.github/workflows/bats-postgres.yml @@ -10,7 +10,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.4"] + go-version: ["1.20.5"] name: "Build + tests" runs-on: ubuntu-latest @@ -35,41 +35,27 @@ jobs: sudo chmod +w /etc/machine-id echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go-version }} - - name: "Check out CrowdSec repository" uses: actions/checkout@v3 with: fetch-depth: 0 submodules: true - - name: Cache Go modules - uses: actions/cache@v3 + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v4 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - %LocalAppData%\go-build - key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.go-version }}-go- + go-version: ${{ matrix.go-version }} + cache-dependency-path: "**/go.sum" - name: "Install bats dependencies" env: GOBIN: /usr/local/bin run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd - go install github.com/mikefarah/yq/v4@latest - go install github.com/cloudflare/cfssl/cmd/cfssl@master - go install github.com/cloudflare/cfssl/cmd/cfssljson@master + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev - name: "Build crowdsec and fixture (DB_BACKEND: pgx)" run: | - make clean bats-build bats-fixture + make clean bats-build bats-fixture BUILD_STATIC=1 env: DB_BACKEND: pgx PGHOST: 127.0.0.1 diff --git a/.github/workflows/bats-sqlite-coverage.yml b/.github/workflows/bats-sqlite-coverage.yml index 7b2e763b3..17b2aac09 100644 --- a/.github/workflows/bats-sqlite-coverage.yml +++ b/.github/workflows/bats-sqlite-coverage.yml @@ -11,7 +11,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.4"] + go-version: ["1.20.5"] name: "Build + tests" runs-on: ubuntu-latest @@ -24,41 +24,27 @@ jobs: sudo chmod +w /etc/machine-id echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go-version }} - - name: "Check out CrowdSec repository" uses: actions/checkout@v3 with: fetch-depth: 0 submodules: true - - name: Cache Go modules - uses: actions/cache@v3 + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v4 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - %LocalAppData%\go-build - key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.go-version }}-go- + go-version: ${{ matrix.go-version }} + cache-dependency-path: "**/go.sum" - name: "Install bats dependencies" env: GOBIN: /usr/local/bin run: | - sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd - go install github.com/mikefarah/yq/v4@latest - go install github.com/cloudflare/cfssl/cmd/cfssl@master - go install github.com/cloudflare/cfssl/cmd/cfssljson@master + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev - name: "Build crowdsec and fixture" run: | - make clean bats-build bats-fixture + make clean bats-build bats-fixture BUILD_STATIC=1 - name: "Run tests" run: make bats-test diff --git a/.github/workflows/cache-cleanup.yaml b/.github/workflows/cache-cleanup.yaml new file mode 100644 index 000000000..d19365024 --- /dev/null +++ b/.github/workflows/cache-cleanup.yaml @@ -0,0 +1,35 @@ +# https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#managing-caches + +name: cleanup caches by a branch +on: + pull_request: + types: + - closed + +jobs: + cleanup: + runs-on: ubuntu-latest + steps: + - name: Check out code + uses: actions/checkout@v3 + + - name: Cleanup + run: | + gh extension install actions/gh-actions-cache + + REPO=${{ github.repository }} + BRANCH="refs/pull/${{ github.event.pull_request.number }}/merge" + + echo "Fetching list of cache key" + cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH | cut -f 1 ) + + ## Setting this to not fail the workflow while deleting cache keys. + set +e + echo "Deleting caches..." + for cacheKey in $cacheKeysForPR + do + gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm + done + echo "Done" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/ci-windows-build-msi.yml b/.github/workflows/ci-windows-build-msi.yml index b9d9722f3..3b66b0c1f 100644 --- a/.github/workflows/ci-windows-build-msi.yml +++ b/.github/workflows/ci-windows-build-msi.yml @@ -23,38 +23,27 @@ jobs: build: strategy: matrix: - go-version: ["1.20.4"] + go-version: ["1.20.5"] name: Build runs-on: windows-2019 steps: - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go-version }} - - name: Check out code into the Go module directory uses: actions/checkout@v3 with: fetch-depth: 0 submodules: false - - name: Cache Go modules - uses: actions/cache@v3 + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v4 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - %LocalAppData%\go-build - key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.go-version }}-go- + go-version: ${{ matrix.go-version }} + cache-dependency-path: "**/go.sum" - name: Build - run: make windows_installer + run: make windows_installer BUILD_RE2_WASM=1 - name: Upload MSI uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml index 6476d9f0a..913c47662 100644 --- a/.github/workflows/docker-tests.yml +++ b/.github/workflows/docker-tests.yml @@ -30,18 +30,6 @@ jobs: with: config: .github/buildkit.toml - # - name: "Build flavor: full" - # uses: docker/build-push-action@v4 - # with: - # context: . - # file: ./Dockerfile - # tags: crowdsecurity/crowdsec:test - # target: full - # platforms: linux/amd64 - # load: true - # cache-from: type=gha - # cache-to: type=gha,mode=min - - name: "Build flavor: slim" uses: docker/build-push-action@v4 with: diff --git a/.github/workflows/go-tests-windows.yml b/.github/workflows/go-tests-windows.yml index b8e81bbe0..500fc58ce 100644 --- a/.github/workflows/go-tests-windows.yml +++ b/.github/workflows/go-tests-windows.yml @@ -22,39 +22,28 @@ jobs: build: strategy: matrix: - go-version: ["1.20.4"] + go-version: ["1.20.5"] name: "Build + tests" runs-on: windows-2022 steps: - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go-version }} - - name: Check out CrowdSec repository uses: actions/checkout@v3 with: fetch-depth: 0 submodules: false - - name: Cache Go modules - uses: actions/cache@v3 + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v4 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - %LocalAppData%\go-build - key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.go-version }}-go- + go-version: ${{ matrix.go-version }} + cache-dependency-path: "**/go.sum" - name: Build run: | - make build + make build BUILD_RE2_WASM=1 - name: Run tests run: | diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 9b4adb4ce..079f6c827 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -34,7 +34,7 @@ jobs: build: strategy: matrix: - go-version: ["1.20.4"] + go-version: ["1.20.5"] name: "Build + tests" runs-on: ubuntu-latest @@ -110,35 +110,30 @@ jobs: steps: - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go-version }} - - name: Check out CrowdSec repository uses: actions/checkout@v3 with: fetch-depth: 0 submodules: false - - name: Cache Go modules - uses: actions/cache@v3 + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v4 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - %LocalAppData%\go-build - key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.go-version }}-go- + go-version: ${{ matrix.go-version }} + cache-dependency-path: "**/go.sum" - - name: Build and run tests + - name: Build and run tests, static run: | + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev go install github.com/ory/go-acc@v0.2.8 go install github.com/kyoh86/richgo@v0.3.10 set -o pipefail - make build + make build BUILD_STATIC=1 + make go-acc | richgo testfilter + + - name: Run tests again, dynamic + run: | + make clean build make go-acc | richgo testfilter - name: Upload unit coverage to Codecov diff --git a/.github/workflows/release_publish-package.yml b/.github/workflows/release_publish-package.yml index 600355739..54419cc6e 100644 --- a/.github/workflows/release_publish-package.yml +++ b/.github/workflows/release_publish-package.yml @@ -14,41 +14,32 @@ jobs: build: strategy: matrix: - go-version: ["1.20.4"] + go-version: ["1.20.5"] name: Build and upload binary package runs-on: ubuntu-latest steps: - - name: "Set up Go ${{ matrix.go-version }}" - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go-version }} - - name: Check out code into the Go module directory uses: actions/checkout@v3 with: fetch-depth: 0 submodules: false - - name: Cache Go modules - uses: actions/cache@v3 + - name: "Set up Go ${{ matrix.go-version }}" + uses: actions/setup-go@v4 with: - path: | - ~/go/pkg/mod - ~/.cache/go-build - ~/Library/Caches/go-build - %LocalAppData%\go-build - key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }} - restore-keys: | - ${{ runner.os }}-${{ matrix.go-version }}-go- + go-version: ${{ matrix.go-version }} + cache-dependency-path: "**/go.sum" - name: Build the binaries - run: make release + run: | + sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev + make vendor release BUILD_STATIC=1 - name: Upload to release env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | tag_name="${GITHUB_REF##*/}" - hub release edit -a crowdsec-release.tgz -m "" "$tag_name" + hub release edit -a crowdsec-release.tgz -a vendor.tgz -m "" "$tag_name" diff --git a/.gitignore b/.gitignore index 4c5cb0a1f..8fe1778ba 100644 --- a/.gitignore +++ b/.gitignore @@ -15,6 +15,9 @@ *.test *.cover +# Test dependencies +test/tools/* + # VMs used for dev/test .vagrant @@ -30,8 +33,10 @@ test/coverage/* *.swp *.swo -# Dependency directories (remove the comment below to include it) -# vendor/ +# Dependencies are not vendored by default, but a tarball is created by "make vendor" +# and provided in the release. Used by freebsd, gentoo, etc. +vendor/ +vendor.tgz # crowdsec binaries cmd/crowdsec-cli/cscli diff --git a/.golangci.yml b/.golangci.yml index 79900ae7d..faa67c4bb 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -65,7 +65,6 @@ linters: # - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers # - bidichk # Checks for dangerous unicode character sequences # - decorder # check declaration order and count of types, constants, variables and functions - # - depguard # Go linter that checks if package imports are in a list of acceptable packages # - dupword # checks for duplicate words in the source code # - durationcheck # check for two durations multiplied together # - errcheck # Errcheck is a program for checking for unchecked errors in go programs. These unchecked errors can be critical bugs in some cases @@ -125,6 +124,7 @@ linters: - thelper # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers - wastedassign # wastedassign finds wasted assignment statements. - wrapcheck # Checks that errors returned from external packages are wrapped + - depguard # Go linter that checks if package imports are in a list of acceptable packages # # Recommended? (requires some work) diff --git a/Dockerfile b/Dockerfile index da1c3ab06..2073d3c5a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,33 +1,31 @@ # vim: set ft=dockerfile: -ARG GOVERSION=1.20.4 +ARG GOVERSION=1.20.5 FROM golang:${GOVERSION}-alpine AS build WORKDIR /go/src/crowdsec -COPY . . - -# Alpine does not ship a static version of re2, we can build it ourselves -# Later versions require 'abseil', which is likewise not available in its static form +# We like to choose the release of re2 to use, and Alpine does not ship a static version anyway. ENV RE2_VERSION=2023-03-01 # wizard.sh requires GNU coreutils -RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils icu-static re2-dev pkgconfig && \ +RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils pkgconfig && \ wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \ tar -xzf ${RE2_VERSION}.tar.gz && \ cd re2-${RE2_VERSION} && \ - make && \ make install && \ echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \ - cd - && \ - make clean release DOCKER_BUILD=1 && \ + go install github.com/mikefarah/yq/v4@v4.34.1 + +COPY . . + +RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \ cd crowdsec-v* && \ ./wizard.sh --docker-mode && \ cd - >/dev/null && \ cscli hub update && \ cscli collections install crowdsecurity/linux && \ - cscli parsers install crowdsecurity/whitelists && \ - go install github.com/mikefarah/yq/v4@v4.31.2 + cscli parsers install crowdsecurity/whitelists # In case we need to remove agents here.. # cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete diff --git a/Dockerfile.debian b/Dockerfile.debian index 10b06befd..61778cd9e 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -1,32 +1,41 @@ # vim: set ft=dockerfile: -ARG GOVERSION=1.20.4 +ARG GOVERSION=1.20.5 -FROM golang:${GOVERSION}-bullseye AS build +FROM golang:${GOVERSION}-bookworm AS build WORKDIR /go/src/crowdsec -COPY . . - ENV DEBIAN_FRONTEND=noninteractive ENV DEBCONF_NOWARNINGS="yes" +# We like to choose the release of re2 to use, the debian version is usually older. +ENV RE2_VERSION=2023-03-01 + # wizard.sh requires GNU coreutils RUN apt-get update && \ - apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata libre2-dev && \ + apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \ + wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \ + tar -xzf ${RE2_VERSION}.tar.gz && \ + cd re2-${RE2_VERSION} && \ + make && \ + make install && \ echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \ - make clean release DOCKER_BUILD=1 && \ + go install github.com/mikefarah/yq/v4@v4.34.1 + +COPY . . + +RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \ cd crowdsec-v* && \ ./wizard.sh --docker-mode && \ cd - >/dev/null && \ cscli hub update && \ cscli collections install crowdsecurity/linux && \ - cscli parsers install crowdsecurity/whitelists && \ - go install github.com/mikefarah/yq/v4@v4.31.2 + cscli parsers install crowdsecurity/whitelists # In case we need to remove agents here.. # cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete -FROM debian:bullseye-slim as slim +FROM debian:bookworm-slim as slim ENV DEBIAN_FRONTEND=noninteractive ENV DEBCONF_NOWARNINGS="yes" @@ -44,9 +53,6 @@ RUN apt-get update && \ mkdir -p /staging/var/lib/crowdsec && \ mkdir -p /var/lib/crowdsec/data -RUN echo "deb http://deb.debian.org/debian bullseye-backports main" >> /etc/apt/sources.list \ - && apt-get update && apt-get install -t bullseye-backports -y libsystemd0 - COPY --from=build /go/bin/yq /usr/local/bin/yq COPY --from=build /etc/crowdsec /staging/etc/crowdsec COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec diff --git a/Makefile b/Makefile index d6f1b95f2..0fb36261f 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,36 @@ include mk/platform.mk +include mk/gmsl +# By default, this build requires the C++ re2 library to be installed. +# +# Debian/Ubuntu: apt install libre2-dev +# Fedora/CentOS: dnf install re2-devel +# FreeBSD: pkg install re2 +# Alpine: apk add re2-dev +# Windows: choco install re2 +# MacOS: brew install re2 + +# To build without re2, run "make BUILD_RE2_WASM=1" +# The WASM version is slower and introduces a short delay when starting a process +# (including cscli) so it is not recommended for production use. +BUILD_RE2_WASM ?= 0 + +# To build static binaries, run "make BUILD_STATIC=1". +# On some platforms, this requires additional packages +# (e.g. glibc-static and libstdc++-static on fedora, centos.. which are on the powertools/crb repository). +# If the static build fails at the link stage, it might be because the static library is not provided +# for your distribution (look for libre2.a). See the Dockerfile for an example of how to build it. +BUILD_STATIC ?= 0 + +# List of plugins to build +PLUGINS ?= $(patsubst ./plugins/notifications/%,%,$(wildcard ./plugins/notifications/*)) + +# Can be overriden, if you can deal with the consequences BUILD_REQUIRE_GO_MAJOR ?= 1 BUILD_REQUIRE_GO_MINOR ?= 20 +#-------------------------------------- + GOCMD = go GOTEST = $(GOCMD) test @@ -10,8 +38,6 @@ BUILD_CODENAME ?= alphaga CROWDSEC_FOLDER = ./cmd/crowdsec CSCLI_FOLDER = ./cmd/crowdsec-cli/ - -PLUGINS ?= $(patsubst ./plugins/notifications/%,%,$(wildcard ./plugins/notifications/*)) PLUGINS_DIR = ./plugins/notifications CROWDSEC_BIN = crowdsec$(EXT) @@ -22,8 +48,14 @@ RELDIR = crowdsec-$(BUILD_VERSION) GO_MODULE_NAME = github.com/crowdsecurity/crowdsec -# see if we have libre2-dev installed for C++ optimizations -RE2_CHECK := $(shell pkg-config --libs re2 2>/dev/null) +# Check if a given value is considered truthy and returns "0" or "1". +# A truthy value is one of the following: "1", "yes", or "true", case-insensitive. +# +# Usage: +# ifeq ($(call bool,$(FOO)),1) +# $(info Let's foo) +# endif +bool = $(if $(filter $(call lc, $1),1 yes true),1,0) #-------------------------------------- # @@ -46,13 +78,30 @@ endif GO_TAGS := netgo,osusergo,sqlite_omit_load_extension -ifneq (,$(RE2_CHECK)) +ifeq ($(call bool,$(BUILD_RE2_WASM)),0) +ifeq ($(PKG_CONFIG),) + $(error "pkg-config is not available. Please install pkg-config.") +endif + +ifeq ($(RE2_CHECK),) +# we could detect the platform and suggest the command to install +RE2_FAIL := "libre2-dev is not installed, please install it or set BUILD_RE2_WASM=1 to use the WebAssembly version" +else # += adds a space that we don't want GO_TAGS := $(GO_TAGS),re2_cgo LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.Libre2=C++' endif +endif -export LD_OPTS=-ldflags "-s -w -extldflags '-static' $(LD_OPTS_VARS)" \ +ifeq ($(call bool,$(BUILD_STATIC)),1) +BUILD_TYPE = static +EXTLDFLAGS := -extldflags '-static' +else +BUILD_TYPE = dynamic +EXTLDFLAGS := +endif + +export LD_OPTS=-ldflags "-s -w $(EXTLDFLAGS) $(LD_OPTS_VARS)" \ -trimpath -tags $(GO_TAGS) ifneq (,$(TEST_COVERAGE)) @@ -64,12 +113,15 @@ endif .PHONY: build build: pre-build goversion crowdsec cscli plugins +# Sanity checks and build information .PHONY: pre-build pre-build: -ifdef BUILD_STATIC - $(warning WARNING: The BUILD_STATIC variable is deprecated and has no effect. Builds are static by default since v1.5.0.) + $(info Building $(BUILD_VERSION) ($(BUILD_TAG)) $(BUILD_TYPE) for $(GOOS)/$(GOARCH)) + +ifneq (,$(RE2_FAIL)) + $(error $(RE2_FAIL)) endif - $(info Building $(BUILD_VERSION) ($(BUILD_TAG)) for $(GOOS)/$(GOARCH)) + ifneq (,$(RE2_CHECK)) $(info Using C++ regexp library) else @@ -113,6 +165,7 @@ testclean: bats-clean @$(RM) pkg/cwhub/install $(WIN_IGNORE_ERR) @$(RM) pkg/types/example.txt $(WIN_IGNORE_ERR) +# for the tests with localstack export AWS_ENDPOINT_FORCE=http://localhost:4566 export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY @@ -120,15 +173,18 @@ export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY testenv: @echo 'NOTE: You need Docker, docker-compose and run "make localstack" in a separate shell ("make localstack-stop" to terminate it)' +# run the tests with localstack .PHONY: test test: testenv goversion $(GOTEST) $(LD_OPTS) ./... +# run the tests with localstack and coverage .PHONY: go-acc go-acc: testenv goversion go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS) | \ sed 's/ *coverage:.*of statements in.*//' +# mock AWS services .PHONY: localstack localstack: docker-compose -f test/localstack/docker-compose.yml up @@ -137,13 +193,27 @@ localstack: localstack-stop: docker-compose -f test/localstack/docker-compose.yml down +# list of plugins that contain go.mod +PLUGIN_VENDOR = $(foreach plugin,$(PLUGINS),$(shell if [ -f $(PLUGINS_DIR)/$(plugin)/go.mod ]; then echo $(PLUGINS_DIR)/$(plugin); fi)) + +# build vendor.tgz to be distributed with the release .PHONY: vendor vendor: - @echo "Vendoring dependencies" - @$(GOCMD) mod vendor - @$(foreach plugin,$(PLUGINS), \ - $(MAKE) -C $(PLUGINS_DIR)/$(plugin) vendor $(MAKE_FLAGS); \ + $(foreach plugin_dir,$(PLUGIN_VENDOR), \ + cd $(plugin_dir) >/dev/null && \ + $(GOCMD) mod vendor && \ + cd - >/dev/null; \ ) + $(GOCMD) mod vendor + tar -czf vendor.tgz vendor $(foreach plugin_dir,$(PLUGIN_VENDOR),$(plugin_dir)/vendor) + +# remove vendor directories and vendor.tgz +.PHONY: vendor-remove +vendor-remove: + $(foreach plugin_dir,$(PLUGIN_VENDOR), \ + $(RM) $(plugin_dir)/vendor; \ + ) + $(RM) vendor vendor.tgz .PHONY: package package: @@ -174,13 +244,16 @@ else @if (Test-Path -Path $(RELDIR)) { echo "$(RELDIR) already exists, abort" ; exit 1 ; } endif +# build a release tarball .PHONY: release release: check_release build package +# build the windows installer .PHONY: windows_installer windows_installer: build @.\make_installer.ps1 -version $(BUILD_VERSION) +# build the chocolatey package .PHONY: chocolatey chocolatey: windows_installer @.\make_chocolatey.ps1 -version $(BUILD_VERSION) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c529ee2fa..b1564b375 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -27,7 +27,7 @@ stages: - task: GoTool@0 displayName: "Install Go 1.20" inputs: - version: '1.20.4' + version: '1.20.5' - pwsh: | choco install -y make @@ -38,7 +38,7 @@ stages: pwsh: true #we are not calling make windows_installer because we want to sign the binaries before they are added to the MSI script: | - make build + make build BUILD_RE2_WASM=1 - task: AzureKeyVault@2 inputs: azureSubscription: 'Azure subscription 1(8a93ab40-7e99-445e-ad47-0f6a3e2ef546)' diff --git a/cmd/crowdsec-cli/alerts.go b/cmd/crowdsec-cli/alerts.go index 25cb26515..6abe3db5a 100644 --- a/cmd/crowdsec-cli/alerts.go +++ b/cmd/crowdsec-cli/alerts.go @@ -15,7 +15,6 @@ import ( "github.com/fatih/color" "github.com/go-openapi/strfmt" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/yaml.v2" @@ -155,7 +154,6 @@ var alertTemplate = ` ` - func DisplayOneAlert(alert *models.Alert, withDetail bool) error { if csConfig.Cscli.Output == "human" { tmpl, err := template.New("alert").Parse(alertTemplate) @@ -211,11 +209,11 @@ func NewAlertsCmd() *cobra.Command { PersistentPreRunE: func(cmd *cobra.Command, args []string) error { var err error if err := csConfig.LoadAPIClient(); err != nil { - return errors.Wrap(err, "loading api client") + return fmt.Errorf("loading api client: %w", err) } apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL) if err != nil { - return errors.Wrapf(err, "parsing api url %s", apiURL) + return fmt.Errorf("parsing api url %s: %w", apiURL, err) } Client, err = apiclient.NewClient(&apiclient.Config{ MachineID: csConfig.API.Client.Credentials.Login, @@ -226,7 +224,7 @@ func NewAlertsCmd() *cobra.Command { }) if err != nil { - return errors.Wrap(err, "new api client") + return fmt.Errorf("new api client: %w", err) } return nil }, diff --git a/cmd/crowdsec-cli/capi.go b/cmd/crowdsec-cli/capi.go index e67d33ce4..af6e9c2e8 100644 --- a/cmd/crowdsec-cli/capi.go +++ b/cmd/crowdsec-cli/capi.go @@ -6,6 +6,11 @@ import ( "net/url" "os" + "github.com/go-openapi/strfmt" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + "gopkg.in/yaml.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/apiclient" @@ -14,11 +19,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/fflag" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "github.com/spf13/cobra" - "gopkg.in/yaml.v2" ) const CAPIBaseURL string = "https://api.crowdsec.net/" @@ -31,8 +31,11 @@ func NewCapiCmd() *cobra.Command { Args: cobra.MinimumNArgs(1), DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI { - return errors.Wrap(err, "Local API is disabled, please run this command on the local API machine") + if err := csConfig.LoadAPIServer(); err != nil { + return fmt.Errorf("local API is disabled, please run this command on the local API machine: %w", err) + } + if csConfig.DisableAPI { + return nil } if csConfig.API.Server.OnlineClient == nil { log.Fatalf("no configuration for Central API in '%s'", *csConfig.FilePath) @@ -133,7 +136,7 @@ func NewCapiStatusCmd() *cobra.Command { Run: func(cmd *cobra.Command, args []string) { var err error if csConfig.API.Server == nil { - log.Fatalln("There is no configuration on 'api.server:'") + log.Fatal("There is no configuration on 'api.server:'") } if csConfig.API.Server.OnlineClient == nil { log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath) diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go index 30cf729fe..717fc990b 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/config_backup.go @@ -5,11 +5,9 @@ import ( "os" "path/filepath" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) @@ -34,17 +32,17 @@ func backupConfigToDirectory(dirPath string) error { /*if parent directory doesn't exist, bail out. create final dir with Mkdir*/ parentDir := filepath.Dir(dirPath) if _, err := os.Stat(parentDir); err != nil { - return errors.Wrapf(err, "while checking parent directory %s existence", parentDir) + return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err) } if err = os.Mkdir(dirPath, 0o700); err != nil { - return errors.Wrapf(err, "while creating %s", dirPath) + return fmt.Errorf("while creating %s: %w", dirPath, err) } if csConfig.ConfigPaths.SimulationFilePath != "" { backupSimulation := filepath.Join(dirPath, "simulation.yaml") - if err = types.CopyFile(csConfig.ConfigPaths.SimulationFilePath, backupSimulation); err != nil { - return errors.Wrapf(err, "failed copy %s to %s", csConfig.ConfigPaths.SimulationFilePath, backupSimulation) + if err = CopyFile(csConfig.ConfigPaths.SimulationFilePath, backupSimulation); err != nil { + return fmt.Errorf("failed copy %s to %s: %w", csConfig.ConfigPaths.SimulationFilePath, backupSimulation, err) } log.Infof("Saved simulation to %s", backupSimulation) @@ -56,14 +54,14 @@ func backupConfigToDirectory(dirPath string) error { */ if csConfig.Crowdsec != nil && csConfig.Crowdsec.AcquisitionFilePath != "" { backupAcquisition := filepath.Join(dirPath, "acquis.yaml") - if err = types.CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil { - return fmt.Errorf("failed copy %s to %s : %s", csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition, err) + if err = CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil { + return fmt.Errorf("failed copy %s to %s: %s", csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition, err) } } acquisBackupDir := filepath.Join(dirPath, "acquis") if err = os.Mkdir(acquisBackupDir, 0o700); err != nil { - return fmt.Errorf("error while creating %s : %s", acquisBackupDir, err) + return fmt.Errorf("error while creating %s: %s", acquisBackupDir, err) } if csConfig.Crowdsec != nil && len(csConfig.Crowdsec.AcquisitionFiles) > 0 { @@ -75,11 +73,11 @@ func backupConfigToDirectory(dirPath string) error { targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile))) if err != nil { - return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir) + return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err) } - if err = types.CopyFile(acquisFile, targetFname); err != nil { - return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err) + if err = CopyFile(acquisFile, targetFname); err != nil { + return fmt.Errorf("failed copy %s to %s: %w", acquisFile, targetFname, err) } log.Infof("Saved acquis %s to %s", acquisFile, targetFname) @@ -88,8 +86,8 @@ func backupConfigToDirectory(dirPath string) error { if ConfigFilePath != "" { backupMain := fmt.Sprintf("%s/config.yaml", dirPath) - if err = types.CopyFile(ConfigFilePath, backupMain); err != nil { - return fmt.Errorf("failed copy %s to %s : %s", ConfigFilePath, backupMain, err) + if err = CopyFile(ConfigFilePath, backupMain); err != nil { + return fmt.Errorf("failed copy %s to %s: %s", ConfigFilePath, backupMain, err) } log.Infof("Saved default yaml to %s", backupMain) @@ -97,8 +95,8 @@ func backupConfigToDirectory(dirPath string) error { if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" { backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath) - if err = types.CopyFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil { - return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err) + if err = CopyFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil { + return fmt.Errorf("failed copy %s to %s: %s", csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err) } log.Infof("Saved online API credentials to %s", backupCAPICreds) @@ -106,8 +104,8 @@ func backupConfigToDirectory(dirPath string) error { if csConfig.API != nil && csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" { backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath) - if err = types.CopyFile(csConfig.API.Client.CredentialsFilePath, backupLAPICreds); err != nil { - return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Client.CredentialsFilePath, backupLAPICreds, err) + if err = CopyFile(csConfig.API.Client.CredentialsFilePath, backupLAPICreds); err != nil { + return fmt.Errorf("failed copy %s to %s: %s", csConfig.API.Client.CredentialsFilePath, backupLAPICreds, err) } log.Infof("Saved local API credentials to %s", backupLAPICreds) @@ -115,21 +113,20 @@ func backupConfigToDirectory(dirPath string) error { if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.ProfilesPath != "" { backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath) - if err = types.CopyFile(csConfig.API.Server.ProfilesPath, backupProfiles); err != nil { - return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.ProfilesPath, backupProfiles, err) + if err = CopyFile(csConfig.API.Server.ProfilesPath, backupProfiles); err != nil { + return fmt.Errorf("failed copy %s to %s: %s", csConfig.API.Server.ProfilesPath, backupProfiles, err) } log.Infof("Saved profiles to %s", backupProfiles) } if err = BackupHub(dirPath); err != nil { - return fmt.Errorf("failed to backup hub config : %s", err) + return fmt.Errorf("failed to backup hub config: %s", err) } return nil } - func runConfigBackup(cmd *cobra.Command, args []string) error { if err := csConfig.LoadHub(); err != nil { return err @@ -147,7 +144,6 @@ func runConfigBackup(cmd *cobra.Command, args []string) error { return nil } - func NewConfigBackupCmd() *cobra.Command { cmdConfigBackup := &cobra.Command{ Use: `backup "directory"`, diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index 79d36d428..55ab7aa9b 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -7,13 +7,11 @@ import ( "os" "path/filepath" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/yaml.v2" "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) @@ -38,7 +36,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { backupMain := fmt.Sprintf("%s/config.yaml", dirPath) if _, err = os.Stat(backupMain); err == nil { if csConfig.ConfigPaths != nil && csConfig.ConfigPaths.ConfigDir != "" { - if err = types.CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)); err != nil { + if err = CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupMain, csConfig.ConfigPaths.ConfigDir, err) } } @@ -51,21 +49,21 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath) if _, err = os.Stat(backupCAPICreds); err == nil { - if err = types.CopyFile(backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath); err != nil { + if err = CopyFile(backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath, err) } } backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath) if _, err = os.Stat(backupLAPICreds); err == nil { - if err = types.CopyFile(backupLAPICreds, csConfig.API.Client.CredentialsFilePath); err != nil { + if err = CopyFile(backupLAPICreds, csConfig.API.Client.CredentialsFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupLAPICreds, csConfig.API.Client.CredentialsFilePath, err) } } backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath) if _, err = os.Stat(backupProfiles); err == nil { - if err = types.CopyFile(backupProfiles, csConfig.API.Server.ProfilesPath); err != nil { + if err = CopyFile(backupProfiles, csConfig.API.Server.ProfilesPath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupProfiles, csConfig.API.Server.ProfilesPath, err) } } @@ -106,7 +104,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { backupSimulation := fmt.Sprintf("%s/simulation.yaml", dirPath) if _, err = os.Stat(backupSimulation); err == nil { - if err = types.CopyFile(backupSimulation, csConfig.ConfigPaths.SimulationFilePath); err != nil { + if err = CopyFile(backupSimulation, csConfig.ConfigPaths.SimulationFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupSimulation, csConfig.ConfigPaths.SimulationFilePath, err) } } @@ -123,7 +121,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { if _, err = os.Stat(backupAcquisition); err == nil { log.Debugf("restoring backup'ed %s", backupAcquisition) - if err = types.CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil { + if err = CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil { return fmt.Errorf("failed copy %s to %s : %s", backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath, err) } } @@ -134,12 +132,12 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { for _, acquisFile := range acquisFiles { targetFname, err := filepath.Abs(csConfig.Crowdsec.AcquisitionDirPath + "/" + filepath.Base(acquisFile)) if err != nil { - return errors.Wrapf(err, "while saving %s to %s", acquisFile, targetFname) + return fmt.Errorf("while saving %s to %s: %w", acquisFile, targetFname, err) } log.Debugf("restoring %s to %s", acquisFile, targetFname) - if err = types.CopyFile(acquisFile, targetFname); err != nil { + if err = CopyFile(acquisFile, targetFname); err != nil { return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err) } } @@ -157,10 +155,10 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile))) if err != nil { - return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir) + return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err) } - if err = types.CopyFile(acquisFile, targetFname); err != nil { + if err = CopyFile(acquisFile, targetFname); err != nil { return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err) } @@ -175,7 +173,6 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error { return nil } - func runConfigRestore(cmd *cobra.Command, args []string) error { flags := cmd.Flags() @@ -200,7 +197,6 @@ func runConfigRestore(cmd *cobra.Command, args []string) error { return nil } - func NewConfigRestoreCmd() *cobra.Command { cmdConfigRestore := &cobra.Command{ Use: `restore "directory"`, diff --git a/cmd/crowdsec-cli/copyfile.go b/cmd/crowdsec-cli/copyfile.go new file mode 100644 index 000000000..4de6cd6e2 --- /dev/null +++ b/cmd/crowdsec-cli/copyfile.go @@ -0,0 +1,73 @@ +package main + +import ( + "fmt" + "io" + "os" + "path/filepath" + + log "github.com/sirupsen/logrus" +) + + +/*help to copy the file, ioutil doesn't offer the feature*/ + +func copyFileContents(src, dst string) (err error) { + in, err := os.Open(src) + if err != nil { + return + } + defer in.Close() + out, err := os.Create(dst) + if err != nil { + return + } + defer func() { + cerr := out.Close() + if err == nil { + err = cerr + } + }() + if _, err = io.Copy(out, in); err != nil { + return + } + err = out.Sync() + return +} + +/*copy the file, ioutile doesn't offer the feature*/ +func CopyFile(sourceSymLink, destinationFile string) (err error) { + sourceFile, err := filepath.EvalSymlinks(sourceSymLink) + if err != nil { + log.Infof("Not a symlink : %s", err) + sourceFile = sourceSymLink + } + + sourceFileStat, err := os.Stat(sourceFile) + if err != nil { + return + } + if !sourceFileStat.Mode().IsRegular() { + // cannot copy non-regular files (e.g., directories, + // symlinks, devices, etc.) + return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String()) + } + destinationFileStat, err := os.Stat(destinationFile) + if err != nil { + if !os.IsNotExist(err) { + return + } + } else { + if !(destinationFileStat.Mode().IsRegular()) { + return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String()) + } + if os.SameFile(sourceFileStat, destinationFileStat) { + return + } + } + if err = os.Link(sourceFile, destinationFile); err != nil { + err = copyFileContents(sourceFile, destinationFile) + } + return +} + diff --git a/cmd/crowdsec-cli/decisions.go b/cmd/crowdsec-cli/decisions.go index f2f3efcf8..ce3d0e46e 100644 --- a/cmd/crowdsec-cli/decisions.go +++ b/cmd/crowdsec-cli/decisions.go @@ -7,19 +7,15 @@ import ( "fmt" "net/url" "os" - "path/filepath" "strconv" "strings" "time" "github.com/fatih/color" "github.com/go-openapi/strfmt" - "github.com/jszwec/csvutil" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" - "github.com/crowdsecurity/go-cs-lib/pkg/ptr" "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/apiclient" @@ -112,12 +108,12 @@ func NewDecisionsCmd() *cobra.Command { DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { if err := csConfig.LoadAPIClient(); err != nil { - return errors.Wrap(err, "loading api client") + return fmt.Errorf("loading api client: %w", err) } password := strfmt.Password(csConfig.API.Client.Credentials.Password) apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL) if err != nil { - return errors.Wrapf(err, "parsing api url %s", csConfig.API.Client.Credentials.URL) + return fmt.Errorf("parsing api url %s: %w", csConfig.API.Client.Credentials.URL, err) } Client, err = apiclient.NewClient(&apiclient.Config{ MachineID: csConfig.API.Client.Credentials.Login, @@ -127,7 +123,7 @@ func NewDecisionsCmd() *cobra.Command { VersionPrefix: "v1", }) if err != nil { - return errors.Wrap(err, "creating api client") + return fmt.Errorf("creating api client: %w", err) } return nil }, @@ -169,11 +165,11 @@ cscli decisions list -t ban `, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { var err error /*take care of shorthand options*/ - if err := manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil { - log.Fatalf("%s", err) + if err = manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil { + return err } filter.ActiveDecisionEquals = new(bool) *filter.ActiveDecisionEquals = true @@ -189,7 +185,7 @@ cscli decisions list -t ban days, err := strconv.Atoi(realDuration) if err != nil { printHelp(cmd) - log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until) + return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until) } *filter.Until = fmt.Sprintf("%d%s", days*24, "h") } @@ -202,7 +198,7 @@ cscli decisions list -t ban days, err := strconv.Atoi(realDuration) if err != nil { printHelp(cmd) - log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until) + return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Since) } *filter.Since = fmt.Sprintf("%d%s", days*24, "h") } @@ -238,13 +234,15 @@ cscli decisions list -t ban alerts, _, err := Client.Alerts.List(context.Background(), filter) if err != nil { - log.Fatalf("Unable to list decisions : %v", err) + return fmt.Errorf("unable to retrieve decisions: %w", err) } err = DecisionsToTable(alerts, printMachine) if err != nil { - log.Fatalf("unable to list decisions : %v", err) + return fmt.Errorf("unable to print decisions: %w", err) } + + return nil }, } cmdDecisionsList.Flags().SortFlags = false @@ -288,7 +286,7 @@ cscli decisions add --scope username --value foobar /*TBD : fix long and example*/ Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { var err error alerts := models.AddAlertsRequest{} origin := types.CscliOrigin @@ -303,7 +301,7 @@ cscli decisions add --scope username --value foobar /*take care of shorthand options*/ if err := manageCliDecisionAlerts(&addIP, &addRange, &addScope, &addValue); err != nil { - log.Fatalf("%s", err) + return err } if addIP != "" { @@ -314,7 +312,7 @@ cscli decisions add --scope username --value foobar addScope = types.Range } else if addValue == "" { printHelp(cmd) - log.Fatalf("Missing arguments, a value is required (--ip, --range or --scope and --value)") + return fmt.Errorf("Missing arguments, a value is required (--ip, --range or --scope and --value)") } if addReason == "" { @@ -357,10 +355,11 @@ cscli decisions add --scope username --value foobar _, _, err = Client.Alerts.Add(context.Background(), alerts) if err != nil { - log.Fatal(err) + return err } log.Info("Decision successfully added") + return nil }, } @@ -401,25 +400,27 @@ cscli decisions delete --id 42 cscli decisions delete --type captcha `, /*TBD : refaire le Long/Example*/ - PreRun: func(cmd *cobra.Command, args []string) { + PreRunE: func(cmd *cobra.Command, args []string) error { if delDecisionAll { - return + return nil } if *delFilter.ScopeEquals == "" && *delFilter.ValueEquals == "" && *delFilter.TypeEquals == "" && *delFilter.IPEquals == "" && *delFilter.RangeEquals == "" && *delFilter.ScenarioEquals == "" && *delFilter.OriginEquals == "" && delDecisionId == "" { cmd.Usage() - log.Fatalln("At least one filter or --all must be specified") + return fmt.Errorf("at least one filter or --all must be specified") } + + return nil }, - Run: func(cmd *cobra.Command, args []string) { + RunE: func(cmd *cobra.Command, args []string) error { var err error var decisions *models.DeleteDecisionResponse /*take care of shorthand options*/ - if err := manageCliDecisionAlerts(delFilter.IPEquals, delFilter.RangeEquals, delFilter.ScopeEquals, delFilter.ValueEquals); err != nil { - log.Fatalf("%s", err) + if err = manageCliDecisionAlerts(delFilter.IPEquals, delFilter.RangeEquals, delFilter.ScopeEquals, delFilter.ValueEquals); err != nil { + return err } if *delFilter.ScopeEquals == "" { delFilter.ScopeEquals = nil @@ -449,18 +450,19 @@ cscli decisions delete --type captcha if delDecisionId == "" { decisions, _, err = Client.Decisions.Delete(context.Background(), delFilter) if err != nil { - log.Fatalf("Unable to delete decisions : %v", err) + return fmt.Errorf("Unable to delete decisions: %v", err) } } else { if _, err = strconv.Atoi(delDecisionId); err != nil { - log.Fatalf("id '%s' is not an integer: %v", delDecisionId, err) + return fmt.Errorf("id '%s' is not an integer: %v", delDecisionId, err) } decisions, _, err = Client.Decisions.DeleteOne(context.Background(), delDecisionId) if err != nil { - log.Fatalf("Unable to delete decision : %v", err) + return fmt.Errorf("Unable to delete decision: %v", err) } } log.Infof("%s decision(s) deleted", decisions.NbDeleted) + return nil }, } @@ -478,192 +480,3 @@ cscli decisions delete --type captcha return cmdDecisionsDelete } - -func NewDecisionsImportCmd() *cobra.Command { - var ( - defaultDuration = "4h" - defaultScope = "ip" - defaultType = "ban" - defaultReason = "manual" - importDuration string - importScope string - importReason string - importType string - importFile string - batchSize int - ) - - var cmdDecisionImport = &cobra.Command{ - Use: "import [options]", - Short: "Import decisions from json or csv file", - Long: "expected format :\n" + - "csv : any of duration,origin,reason,scope,type,value, with a header line\n" + - `json : {"duration" : "24h", "origin" : "my-list", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`, - DisableAutoGenTag: true, - Example: `decisions.csv : -duration,scope,value -24h,ip,1.2.3.4 - -cscsli decisions import -i decisions.csv - -decisions.json : -[{"duration" : "4h", "scope" : "ip", "type" : "ban", "value" : "1.2.3.4"}] -`, - Run: func(cmd *cobra.Command, args []string) { - if importFile == "" { - log.Fatalf("Please provide a input file containing decisions with -i flag") - } - csvData, err := os.ReadFile(importFile) - if err != nil { - log.Fatalf("unable to open '%s': %s", importFile, err) - } - type decisionRaw struct { - Duration string `csv:"duration,omitempty" json:"duration,omitempty"` - Origin string `csv:"origin,omitempty" json:"origin,omitempty"` - Scenario string `csv:"reason,omitempty" json:"reason,omitempty"` - Scope string `csv:"scope,omitempty" json:"scope,omitempty"` - Type string `csv:"type,omitempty" json:"type,omitempty"` - Value string `csv:"value" json:"value"` - } - var decisionsListRaw []decisionRaw - switch fileFormat := filepath.Ext(importFile); fileFormat { - case ".json": - if err := json.Unmarshal(csvData, &decisionsListRaw); err != nil { - log.Fatalf("unable to unmarshall json: '%s'", err) - } - case ".csv": - if err := csvutil.Unmarshal(csvData, &decisionsListRaw); err != nil { - log.Fatalf("unable to unmarshall csv: '%s'", err) - } - default: - log.Fatalf("file format not supported for '%s'. supported format are 'json' and 'csv'", importFile) - } - - decisionsList := make([]*models.Decision, 0) - for i, decisionLine := range decisionsListRaw { - line := i + 2 - if decisionLine.Value == "" { - log.Fatalf("please provide a 'value' in your csv line %d", line) - } - /*deal with defaults and cli-override*/ - if decisionLine.Duration == "" { - decisionLine.Duration = defaultDuration - log.Debugf("No 'duration' line %d, using default value: '%s'", line, defaultDuration) - } - if importDuration != "" { - decisionLine.Duration = importDuration - log.Debugf("'duration' line %d, using supplied value: '%s'", line, importDuration) - } - decisionLine.Origin = types.CscliImportOrigin - - if decisionLine.Scenario == "" { - decisionLine.Scenario = defaultReason - log.Debugf("No 'reason' line %d, using value: '%s'", line, decisionLine.Scenario) - } - if importReason != "" { - decisionLine.Scenario = importReason - log.Debugf("No 'reason' line %d, using supplied value: '%s'", line, importReason) - } - if decisionLine.Type == "" { - decisionLine.Type = defaultType - log.Debugf("No 'type' line %d, using default value: '%s'", line, decisionLine.Type) - } - if importType != "" { - decisionLine.Type = importType - log.Debugf("'type' line %d, using supplied value: '%s'", line, importType) - } - if decisionLine.Scope == "" { - decisionLine.Scope = defaultScope - log.Debugf("No 'scope' line %d, using default value: '%s'", line, decisionLine.Scope) - } - if importScope != "" { - decisionLine.Scope = importScope - log.Debugf("'scope' line %d, using supplied value: '%s'", line, importScope) - } - decision := models.Decision{ - Value: ptr.Of(decisionLine.Value), - Duration: ptr.Of(decisionLine.Duration), - Origin: ptr.Of(decisionLine.Origin), - Scenario: ptr.Of(decisionLine.Scenario), - Type: ptr.Of(decisionLine.Type), - Scope: ptr.Of(decisionLine.Scope), - Simulated: new(bool), - } - decisionsList = append(decisionsList, &decision) - } - alerts := models.AddAlertsRequest{} - - if batchSize > 0 { - for i := 0; i < len(decisionsList); i += batchSize { - end := i + batchSize - if end > len(decisionsList) { - end = len(decisionsList) - } - decisionBatch := decisionsList[i:end] - importAlert := models.Alert{ - CreatedAt: time.Now().UTC().Format(time.RFC3339), - Scenario: ptr.Of(fmt.Sprintf("import %s : %d IPs", importFile, len(decisionBatch))), - - Message: ptr.Of(""), - Events: []*models.Event{}, - Source: &models.Source{ - Scope: ptr.Of(""), - Value: ptr.Of(""), - }, - StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), - StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), - Capacity: ptr.Of(int32(0)), - Simulated: ptr.Of(false), - EventsCount: ptr.Of(int32(len(decisionBatch))), - Leakspeed: ptr.Of(""), - ScenarioHash: ptr.Of(""), - ScenarioVersion: ptr.Of(""), - Decisions: decisionBatch, - } - alerts = append(alerts, &importAlert) - } - } else { - importAlert := models.Alert{ - CreatedAt: time.Now().UTC().Format(time.RFC3339), - Scenario: ptr.Of(fmt.Sprintf("import %s : %d IPs", importFile, len(decisionsList))), - Message: ptr.Of(""), - Events: []*models.Event{}, - Source: &models.Source{ - Scope: ptr.Of(""), - Value: ptr.Of(""), - }, - StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), - StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), - Capacity: ptr.Of(int32(0)), - Simulated: ptr.Of(false), - EventsCount: ptr.Of(int32(len(decisionsList))), - Leakspeed: ptr.Of(""), - ScenarioHash: ptr.Of(""), - ScenarioVersion: ptr.Of(""), - Decisions: decisionsList, - } - alerts = append(alerts, &importAlert) - } - - if len(decisionsList) > 1000 { - log.Infof("You are about to add %d decisions, this may take a while", len(decisionsList)) - } - - _, _, err = Client.Alerts.Add(context.Background(), alerts) - if err != nil { - log.Fatal(err) - } - log.Infof("%d decisions successfully imported", len(decisionsList)) - }, - } - - cmdDecisionImport.Flags().SortFlags = false - cmdDecisionImport.Flags().StringVarP(&importFile, "input", "i", "", "Input file") - cmdDecisionImport.Flags().StringVarP(&importDuration, "duration", "d", "", "Decision duration (ie. 1h,4h,30m)") - cmdDecisionImport.Flags().StringVar(&importScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)") - cmdDecisionImport.Flags().StringVarP(&importReason, "reason", "R", "", "Decision reason (ie. scenario-name)") - cmdDecisionImport.Flags().StringVarP(&importType, "type", "t", "", "Decision type (ie. ban,captcha,throttle)") - cmdDecisionImport.Flags().IntVar(&batchSize, "batch", 0, "Split import in batches of N decisions") - - return cmdDecisionImport -} diff --git a/cmd/crowdsec-cli/decisions_import.go b/cmd/crowdsec-cli/decisions_import.go new file mode 100644 index 000000000..6a47a96b3 --- /dev/null +++ b/cmd/crowdsec-cli/decisions_import.go @@ -0,0 +1,272 @@ +package main + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "os" + "strings" + "time" + + "github.com/jszwec/csvutil" + log "github.com/sirupsen/logrus" + "github.com/spf13/cobra" + + "github.com/crowdsecurity/go-cs-lib/pkg/ptr" + "github.com/crowdsecurity/go-cs-lib/pkg/slicetools" + + "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/crowdsecurity/crowdsec/pkg/types" +) + +// decisionRaw is only used to unmarshall json/csv decisions +type decisionRaw struct { + Duration string `csv:"duration,omitempty" json:"duration,omitempty"` + Scenario string `csv:"reason,omitempty" json:"reason,omitempty"` + Scope string `csv:"scope,omitempty" json:"scope,omitempty"` + Type string `csv:"type,omitempty" json:"type,omitempty"` + Value string `csv:"value" json:"value"` +} + +func parseDecisionList(content []byte, format string) ([]decisionRaw, error) { + ret := []decisionRaw{} + + switch format { + case "values": + log.Infof("Parsing values") + scanner := bufio.NewScanner(bytes.NewReader(content)) + for scanner.Scan() { + value := strings.TrimSpace(scanner.Text()) + ret = append(ret, decisionRaw{Value: value}) + } + if err := scanner.Err(); err != nil { + return nil, fmt.Errorf("unable to parse values: '%s'", err) + } + case "json": + log.Infof("Parsing json") + if err := json.Unmarshal(content, &ret); err != nil { + return nil, err + } + case "csv": + log.Infof("Parsing csv") + if err := csvutil.Unmarshal(content, &ret); err != nil { + return nil, fmt.Errorf("unable to parse csv: '%s'", err) + } + default: + return nil, fmt.Errorf("invalid format '%s', expected one of 'json', 'csv', 'values'", format) + } + + return ret, nil +} + + +func runDecisionsImport(cmd *cobra.Command, args []string) error { + flags := cmd.Flags() + + input, err := flags.GetString("input") + if err != nil { + return err + } + + defaultDuration, err := flags.GetString("duration") + if err != nil { + return err + } + if defaultDuration == "" { + return fmt.Errorf("--duration cannot be empty") + } + + defaultScope, err := flags.GetString("scope") + if err != nil { + return err + } + if defaultScope == "" { + return fmt.Errorf("--scope cannot be empty") + } + + defaultReason, err := flags.GetString("reason") + if err != nil { + return err + } + if defaultReason == "" { + return fmt.Errorf("--reason cannot be empty") + } + + defaultType, err := flags.GetString("type") + if err != nil { + return err + } + if defaultType == "" { + return fmt.Errorf("--type cannot be empty") + } + + batchSize, err := flags.GetInt("batch") + if err != nil { + return err + } + + format, err := flags.GetString("format") + if err != nil { + return err + } + + var ( + content []byte + fin *os.File + ) + + // set format if the file has a json or csv extension + if format == "" { + if strings.HasSuffix(input, ".json") { + format = "json" + } else if strings.HasSuffix(input, ".csv") { + format = "csv" + } + } + + if format == "" { + return fmt.Errorf("unable to guess format from file extension, please provide a format with --format flag") + } + + if input == "-" { + fin = os.Stdin + input = "stdin" + } else { + fin, err = os.Open(input) + if err != nil { + return fmt.Errorf("unable to open %s: %s", input, err) + } + } + + content, err = io.ReadAll(fin) + if err != nil { + return fmt.Errorf("unable to read from %s: %s", input, err) + } + + decisionsListRaw, err := parseDecisionList(content, format) + if err != nil { + return err + } + + decisions := make([]*models.Decision, len(decisionsListRaw)) + for i, d := range decisionsListRaw { + if d.Value == "" { + return fmt.Errorf("item %d: missing 'value'", i) + } + + if d.Duration == "" { + d.Duration = defaultDuration + log.Debugf("item %d: missing 'duration', using default '%s'", i, defaultDuration) + } + + if d.Scenario == "" { + d.Scenario = defaultReason + log.Debugf("item %d: missing 'reason', using default '%s'", i, defaultReason) + } + + if d.Type == "" { + d.Type = defaultType + log.Debugf("item %d: missing 'type', using default '%s'", i, defaultType) + } + + if d.Scope == "" { + d.Scope = defaultScope + log.Debugf("item %d: missing 'scope', using default '%s'", i, defaultScope) + } + + decisions[i] = &models.Decision{ + Value: ptr.Of(d.Value), + Duration: ptr.Of(d.Duration), + Origin: ptr.Of(types.CscliImportOrigin), + Scenario: ptr.Of(d.Scenario), + Type: ptr.Of(d.Type), + Scope: ptr.Of(d.Scope), + Simulated: ptr.Of(false), + } + } + + alerts := models.AddAlertsRequest{} + + for _, chunk := range slicetools.Chunks(decisions, batchSize) { + log.Debugf("Processing chunk of %d decisions", len(chunk)) + importAlert := models.Alert{ + CreatedAt: time.Now().UTC().Format(time.RFC3339), + Scenario: ptr.Of(fmt.Sprintf("import %s: %d IPs", input, len(chunk))), + + Message: ptr.Of(""), + Events: []*models.Event{}, + Source: &models.Source{ + Scope: ptr.Of(""), + Value: ptr.Of(""), + }, + StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), + StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)), + Capacity: ptr.Of(int32(0)), + Simulated: ptr.Of(false), + EventsCount: ptr.Of(int32(len(chunk))), + Leakspeed: ptr.Of(""), + ScenarioHash: ptr.Of(""), + ScenarioVersion: ptr.Of(""), + Decisions: chunk, + } + alerts = append(alerts, &importAlert) + } + + if len(decisions) > 1000 { + log.Infof("You are about to add %d decisions, this may take a while", len(decisions)) + } + + _, _, err = Client.Alerts.Add(context.Background(), alerts) + if err != nil { + return err + } + + log.Infof("Imported %d decisions", len(decisions)) + return nil +} + + +func NewDecisionsImportCmd() *cobra.Command { + var cmdDecisionsImport = &cobra.Command{ + Use: "import [options]", + Short: "Import decisions from a file or pipe", + Long: "expected format:\n" + + "csv : any of duration,reason,scope,type,value, with a header line\n" + + `json : {"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`, + DisableAutoGenTag: true, + Example: `decisions.csv: +duration,scope,value +24h,ip,1.2.3.4 + +$ cscli decisions import -i decisions.csv + +decisions.json: +[{"duration" : "4h", "scope" : "ip", "type" : "ban", "value" : "1.2.3.4"}] + +The file format is detected from the extension, but can be forced with the --format option +which is required when reading from standard input. + +Raw values, standard input: + +$ echo "1.2.3.4" | cscli decisions import -i - --format values +`, + RunE: runDecisionsImport, + } + + flags := cmdDecisionsImport.Flags() + flags.SortFlags = false + flags.StringP("input", "i", "", "Input file") + flags.StringP("duration", "d", "4h", "Decision duration: 1h,4h,30m") + flags.String("scope", types.Ip, "Decision scope: ip,range,username") + flags.StringP("reason", "R", "manual", "Decision reason: ") + flags.StringP("type", "t", "ban", "Decision type: ban,captcha,throttle") + flags.Int("batch", 0, "Split import in batches of N decisions") + flags.String("format", "", "Input format: 'json', 'csv' or 'values' (each line is a value, no headers)") + + cmdDecisionsImport.MarkFlagRequired("input") + + return cmdDecisionsImport +} diff --git a/cmd/crowdsec-cli/lapi.go b/cmd/crowdsec-cli/lapi.go index e8a29d9cd..d03662c48 100644 --- a/cmd/crowdsec-cli/lapi.go +++ b/cmd/crowdsec-cli/lapi.go @@ -9,7 +9,6 @@ import ( "strings" "github.com/go-openapi/strfmt" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "golang.org/x/exp/slices" @@ -24,7 +23,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/parser" - "github.com/crowdsecurity/crowdsec/pkg/types" ) var LAPIURLPrefix string = "v1" @@ -205,7 +203,7 @@ func NewLapiCmd() *cobra.Command { DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { if err := csConfig.LoadAPIClient(); err != nil { - return errors.Wrap(err, "loading api client") + return fmt.Errorf("loading api client: %w", err) } return nil }, @@ -440,7 +438,7 @@ cscli lapi context delete --value evt.Line.Src return cmdContext } -func detectStaticField(GrokStatics []types.ExtraField) []string { +func detectStaticField(GrokStatics []parser.ExtraField) []string { ret := make([]string, 0) for _, static := range GrokStatics { if static.Parsed != "" { diff --git a/cmd/crowdsec-cli/machines.go b/cmd/crowdsec-cli/machines.go index 25bd5acec..215943102 100644 --- a/cmd/crowdsec-cli/machines.go +++ b/cmd/crowdsec-cli/machines.go @@ -12,7 +12,6 @@ import ( "time" "github.com/AlecAivazis/survey/v2" - "github.com/enescakir/emoji" "github.com/fatih/color" "github.com/go-openapi/strfmt" "github.com/google/uuid" @@ -85,22 +84,21 @@ func generateID(prefix string) (string, error) { return prefix + suffix, nil } -func displayLastHeartBeat(m *ent.Machine, fancy bool) string { - var hbDisplay string - - if m.LastHeartbeat != nil { - lastHeartBeat := time.Now().UTC().Sub(*m.LastHeartbeat) - hbDisplay = lastHeartBeat.Truncate(time.Second).String() - if fancy && lastHeartBeat > 2*time.Minute { - hbDisplay = fmt.Sprintf("%s %s", emoji.Warning.String(), lastHeartBeat.Truncate(time.Second).String()) - } - } else { - hbDisplay = "-" - if fancy { - hbDisplay = emoji.Warning.String() + " -" - } +// getLastHeartbeat returns the last heartbeat timestamp of a machine +// and a boolean indicating if the machine is considered active or not. +func getLastHeartbeat(m *ent.Machine) (string, bool) { + if m.LastHeartbeat == nil { + return "-", false } - return hbDisplay + + elapsed := time.Now().UTC().Sub(*m.LastHeartbeat) + + hb := elapsed.Truncate(time.Second).String() + if elapsed > 2*time.Minute { + return hb, false + } + + return hb, true } func getAgents(out io.Writer, dbClient *database.Client) error { @@ -130,9 +128,10 @@ func getAgents(out io.Writer, dbClient *database.Client) error { } else { validated = "false" } - err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, false)}) + hb, _ := getLastHeartbeat(m) + err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb}) if err != nil { - return fmt.Errorf("failed to write raw output : %s", err) + return fmt.Errorf("failed to write raw output: %w", err) } } csvwriter.Flush() diff --git a/cmd/crowdsec-cli/machines_table.go b/cmd/crowdsec-cli/machines_table.go index cc15bb51b..e166fb785 100644 --- a/cmd/crowdsec-cli/machines_table.go +++ b/cmd/crowdsec-cli/machines_table.go @@ -24,7 +24,11 @@ func getAgentsTable(out io.Writer, machines []*ent.Machine) { validated = emoji.Prohibited.String() } - t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, true)) + hb, active := getLastHeartbeat(m) + if !active { + hb = emoji.Warning.String() + " " + hb + } + t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb) } t.Render() diff --git a/cmd/crowdsec-cli/notifications.go b/cmd/crowdsec-cli/notifications.go index ebbefd6b7..fe4c14f27 100644 --- a/cmd/crowdsec-cli/notifications.go +++ b/cmd/crowdsec-cli/notifications.go @@ -15,7 +15,6 @@ import ( "github.com/fatih/color" "github.com/go-openapi/strfmt" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/tomb.v2" @@ -28,14 +27,12 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csprofiles" ) - type NotificationsCfg struct { Config csplugin.PluginConfig `json:"plugin_config"` Profiles []*csconfig.ProfileCfg `json:"associated_profiles"` ids []uint } - func NewNotificationsCmd() *cobra.Command { var cmdNotifications = &cobra.Command{ Use: "notifications [action]", @@ -57,7 +54,6 @@ func NewNotificationsCmd() *cobra.Command { }, } - cmdNotifications.AddCommand(NewNotificationsListCmd()) cmdNotifications.AddCommand(NewNotificationsInspectCmd()) cmdNotifications.AddCommand(NewNotificationsReinjectCmd()) @@ -65,18 +61,17 @@ func NewNotificationsCmd() *cobra.Command { return cmdNotifications } - func getNotificationsConfiguration() (map[string]NotificationsCfg, error) { pcfgs := map[string]csplugin.PluginConfig{} wf := func(path string, info fs.FileInfo, err error) error { if info == nil { - return errors.Wrapf(err, "error while traversing directory %s", path) + return fmt.Errorf("error while traversing directory %s: %w", path, err) } name := filepath.Join(csConfig.ConfigPaths.NotificationDir, info.Name()) //Avoid calling info.Name() twice if (strings.HasSuffix(name, "yaml") || strings.HasSuffix(name, "yml")) && !(info.IsDir()) { ts, err := csplugin.ParsePluginConfigFile(name) if err != nil { - return errors.Wrapf(err, "Loading notifification plugin configuration with %s", name) + return fmt.Errorf("loading notifification plugin configuration with %s: %w", name, err) } for _, t := range ts { pcfgs[t.Name] = t @@ -86,14 +81,14 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) { } if err := filepath.Walk(csConfig.ConfigPaths.NotificationDir, wf); err != nil { - return nil, errors.Wrap(err, "Loading notifification plugin configuration") + return nil, fmt.Errorf("while loading notifification plugin configuration: %w", err) } // A bit of a tricky stuf now: reconcile profiles and notification plugins ncfgs := map[string]NotificationsCfg{} profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles) if err != nil { - return nil, errors.Wrap(err, "Cannot extract profiles from configuration") + return nil, fmt.Errorf("while extracting profiles from configuration: %w", err) } for profileID, profile := range profiles { loop: @@ -129,7 +124,6 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) { return ncfgs, nil } - func NewNotificationsListCmd() *cobra.Command { var cmdNotificationsList = &cobra.Command{ Use: "list", @@ -141,7 +135,7 @@ func NewNotificationsListCmd() *cobra.Command { RunE: func(cmd *cobra.Command, arg []string) error { ncfgs, err := getNotificationsConfiguration() if err != nil { - return errors.Wrap(err, "Can't build profiles configuration") + return fmt.Errorf("can't build profiles configuration: %w", err) } if csConfig.Cscli.Output == "human" { @@ -149,14 +143,14 @@ func NewNotificationsListCmd() *cobra.Command { } else if csConfig.Cscli.Output == "json" { x, err := json.MarshalIndent(ncfgs, "", " ") if err != nil { - return errors.New("failed to marshal notification configuration") + return fmt.Errorf("failed to marshal notification configuration: %w", err) } fmt.Printf("%s", string(x)) } else if csConfig.Cscli.Output == "raw" { csvwriter := csv.NewWriter(os.Stdout) err := csvwriter.Write([]string{"Name", "Type", "Profile name"}) if err != nil { - return errors.Wrap(err, "failed to write raw header") + return fmt.Errorf("failed to write raw header: %w", err) } for _, b := range ncfgs { profilesList := []string{} @@ -165,7 +159,7 @@ func NewNotificationsListCmd() *cobra.Command { } err := csvwriter.Write([]string{b.Config.Name, b.Config.Type, strings.Join(profilesList, ", ")}) if err != nil { - return errors.Wrap(err, "failed to write raw content") + return fmt.Errorf("failed to write raw content: %w", err) } } csvwriter.Flush() @@ -177,7 +171,6 @@ func NewNotificationsListCmd() *cobra.Command { return cmdNotificationsList } - func NewNotificationsInspectCmd() *cobra.Command { var cmdNotificationsInspect = &cobra.Command{ Use: "inspect", @@ -195,14 +188,14 @@ func NewNotificationsInspectCmd() *cobra.Command { pluginName := arg[0] if pluginName == "" { - errors.New("Please provide a plugin name to inspect") + return fmt.Errorf("please provide a plugin name to inspect") } ncfgs, err := getNotificationsConfiguration() if err != nil { - return errors.Wrap(err, "Can't build profiles configuration") + return fmt.Errorf("can't build profiles configuration: %w", err) } if cfg, ok = ncfgs[pluginName]; !ok { - return errors.New("The provided plugin name doesn't exist or isn't active") + return fmt.Errorf("plugin '%s' does not exist or is not active", pluginName) } if csConfig.Cscli.Output == "human" || csConfig.Cscli.Output == "raw" { @@ -216,7 +209,7 @@ func NewNotificationsInspectCmd() *cobra.Command { } else if csConfig.Cscli.Output == "json" { x, err := json.MarshalIndent(cfg, "", " ") if err != nil { - return errors.New("failed to marshal notification configuration") + return fmt.Errorf("failed to marshal notification configuration: %w", err) } fmt.Printf("%s", string(x)) } @@ -227,7 +220,6 @@ func NewNotificationsInspectCmd() *cobra.Command { return cmdNotificationsInspect } - func NewNotificationsReinjectCmd() *cobra.Command { var remediation bool var alertOverride string @@ -250,26 +242,26 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not ) if len(args) != 1 { printHelp(cmd) - return errors.New("Wrong number of argument: there should be one argument") + return fmt.Errorf("wrong number of argument: there should be one argument") } //first: get the alert id, err := strconv.Atoi(args[0]) if err != nil { - return errors.New(fmt.Sprintf("bad alert id %s", args[0])) + return fmt.Errorf("bad alert id %s", args[0]) } if err := csConfig.LoadAPIClient(); err != nil { - return errors.Wrapf(err, "loading api client") + return fmt.Errorf("loading api client: %w", err) } if csConfig.API.Client == nil { - return errors.New("There is no configuration on 'api_client:'") + return fmt.Errorf("missing configuration on 'api_client:'") } if csConfig.API.Client.Credentials == nil { - return errors.New(fmt.Sprintf("Please provide credentials for the API in '%s'", csConfig.API.Client.CredentialsFilePath)) + return fmt.Errorf("missing API credentials in '%s'", csConfig.API.Client.CredentialsFilePath) } apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL) if err != nil { - return errors.Wrapf(err, "error parsing the URL of the API") + return fmt.Errorf("error parsing the URL of the API: %w", err) } client, err := apiclient.NewClient(&apiclient.Config{ MachineID: csConfig.API.Client.Credentials.Login, @@ -279,16 +271,16 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not VersionPrefix: "v1", }) if err != nil { - return errors.Wrapf(err, "error creating the client for the API") + return fmt.Errorf("error creating the client for the API: %w", err) } alert, _, err := client.Alerts.GetByID(context.Background(), id) if err != nil { - return errors.Wrapf(err, fmt.Sprintf("can't find alert with id %s", args[0])) + return fmt.Errorf("can't find alert with id %s: %w", args[0], err) } if alertOverride != "" { if err = json.Unmarshal([]byte(alertOverride), alert); err != nil { - return errors.Wrapf(err, "Can't unmarshal the data given in the alert flag") + return fmt.Errorf("can't unmarshal data in the alert flag: %w", err) } } if !remediation { @@ -298,7 +290,7 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not // second we start plugins err = pluginBroker.Init(csConfig.PluginConfig, csConfig.API.Server.Profiles, csConfig.ConfigPaths) if err != nil { - return errors.Wrapf(err, "Can't initialize plugins") + return fmt.Errorf("can't initialize plugins: %w", err) } pluginTomb.Go(func() error { @@ -310,13 +302,13 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles) if err != nil { - return errors.Wrap(err, "Cannot extract profiles from configuration") + return fmt.Errorf("cannot extract profiles from configuration: %w", err) } for id, profile := range profiles { _, matched, err := profile.EvaluateProfile(alert) if err != nil { - return errors.Wrapf(err, "can't evaluate profile %s", profile.Cfg.Name) + return fmt.Errorf("can't evaluate profile %s: %w", profile.Cfg.Name, err) } if !matched { log.Infof("The profile %s didn't match", profile.Cfg.Name) @@ -344,7 +336,7 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not } // time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent - pluginTomb.Kill(errors.New("terminating")) + pluginTomb.Kill(fmt.Errorf("terminating")) pluginTomb.Wait() return nil }, diff --git a/cmd/crowdsec-cli/scenarios.go b/cmd/crowdsec-cli/scenarios.go index a5b433228..de52dcb48 100644 --- a/cmd/crowdsec-cli/scenarios.go +++ b/cmd/crowdsec-cli/scenarios.go @@ -4,7 +4,6 @@ import ( "fmt" "github.com/fatih/color" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -33,7 +32,7 @@ cscli scenarios remove crowdsecurity/ssh-bf } if err := cwhub.SetHubBranch(); err != nil { - return errors.Wrap(err, "while setting hub branch") + return fmt.Errorf("while setting hub branch: %w", err) } if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index 013abf4b2..66c1493a4 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -26,7 +26,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/fflag" "github.com/crowdsecurity/crowdsec/pkg/models" - "github.com/crowdsecurity/crowdsec/pkg/types" ) const ( @@ -48,6 +47,14 @@ const ( SUPPORT_CROWDSEC_PROFILE_PATH = "config/profiles.yaml" ) +// from https://github.com/acarl005/stripansi +var reStripAnsi = regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))") + +func stripAnsiString(str string) string { + // the byte version doesn't strip correctly + return reStripAnsi.ReplaceAllString(str, "") +} + func collectMetrics() ([]byte, []byte, error) { log.Info("Collecting prometheus metrics") err := csConfig.LoadPrometheus() @@ -400,7 +407,7 @@ cscli support dump -f /tmp/crowdsec-support.zip log.Errorf("Could not add zip entry for %s: %s", filename, err) continue } - fw.Write([]byte(types.StripAnsiString(string(data)))) + fw.Write([]byte(stripAnsiString(string(data)))) } err = zipWriter.Close() diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index 1a99c2acf..f77342cc9 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -598,7 +598,7 @@ func RestoreHub(dirPath string) error { log.Infof("Going to restore local/tainted [%s]", tfile.Name()) sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name()) destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name()) - if err = types.CopyFile(sourceFile, destinationFile); err != nil { + if err = CopyFile(sourceFile, destinationFile); err != nil { return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err) } log.Infof("restored %s to %s", sourceFile, destinationFile) @@ -607,7 +607,7 @@ func RestoreHub(dirPath string) error { log.Infof("Going to restore local/tainted [%s]", file.Name()) sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name()) destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name()) - if err = types.CopyFile(sourceFile, destinationFile); err != nil { + if err = CopyFile(sourceFile, destinationFile); err != nil { return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err) } log.Infof("restored %s to %s", sourceFile, destinationFile) @@ -657,7 +657,7 @@ func BackupHub(dirPath string) error { } clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate) tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName) - if err = types.CopyFile(v.LocalPath, tfile); err != nil { + if err = CopyFile(v.LocalPath, tfile); err != nil { return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err) } clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile) diff --git a/cmd/crowdsec/api.go b/cmd/crowdsec/api.go index 3ce249d4c..fd2e2ce08 100644 --- a/cmd/crowdsec/api.go +++ b/cmd/crowdsec/api.go @@ -1,6 +1,7 @@ package main import ( + "fmt" "runtime" "time" @@ -20,7 +21,7 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) { apiServer, err := apiserver.NewServer(cConfig.API.Server) if err != nil { - return nil, errors.Wrap(err, "unable to run local API") + return nil, fmt.Errorf("unable to run local API: %w", err) } if hasPlugins(cConfig.API.Server.Profiles) { @@ -29,23 +30,27 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) { if cConfig.PluginConfig == nil && runtime.GOOS != "windows" { return nil, errors.New("plugins are enabled, but the plugin_config section is missing in the configuration") } + if cConfig.ConfigPaths.NotificationDir == "" { return nil, errors.New("plugins are enabled, but config_paths.notification_dir is not defined") } + if cConfig.ConfigPaths.PluginDir == "" { return nil, errors.New("plugins are enabled, but config_paths.plugin_dir is not defined") } + err = pluginBroker.Init(cConfig.PluginConfig, cConfig.API.Server.Profiles, cConfig.ConfigPaths) if err != nil { - return nil, errors.Wrap(err, "unable to run local API") + return nil, fmt.Errorf("unable to run plugin broker: %w", err) } + log.Info("initiated plugin broker") apiServer.AttachPluginBroker(&pluginBroker) } err = apiServer.InitController() if err != nil { - return nil, errors.Wrap(err, "unable to run local API") + return nil, fmt.Errorf("unable to run local API: %w", err) } return apiServer, nil diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 8b4487e15..68a7c6180 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -3,10 +3,12 @@ package main import ( "fmt" "os" + "path/filepath" "sync" "time" - "path/filepath" + log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" "github.com/crowdsecurity/go-cs-lib/pkg/trace" @@ -16,31 +18,28 @@ import ( leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" "github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" ) func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { var err error // Populate cwhub package tools - if err := cwhub.GetHubIdx(cConfig.Hub); err != nil { - return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err) + if err = cwhub.GetHubIdx(cConfig.Hub); err != nil { + return nil, fmt.Errorf("while loading hub index: %w", err) } // Start loading configs csParsers := parser.NewParsers() if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil { - return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err) + return nil, fmt.Errorf("while loading parsers: %w", err) } if err := LoadBuckets(cConfig); err != nil { - return &parser.Parsers{}, fmt.Errorf("Failed to load scenarios: %s", err) + return nil, fmt.Errorf("while loading scenarios: %w", err) } if err := LoadAcquisition(cConfig); err != nil { - return &parser.Parsers{}, fmt.Errorf("Error while loading acquisition config : %s", err) + return nil, fmt.Errorf("while loading acquisition config: %w", err) } return csParsers, nil } @@ -118,7 +117,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error { aggregated = true } if err := acquisition.GetMetrics(dataSources, aggregated); err != nil { - return errors.Wrap(err, "while fetching prometheus metrics for datasources.") + return fmt.Errorf("while fetching prometheus metrics for datasources: %w", err) } } diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index 767097f0e..0de6f0378 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -51,12 +51,15 @@ var ( ) type Flags struct { - ConfigFile string - TraceLevel bool - DebugLevel bool - InfoLevel bool - WarnLevel bool - ErrorLevel bool + ConfigFile string + + LogLevelTrace bool + LogLevelDebug bool + LogLevelInfo bool + LogLevelWarn bool + LogLevelError bool + LogLevelFatal bool + PrintVersion bool SingleFileType string Labels map[string]string @@ -107,7 +110,7 @@ func LoadAcquisition(cConfig *csconfig.Config) error { dataSources, err = acquisition.LoadAcquisitionFromDSN(flags.OneShotDSN, flags.Labels, flags.Transform) if err != nil { - return errors.Wrapf(err, "failed to configure datasource for %s", flags.OneShotDSN) + return fmt.Errorf("failed to configure datasource for %s: %w", flags.OneShotDSN, err) } } else { dataSources, err = acquisition.LoadAcquisitionFromFile(cConfig.Crowdsec) @@ -116,6 +119,10 @@ func LoadAcquisition(cConfig *csconfig.Config) error { } } + if len(dataSources) == 0 { + return fmt.Errorf("no datasource enabled") + } + return nil } @@ -140,11 +147,14 @@ func (l labelsMap) Set(label string) error { func (f *Flags) Parse() { flag.StringVar(&f.ConfigFile, "c", csconfig.DefaultConfigPath("config.yaml"), "configuration file") - flag.BoolVar(&f.TraceLevel, "trace", false, "VERY verbose") - flag.BoolVar(&f.DebugLevel, "debug", false, "print debug-level on stderr") - flag.BoolVar(&f.InfoLevel, "info", false, "print info-level on stderr") - flag.BoolVar(&f.WarnLevel, "warning", false, "print warning-level on stderr") - flag.BoolVar(&f.ErrorLevel, "error", false, "print error-level on stderr") + + flag.BoolVar(&f.LogLevelTrace, "trace", false, "set log level to 'trace' (VERY verbose)") + flag.BoolVar(&f.LogLevelDebug, "debug", false, "set log level to 'debug'") + flag.BoolVar(&f.LogLevelInfo, "info", false, "set log level to 'info'") + flag.BoolVar(&f.LogLevelWarn, "warning", false, "set log level to 'warning'") + flag.BoolVar(&f.LogLevelError, "error", false, "set log level to 'error'") + flag.BoolVar(&f.LogLevelFatal, "fatal", false, "set log level to 'fatal'") + flag.BoolVar(&f.PrintVersion, "version", false, "display version") flag.StringVar(&f.OneShotDSN, "dsn", "", "Process a single data source in time-machine") flag.StringVar(&f.Transform, "transform", "", "expr to apply on the event after acquisition") @@ -172,16 +182,18 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level { // override from flags switch { - case f.TraceLevel: + case f.LogLevelTrace: ret = log.TraceLevel - case f.DebugLevel: + case f.LogLevelDebug: ret = log.DebugLevel - case f.InfoLevel: + case f.LogLevelInfo: ret = log.InfoLevel - case f.WarnLevel: + case f.LogLevelWarn: ret = log.WarnLevel - case f.ErrorLevel: + case f.LogLevelError: ret = log.ErrorLevel + case f.LogLevelFatal: + ret = log.FatalLevel default: } diff --git a/cmd/crowdsec/output.go b/cmd/crowdsec/output.go index 17cc99827..67489f459 100644 --- a/cmd/crowdsec/output.go +++ b/cmd/crowdsec/output.go @@ -7,6 +7,10 @@ import ( "sync" "time" + "github.com/go-openapi/strfmt" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/apiclient" @@ -16,9 +20,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" ) func dedupAlerts(alerts []types.RuntimeAlert) ([]*models.Alert, error) { @@ -50,11 +51,11 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error alertsToPush, err := dedupAlerts(alerts) if err != nil { - return errors.Wrap(err, "failed to transform alerts for api") + return fmt.Errorf("failed to transform alerts for api: %w", err) } _, _, err = client.Alerts.Add(ctx, alertsToPush) if err != nil { - return errors.Wrap(err, "failed sending alert to LAPI") + return fmt.Errorf("failed sending alert to LAPI: %w", err) } return nil } @@ -104,11 +105,11 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky Scenarios: scenarios, }) if err != nil { - return errors.Wrapf(err, "authenticate watcher (%s)", apiConfig.Login) + return fmt.Errorf("authenticate watcher (%s): %w", apiConfig.Login, err) } if err := Client.GetClient().Transport.(*apiclient.JWTTransport).Expiration.UnmarshalText([]byte(authResp.Expire)); err != nil { - return errors.Wrap(err, "unable to parse jwt expiration") + return fmt.Errorf("unable to parse jwt expiration: %w", err) } Client.GetClient().Transport.(*apiclient.JWTTransport).Token = authResp.Token diff --git a/cmd/crowdsec/pour.go b/cmd/crowdsec/pour.go index adb072376..3f717e397 100644 --- a/cmd/crowdsec/pour.go +++ b/cmd/crowdsec/pour.go @@ -12,9 +12,7 @@ import ( ) func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *leaky.Buckets, cConfig *csconfig.Config) error { - var ( - count int - ) + count := 0 for { //bucket is now ready select { diff --git a/cmd/crowdsec/run_in_svc_windows.go b/cmd/crowdsec/run_in_svc_windows.go index c51d24147..d63a587ac 100644 --- a/cmd/crowdsec/run_in_svc_windows.go +++ b/cmd/crowdsec/run_in_svc_windows.go @@ -3,7 +3,6 @@ package main import ( "fmt" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/sys/windows/svc" @@ -22,7 +21,7 @@ func StartRunSvc() error { isRunninginService, err := svc.IsWindowsService() if err != nil { - return errors.Wrap(err, "failed to determine if we are running in windows service mode") + return fmt.Errorf("failed to determine if we are running in windows service mode: %w", err) } if isRunninginService { return runService(svcName) @@ -31,22 +30,22 @@ func StartRunSvc() error { if flags.WinSvc == "Install" { err = installService(svcName, svcDescription) if err != nil { - return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName) + return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err) } } else if flags.WinSvc == "Remove" { err = removeService(svcName) if err != nil { - return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName) + return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err) } } else if flags.WinSvc == "Start" { err = startService(svcName) if err != nil { - return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName) + return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err) } } else if flags.WinSvc == "Stop" { err = controlService(svcName, svc.Stop, svc.Stopped) if err != nil { - return errors.Wrapf(err, "failed to %s %s", flags.WinSvc, svcName) + return fmt.Errorf("failed to %s %s: %w", flags.WinSvc, svcName, err) } } else if flags.WinSvc == "" { return WindowsRun() @@ -66,7 +65,7 @@ func WindowsRun() error { if err != nil { return err } - // Configure logging + log.Infof("Crowdsec %s", version.String()) apiReady := make(chan bool, 1) diff --git a/cmd/crowdsec/serve.go b/cmd/crowdsec/serve.go index 5e2e8b720..5d365b410 100644 --- a/cmd/crowdsec/serve.go +++ b/cmd/crowdsec/serve.go @@ -1,16 +1,16 @@ package main import ( + "fmt" "os" "os/signal" "syscall" "time" - "github.com/coreos/go-systemd/v22/daemon" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/csdaemon" "github.com/crowdsecurity/go-cs-lib/pkg/trace" "github.com/crowdsecurity/crowdsec/pkg/csconfig" @@ -68,7 +68,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) { } apiServer, err := initAPIServer(cConfig) if err != nil { - return nil, errors.Wrap(err, "unable to init api server") + return nil, fmt.Errorf("unable to init api server: %w", err) } apiReady := make(chan bool, 1) @@ -78,7 +78,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) { if !cConfig.DisableAgent { csParsers, err := initCrowdsec(cConfig) if err != nil { - return nil, errors.Wrap(err, "unable to init crowdsec") + return nil, fmt.Errorf("unable to init crowdsec: %w", err) } // restore bucket state @@ -180,13 +180,13 @@ func shutdownCrowdsec() error { func shutdown(sig os.Signal, cConfig *csconfig.Config) error { if !cConfig.DisableAgent { if err := shutdownCrowdsec(); err != nil { - return errors.Wrap(err, "failed to shut down crowdsec") + return fmt.Errorf("failed to shut down crowdsec: %w", err) } } if !cConfig.DisableAPI { if err := shutdownAPI(); err != nil { - return errors.Wrap(err, "failed to shut down api routines") + return fmt.Errorf("failed to shut down api routines: %w", err) } } @@ -238,13 +238,13 @@ func HandleSignals(cConfig *csconfig.Config) error { log.Warning("SIGHUP received, reloading") if err = shutdown(s, cConfig); err != nil { - exitChan <- errors.Wrap(err, "failed shutdown") + exitChan <- fmt.Errorf("failed shutdown: %w", err) break Loop } if newConfig, err = reloadHandler(s); err != nil { - exitChan <- errors.Wrap(err, "reload handler failure") + exitChan <- fmt.Errorf("reload handler failure: %w", err) break Loop } @@ -256,7 +256,7 @@ func HandleSignals(cConfig *csconfig.Config) error { case os.Interrupt, syscall.SIGTERM: log.Warning("SIGTERM received, shutting down") if err = shutdown(s, cConfig); err != nil { - exitChan <- errors.Wrap(err, "failed shutdown") + exitChan <- fmt.Errorf("failed shutdown: %w", err) break Loop } @@ -284,17 +284,17 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e if cConfig.API.Server != nil && cConfig.API.Server.DbConfig != nil { dbClient, err := database.NewClient(cConfig.API.Server.DbConfig) if err != nil { - return errors.Wrap(err, "failed to get database client") + return fmt.Errorf("failed to get database client: %w", err) } err = exprhelpers.Init(dbClient) if err != nil { - return errors.Wrap(err, "failed to init expr helpers") + return fmt.Errorf("failed to init expr helpers: %w", err) } } else { err := exprhelpers.Init(nil) if err != nil { - return errors.Wrap(err, "failed to init expr helpers") + return fmt.Errorf("failed to init expr helpers: %w", err) } log.Warningln("Exprhelpers loaded without database client.") @@ -303,7 +303,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e if cConfig.API.CTI != nil && *cConfig.API.CTI.Enabled { log.Infof("Crowdsec CTI helper enabled") if err := exprhelpers.InitCrowdsecCTI(cConfig.API.CTI.Key, cConfig.API.CTI.CacheTimeout, cConfig.API.CTI.CacheSize, cConfig.API.CTI.LogLevel); err != nil { - return errors.Wrap(err, "failed to init crowdsec cti") + return fmt.Errorf("failed to init crowdsec cti: %w", err) } } @@ -319,7 +319,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e apiServer, err := initAPIServer(cConfig) if err != nil { - return errors.Wrap(err, "api server init") + return fmt.Errorf("api server init: %w", err) } if !flags.TestMode { @@ -332,7 +332,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e if !cConfig.DisableAgent { csParsers, err := initCrowdsec(cConfig) if err != nil { - return errors.Wrap(err, "crowdsec init") + return fmt.Errorf("crowdsec init: %w", err) } // if it's just linting, we're done @@ -350,10 +350,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e } if cConfig.Common != nil && cConfig.Common.Daemonize { - sent, err := daemon.SdNotify(false, daemon.SdNotifyReady) - if !sent || err != nil { - log.Errorf("Failed to notify(sent: %v): %v", sent, err) - } + csdaemon.NotifySystemd(log.StandardLogger()) // wait for signals return HandleSignals(cConfig) } diff --git a/cmd/crowdsec/win_service.go b/cmd/crowdsec/win_service.go index d0e80c58a..ab9ecc815 100644 --- a/cmd/crowdsec/win_service.go +++ b/cmd/crowdsec/win_service.go @@ -8,10 +8,10 @@ package main import ( + "fmt" "syscall" "time" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/sys/windows" "golang.org/x/sys/windows/svc" @@ -106,7 +106,7 @@ func runService(name string) error { winsvc := crowdsec_winservice{config: cConfig} if err := svc.Run(name, &winsvc); err != nil { - return errors.Wrapf(err, "%s service failed", name) + return fmt.Errorf("%s service failed: %w", name, err) } log.Infof("%s service stopped", name) diff --git a/debian/control b/debian/control index d06d38884..4673284e7 100644 --- a/debian/control +++ b/debian/control @@ -1,6 +1,8 @@ Source: crowdsec Maintainer: Crowdsec Team -Build-Depends: debhelper, bash, git +Build-Depends: debhelper, bash +Section: admin +Priority: optional Package: crowdsec Architecture: any diff --git a/debian/rules b/debian/rules index 6683e5443..e6202a6f7 100755 --- a/debian/rules +++ b/debian/rules @@ -1,6 +1,6 @@ #!/usr/bin/make -f -export DEB_VERSION=$(shell dpkg-parsechangelog | egrep '^Version:' | cut -f 2 -d ' ') +export DEB_VERSION=$(shell dpkg-parsechangelog | grep -E '^Version:' | cut -f 2 -d ' ') export BUILD_VERSION=v${DEB_VERSION}-debian-pragmatic export GO111MODULE=on @@ -11,12 +11,10 @@ override_dh_auto_clean: override_dh_auto_test: override_dh_auto_build: override_dh_auto_install: -# mkdir /tmp/go -# echo $(go version) -# echo $($GOCMD version) -# cd cmd/crowdsec && GOROOT=/tmp/go GO111MODULE=on $(GOBUILD) $(LD_OPTS) -o $(CROWDSEC_BIN) -v && cd .. -# cd cmd/crowdsec-cli && GOROOT=/tmp/go GO111MODULE=on $(GOBUILD) $(LD_OPTS) -o cscli -v && cd .. - make build + + # just use the prebuilt binaries, otherwise: + # make build BUILD_RE_WASM=0 BUILD_STATIC=1 + mkdir -p debian/crowdsec/usr/bin mkdir -p debian/crowdsec/etc/crowdsec mkdir -p debian/crowdsec/usr/share/crowdsec diff --git a/debian/templates b/debian/templates index c07ef8446..c6998eb85 100644 --- a/debian/templates +++ b/debian/templates @@ -17,7 +17,7 @@ Description: Address of the local API server Template: crowdsec/capi Type: boolean Default: true -Description: Do you want to the centralized remote API server ? +Description: Do you want to use the centralized remote API server ? To share information with other crowdsec you can register to the centralized remote API server. . If you don't know what to do, answer yes. diff --git a/docker/docker_start.sh b/docker/docker_start.sh index 8ec449103..21b42dcb0 100755 --- a/docker/docker_start.sh +++ b/docker/docker_start.sh @@ -56,7 +56,7 @@ conf_get() { if [ $# -ge 2 ]; then yq e "$1" "$2" else - yq e "$1" "$CONFIG_FILE" + cscli config show-yaml | yq e "$1" fi } diff --git a/docker/test/tests/test_hub_collections.py b/docker/test/tests/test_hub_collections.py index 81567954b..b890bebb9 100644 --- a/docker/test/tests/test_hub_collections.py +++ b/docker/test/tests/test_hub_collections.py @@ -6,11 +6,8 @@ Test collection management from http import HTTPStatus import json -import os -import pwd import pytest -import yaml pytestmark = pytest.mark.docker @@ -85,12 +82,7 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor): 'COLLECTIONS': f'{coll}' } - hub = tmp_path_factory.mktemp("hub") - volumes = { - hub: {'bind': '/etc/crowdsec/hub', 'mode': 'rw'} - } - - with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs: + with crowdsec(flavor=flavor, environment=env) as cs: cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK) res = cs.cont.exec_run('cscli collections list -o json') assert res.exit_code == 0 @@ -102,25 +94,13 @@ def test_taint_bubble_up(crowdsec, tmp_path_factory, flavor): f'*Enabled collections : {coll}*', ]) - # change file permissions to allow edit - current_uid = pwd.getpwuid(os.getuid()).pw_uid - res = cs.cont.exec_run(f'chown -R {current_uid} /etc/crowdsec/hub') + scenario = 'crowdsecurity/http-crawl-non_statics' + + # the description won't be read back, it's from the index + yq_command = f"yq -e -i '.description=\"tainted\"' /etc/crowdsec/hub/scenarios/{scenario}.yaml" + res = cs.cont.exec_run(yq_command) assert res.exit_code == 0 - scenario = 'crowdsecurity/http-crawl-non_statics' - scenario_file = hub / f'scenarios/{scenario}.yaml' - - with open(scenario_file) as f: - yml = yaml.safe_load(f) - - yml['description'] += ' (tainted)' - # won't be able to read it back because description is taken from the index - - with open(scenario_file, 'w') as f: - yaml.dump(yml, f) - - with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs: - cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK) res = cs.cont.exec_run(f'cscli scenarios inspect {scenario} -o json') assert res.exit_code == 0 j = json.loads(res.output) diff --git a/go.mod b/go.mod index d64e1c075..f547709e8 100644 --- a/go.mod +++ b/go.mod @@ -5,14 +5,22 @@ go 1.20 require ( entgo.io/ent v0.11.3 github.com/AlecAivazis/survey/v2 v2.2.7 - github.com/Microsoft/go-winio v0.5.2 // indirect + github.com/Masterminds/semver/v3 v3.1.1 + github.com/Masterminds/sprig/v3 v3.2.2 github.com/alexliesenfeld/health v0.5.1 github.com/antonmedv/expr v1.12.5 github.com/appleboy/gin-jwt/v2 v2.8.0 + github.com/aquasecurity/table v1.8.0 + github.com/aws/aws-lambda-go v1.38.0 github.com/aws/aws-sdk-go v1.42.25 + github.com/beevik/etree v1.1.0 + github.com/blackfireio/osinfo v1.0.3 + github.com/bluele/gcache v0.0.2 github.com/buger/jsonparser v1.1.1 github.com/c-robinson/iplib v1.0.3 + github.com/cespare/xxhash/v2 v2.1.2 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 + github.com/crowdsecurity/go-cs-lib v0.0.2 github.com/crowdsecurity/grokky v0.2.1 github.com/crowdsecurity/machineid v1.0.2 github.com/davecgh/go-spew v1.1.1 @@ -20,7 +28,7 @@ require ( github.com/docker/docker v20.10.24+incompatible github.com/docker/go-connections v0.4.0 github.com/enescakir/emoji v1.0.0 - github.com/fatih/color v1.13.0 + github.com/fatih/color v1.15.0 github.com/fsnotify/fsnotify v1.6.0 github.com/gin-gonic/gin v1.7.7 github.com/go-co-op/gocron v1.17.0 @@ -29,65 +37,50 @@ require ( github.com/go-openapi/swag v0.19.14 github.com/go-openapi/validate v0.20.0 github.com/go-sql-driver/mysql v1.6.0 + github.com/goccy/go-yaml v1.9.7 + github.com/gofrs/uuid v4.0.0+incompatible + github.com/golang-jwt/jwt/v4 v4.2.0 github.com/google/go-querystring v1.0.0 github.com/google/uuid v1.3.0 + github.com/google/winops v0.0.0-20211216095627-f0e86eb1453b github.com/goombaio/namegenerator v0.0.0-20181006234301-989e774b106e github.com/hashicorp/go-hclog v1.0.0 github.com/hashicorp/go-plugin v1.4.2 github.com/hashicorp/go-version v1.2.1 + github.com/ivanpirog/coloredcobra v1.0.1 github.com/jackc/pgx/v4 v4.14.1 github.com/jarcoal/httpmock v1.1.0 github.com/jszwec/csvutil v1.5.1 + github.com/lithammer/dedent v1.1.0 + github.com/mattn/go-isatty v0.0.17 github.com/mattn/go-sqlite3 v1.14.16 github.com/mohae/deepcopy v0.0.0-20170929034955-c48cc78d4826 github.com/nxadm/tail v1.4.8 github.com/oschwald/geoip2-golang v1.4.0 github.com/oschwald/maxminddb-golang v1.8.0 + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 github.com/pkg/errors v0.9.1 github.com/prometheus/client_golang v1.14.0 github.com/prometheus/client_model v0.3.0 github.com/prometheus/prom2json v1.3.0 github.com/r3labs/diff/v2 v2.14.1 + github.com/segmentio/kafka-go v0.4.34 + github.com/shirou/gopsutil/v3 v3.23.5 github.com/sirupsen/logrus v1.9.2 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.3 + github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c + github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 + github.com/wasilibs/go-re2 v0.2.1 golang.org/x/crypto v0.1.0 - golang.org/x/mod v0.8.0 + golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 + golang.org/x/mod v0.11.0 + golang.org/x/sys v0.9.0 google.golang.org/grpc v1.47.0 google.golang.org/protobuf v1.28.1 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637 gopkg.in/yaml.v2 v2.4.0 - gotest.tools/v3 v3.0.3 -) - -require ( - github.com/Masterminds/semver v1.5.0 - github.com/Masterminds/sprig/v3 v3.2.2 - github.com/aquasecurity/table v1.8.0 - github.com/aws/aws-lambda-go v1.38.0 - github.com/beevik/etree v1.1.0 - github.com/blackfireio/osinfo v1.0.3 - github.com/bluele/gcache v0.0.2 - github.com/cespare/xxhash/v2 v2.1.2 - github.com/corazawaf/coraza/v3 v3.0.0-00010101000000-000000000000 - github.com/coreos/go-systemd/v22 v22.5.0 - github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd - github.com/goccy/go-yaml v1.9.7 - github.com/gofrs/uuid v4.0.0+incompatible - github.com/golang-jwt/jwt/v4 v4.2.0 - github.com/google/winops v0.0.0-20211216095627-f0e86eb1453b - github.com/ivanpirog/coloredcobra v1.0.1 - github.com/lithammer/dedent v1.1.0 - github.com/mattn/go-isatty v0.0.14 - github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 - github.com/segmentio/kafka-go v0.4.34 - github.com/shirou/gopsutil/v3 v3.22.12 - github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c - github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 - github.com/wasilibs/go-re2 v0.2.1 - golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc - golang.org/x/sys v0.7.0 gopkg.in/yaml.v3 v3.0.1 k8s.io/apiserver v0.22.5 ) @@ -95,7 +88,7 @@ require ( require ( ariga.io/atlas v0.7.2-0.20220927111110-867ee0cca56a // indirect github.com/Masterminds/goutils v1.1.1 // indirect - github.com/Masterminds/semver/v3 v3.1.1 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/agext/levenshtein v1.2.1 // indirect @@ -103,7 +96,7 @@ require ( github.com/apparentlymart/go-textseg/v13 v13.0.0 // indirect github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/corazawaf/libinjection-go v0.1.2 // indirect + github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-units v0.4.0 // indirect @@ -147,7 +140,7 @@ require ( github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magefile/mage v1.14.0 // indirect github.com/mailru/easyjson v0.7.6 // indirect - github.com/mattn/go-colorable v0.1.12 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-runewidth v0.0.13 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect @@ -162,7 +155,6 @@ require ( github.com/oklog/run v1.0.0 // indirect github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.0.3-0.20211202183452-c5a74bcca799 // indirect - github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 // indirect github.com/pierrec/lz4/v4 v4.1.15 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect @@ -171,24 +163,24 @@ require ( github.com/rivo/uniseg v0.2.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect github.com/shopspring/decimal v1.2.0 // indirect github.com/spf13/cast v1.3.1 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/tetratelabs/wazero v1.0.0-rc.2 // indirect - github.com/tidwall/gjson v1.14.4 // indirect - github.com/tidwall/match v1.1.1 // indirect - github.com/tidwall/pretty v1.2.1 // indirect + github.com/tidwall/gjson v1.13.0 // indirect github.com/tklauser/go-sysconf v0.3.11 // indirect github.com/tklauser/numcpus v0.6.0 // indirect github.com/ugorji/go/codec v1.2.6 // indirect - github.com/vmihailenco/msgpack v4.0.4+incompatible // indirect - github.com/yusufpapurcu/wmi v1.2.2 // indirect + github.com/vmihailenco/msgpack v4.0 .4+incompatible // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect github.com/zclconf/go-cty v1.8.0 // indirect - go.mongodb.org/mongo-driver v1.9.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect - golang.org/x/term v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect + go.mongodb.org/mongo-driver v1.9.4 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/sync v0.1.0 // indirect + golang.org/x/term v0.5.0 // indirect + golang.org/x/text v0.7.0 // indirect + golang.org/x/tools v0.6.0 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect @@ -198,11 +190,8 @@ require ( k8s.io/apimachinery v0.25.2 // indirect k8s.io/klog/v2 v2.70.1 // indirect k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed // indirect - rsc.io/binaryregexp v0.2.0 // indirect sigs.k8s.io/json v0.0.0-20220713155537-f223a00ba0e2 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.2.3 // indirect ) replace golang.org/x/time/rate => github.com/crowdsecurity/crowdsec/pkg/time/rate v0.0.0 - -replace github.com/corazawaf/coraza/v3 => ./coraza diff --git a/go.sum b/go.sum index 8d4506758..41bff5d47 100644 --- a/go.sum +++ b/go.sum @@ -55,14 +55,12 @@ github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/Masterminds/goutils v1.1.1 h1:5nUrii3FMTL5diU80unEVvNevw1nH4+ZV4DSLVJLSYI= github.com/Masterminds/goutils v1.1.1/go.mod h1:8cTjp+g8YejhMuvIA5y2vz3BpJxksy863GQaJW2MFNU= -github.com/Masterminds/semver v1.5.0 h1:H65muMkzWKEuNDnfl9d70GUjFniHKHRbFPGBuZ3QEww= -github.com/Masterminds/semver v1.5.0/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/sprig/v3 v3.2.2 h1:17jRggJu518dr3QaafizSXOjKYp94wKfABxUmyxvxX8= github.com/Masterminds/sprig/v3 v3.2.2/go.mod h1:UoaO7Yp8KlPnJIYWTFkMaqPUYKTfGFPhxNuwnnxkKlk= -github.com/Microsoft/go-winio v0.5.2 h1:a9IhgEQBCUEk6QCdml9CiJGhAws+YwffDHEMp1VMrpA= -github.com/Microsoft/go-winio v0.5.2/go.mod h1:WpS1mjBmmwHBEWmogvA2mj8546UReBk4v8QkMxJ6pZY= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8 h1:xzYJEypr/85nBpB11F9br+3HUrpgb+fcm5iADzXXYEw= @@ -151,8 +149,6 @@ github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMe github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/corazawaf/libinjection-go v0.1.2 h1:oeiV9pc5rvJ+2oqOqXEAMJousPpGiup6f7Y3nZj5GoM= -github.com/corazawaf/libinjection-go v0.1.2/go.mod h1:OP4TM7xdJ2skyXqNX1AN1wN5nNZEmJNuWbNPOItn7aw= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= @@ -174,8 +170,8 @@ github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= -github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd h1:Y70ceDKAKYFXTnxEjXuBDSh07umvDhbX3PCCYhdtsZ0= -github.com/crowdsecurity/go-cs-lib v0.0.0-20230531105801-4c1535c2b3bd/go.mod h1:9JJLSpGj1ZXnROV3xAcJvS/HTaUvuA8K3gGOpO4tfVc= +github.com/crowdsecurity/go-cs-lib v0.0.2 h1:+Tjmf/IclOXNzU9sxKVQvUl9CkMfbM60xQ0zA05NWps= +github.com/crowdsecurity/go-cs-lib v0.0.2/go.mod h1:iznTJ19qLTYdZBcRb5RVDlcUdSlayBCivBkWsXlOY3g= github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4= github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM= github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc= @@ -213,12 +209,12 @@ github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7 github.com/evanphx/json-patch v4.11.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.10.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= -github.com/foxcpp/go-mockdns v1.0.0 h1:7jBqxd3WDWwi/6WhDvacvH1XsN3rOLXyHM1uhvIx6FI= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= @@ -661,16 +657,18 @@ github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= @@ -683,7 +681,6 @@ github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyex github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= -github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/copystructure v1.0.0/go.mod h1:SNtv71yrdKgLRyLFxmLdkAbkKEFWgYaq1OVrnRcwhnw= github.com/mitchellh/copystructure v1.2.0 h1:vpKXTN4ewci03Vljg/q9QvCGUDttBOGBIa15WveJJGw= @@ -758,8 +755,6 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.4.0/go.mod h1:PN7xzY2wHTK0K9p34ErDQMlFxa51Fk0OUruD3k1mMwo= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9 h1:lL+y4Xv20pVlCGyLzNHRC0I0rIHhIL1lTvHizoS/dU8= -github.com/petar-dambovaliev/aho-corasick v0.0.0-20211021192214-5ab2d9280aa9/go.mod h1:EHPiTAKtiFmrMldLUNswFwfZ2eJIYBHktdaUTZxYWRw= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0= github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= @@ -836,8 +831,12 @@ github.com/segmentio/kafka-go v0.4.34 h1:Dm6YlLMiVSiwwav20KY0AoY63s661FXevwJ3CVH github.com/segmentio/kafka-go v0.4.34/go.mod h1:GAjxBQJdQMB5zfNA21AhpaqOB2Mu+w3De4ni3Gbm8y0= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= -github.com/shirou/gopsutil/v3 v3.22.12 h1:oG0ns6poeUSxf78JtOsfygNWuEHYYz8hnnNg7P04TJs= -github.com/shirou/gopsutil/v3 v3.22.12/go.mod h1:Xd7P1kwZcp5VW52+9XsirIKd/BROzbb2wdX3Kqlz9uI= +github.com/shirou/gopsutil/v3 v3.23.5 h1:5SgDCeQ0KW0S4N0znjeM/eFHXXOKyv2dVNgRq/c9P6Y= +github.com/shirou/gopsutil/v3 v3.23.5/go.mod h1:Ng3Maa27Q2KARVJ0SPZF5NdrQSC3XHKP8IIWrHgMeLY= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= @@ -888,7 +887,6 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= @@ -897,14 +895,13 @@ github.com/tetratelabs/wazero v1.0.0-rc.2/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+Gk github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c h1:HelZ2kAFadG0La9d+4htN4HzQ68Bm2iM9qKMSMES6xg= github.com/texttheater/golang-levenshtein/levenshtein v0.0.0-20200805054039-cae8b0eaed6c/go.mod h1:JlzghshsemAMDGZLytTFY8C1JQxQPhnatWqNwUXjggo= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= -github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.13.0 h1:3TFY9yxOQShrvmjdM76K+jc66zJeT6D3/VFFYCGQf7M= +github.com/tidwall/gjson v1.13.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= -github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= -github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tklauser/go-sysconf v0.3.11 h1:89WgdJhk5SNwJfu+GKyYveZ4IaJ7xAkecBo+KdJV0CM= github.com/tklauser/go-sysconf v0.3.11/go.mod h1:GqXfhXY3kiPa0nAXPDIQIWzJbMCB7AmcWpGR8lSZfqI= github.com/tklauser/numcpus v0.6.0 h1:kebhY2Qt+3U6RNK7UqpYNA+tJ23IBEGKkB7JQBfDYms= @@ -943,8 +940,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= -github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= -github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zclconf/go-cty v1.8.0 h1:s4AvqaeQzJIu3ndv4gVIhplVD0krU+bgrcLSVUnaWuA= github.com/zclconf/go-cty v1.8.0/go.mod h1:vVKLxnk3puL4qRAv72AO+W99LUD4da90g3uUAzyuvAk= github.com/zenazn/goji v0.9.0/go.mod h1:7S9M489iMyHBNxwZnk9/EHS098H4/F6TATF2mIxtB1Q= @@ -963,8 +960,8 @@ go.mongodb.org/mongo-driver v1.3.0/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS go.mongodb.org/mongo-driver v1.3.4/go.mod h1:MSWZXKOynuguX+JSvwP8i+58jYCXxbia8HS3gZBapIE= go.mongodb.org/mongo-driver v1.4.3/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= go.mongodb.org/mongo-driver v1.4.4/go.mod h1:WcMNYLx/IlOxLe6JRJiv2uXuCz6zBLndR4SoGjYphSc= -go.mongodb.org/mongo-driver v1.9.0 h1:f3aLGJvQmBl8d9S40IL+jEyBC6hfLPbJjv9t5hEM9ck= -go.mongodb.org/mongo-driver v1.9.0/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= +go.mongodb.org/mongo-driver v1.9.4 h1:qXWlnK2WCOWSxJ/Hm3XyYOGKv3ujA2btBsCyuIFvQjc= +go.mongodb.org/mongo-driver v1.9.4/go.mod h1:0sQWfOeY63QTntERDJJ/0SuKK0T1uVSgKCuAROlKEPY= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -1031,8 +1028,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc h1:mCRnTeVUjcrhlRmO0VK8a6k6Rrf6TF9htwo2pJVSjIU= -golang.org/x/exp v0.0.0-20230515195305-f3d0a9c9a5cc/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc= +golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1055,8 +1052,8 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.11.0 h1:bUO06HqtnRcc/7l71XBe4WcqTZ+3AH1J59zWDDwLKgU= +golang.org/x/mod v0.11.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180530234432-1e491301e022/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1107,8 +1104,8 @@ golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qx golang.org/x/net v0.0.0-20220127200216-cd36cc0744dd/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220225172249-27dd8689420f/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220706163947-c90051bbdb60/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.9.0 h1:aWJ/m6xSmxWBx+V0XRHTlrYrPG56jKsLdTFmsSsCzOM= -golang.org/x/net v0.9.0/go.mod h1:d48xBJpPfHeWQsugry2m+kC02ZBRGRgulfHnEXEuWns= +golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -1128,8 +1125,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= -golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1201,23 +1198,23 @@ golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220406163625-3f8b81556e12/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.7.0 h1:BEvjmm5fURWqcfbSKTdpkDXYBrUS1c0m8agp14W48vQ= -golang.org/x/term v0.7.0/go.mod h1:P32HKFT3hSsZrRxla30E9HqToFYAQPCMs/zFMBUFqPY= +golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -1227,8 +1224,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1297,6 +1294,7 @@ golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190410155217-1f06c39b4373/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190513163551-3ee3066db522/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1467,7 +1465,6 @@ k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c/go.mod h1:vHXdDvt9+2spS2R k8s.io/utils v0.0.0-20210819203725-bdf08cb9a70a/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed h1:jAne/RjBTyawwAy0utX5eqigAwz/lQhTmy+Hr/Cpue4= k8s.io/utils v0.0.0-20220728103510-ee6ede2d64ed/go.mod h1:jPW/WVKK9YHAvNhRxK0md/EJ228hCsBRufyofKtW8HA= -rsc.io/binaryregexp v0.2.0 h1:HfqmD5MEmC0zvwBuF187nq9mdnXjXsSivRiXN7SmRkE= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/mk/__gmsl b/mk/__gmsl new file mode 100644 index 000000000..6cd4a3090 --- /dev/null +++ b/mk/__gmsl @@ -0,0 +1,969 @@ +# ---------------------------------------------------------------------------- +# +# GNU Make Standard Library (GMSL) +# +# A library of functions to be used with GNU Make's $(call) that +# provides functionality not available in standard GNU Make. +# +# Copyright (c) 2005-2022 John Graham-Cumming +# +# This file is part of GMSL +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# Neither the name of the John Graham-Cumming nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# ---------------------------------------------------------------------------- + +# This is the GNU Make Standard Library version number as a list with +# three items: major, minor, revision + +gmsl_version := 1 2 0 + +__gmsl_name := GNU Make Standard Library + +# Used to output warnings and error from the library, it's possible to +# disable any warnings or errors by overriding these definitions +# manually or by setting GMSL_NO_WARNINGS or GMSL_NO_ERRORS + +ifdef GMSL_NO_WARNINGS +__gmsl_warning := +else +__gmsl_warning = $(if $1,$(warning $(__gmsl_name): $1)) +endif + +ifdef GMSL_NO_ERRORS +__gmsl_error := +else + __gmsl_error = $(if $1,$(error $(__gmsl_name): $1)) +endif + +# If GMSL_TRACE is enabled then calls to the library functions are +# traced to stdout using warning messages with their arguments + +ifdef GMSL_TRACE +__gmsl_tr1 = $(warning $0('$1')) +__gmsl_tr2 = $(warning $0('$1','$2')) +__gmsl_tr3 = $(warning $0('$1','$2','$3')) +else +__gmsl_tr1 := +__gmsl_tr2 := +__gmsl_tr3 := +endif + +# See if spaces are valid in variable names (this was the case until +# GNU Make 3.82) +ifeq ($(MAKE_VERSION),3.82) +__gmsl_spaced_vars := $(false) +else +__gmsl_spaced_vars := $(true) +endif + +# Figure out whether we have $(eval) or not (GNU Make 3.80 and above) +# if we do not then output a warning message, if we do then some +# functions will be enabled. + +__gmsl_have_eval := $(false) +__gmsl_ignore := $(eval __gmsl_have_eval := $(true)) + +# If this is being run with Electric Cloud's emake then warn that +# their $(eval) support is incomplete in 1.x, 2.x, 3.x, 4.x and 5.0, +# 5.1, 5.2 and 5.3 + +ifdef ECLOUD_BUILD_ID +__gmsl_emake_major := $(word 1,$(subst ., ,$(EMAKE_VERSION))) +__gmsl_emake_minor := $(word 2,$(subst ., ,$(EMAKE_VERSION))) +ifneq ("$(findstring $(__gmsl_emake_major),1 2 3 4)$(findstring $(__gmsl_emake_major)$(__gmsl_emake_minor),50 51 52 53)","") +$(warning You are using a version of Electric Cloud's emake which has incomplete $$(eval) support) +__gmsl_have_eval := $(false) +endif +endif + +# See if we have $(lastword) (GNU Make 3.81 and above) + +__gmsl_have_lastword := $(lastword $(false) $(true)) + +# See if we have native or and and (GNU Make 3.81 and above) + +__or_tt := /$(or $(true),$(true))/$(or $(true),$(false))/$(or $(false),$(true))/$(or $(false),$(false))/ +__and_tt := /$(and $(true),$(true))/$(and $(true),$(false))/$(and $(false),$(true))/$(and $(false),$(false))/ +__gmsl_have_or := $(if $(filter /T/T/T//,$(__or_tt)),$(true),$(false)) +__gmsl_have_and := $(if $(filter /T////,$(__and_tt)),$(true),$(false)) + +ifneq ($(__gmsl_have_eval),$(true)) +$(call __gmsl_warning,Your make version $(MAKE_VERSION) does not support $$$$(eval): some functions disabled) +endif + +__gmsl_dollar := $$ +__gmsl_hash := \# + +# ---------------------------------------------------------------------------- +# ---------------------------------------------------------------------------- +# Function: gmsl_compatible +# Arguments: List containing the desired library version number (maj min rev) +# Returns: $(true) if this version of the library is compatible +# with the requested version number, otherwise $(false) +# ---------------------------------------------------------------------------- +gmsl_compatible = $(strip \ + $(if $(call gt,$(word 1,$1),$(word 1,$(gmsl_version))), \ + $(false), \ + $(if $(call lt,$(word 1,$1),$(word 1,$(gmsl_version))), \ + $(true), \ + $(if $(call gt,$(word 2,$1),$(word 2,$(gmsl_version))), \ + $(false), \ + $(if $(call lt,$(word 2,$1),$(word 2,$(gmsl_version))), \ + $(true), \ + $(call lte,$(word 3,$1),$(word 3,$(gmsl_version)))))))) + +# ########################################################################### +# LOGICAL OPERATORS +# ########################################################################### + +# not is defined in gmsl + +# ---------------------------------------------------------------------------- +# Function: and +# Arguments: Two boolean values +# Returns: Returns $(true) if both of the booleans are true +# ---------------------------------------------------------------------------- +ifneq ($(__gmsl_have_and),$(true)) +and = $(__gmsl_tr2)$(if $1,$(if $2,$(true),$(false)),$(false)) +endif + +# ---------------------------------------------------------------------------- +# Function: or +# Arguments: Two boolean values +# Returns: Returns $(true) if either of the booleans is true +# ---------------------------------------------------------------------------- +ifneq ($(__gmsl_have_or),$(true)) +or = $(__gmsl_tr2)$(if $1$2,$(true),$(false)) +endif + +# ---------------------------------------------------------------------------- +# Function: xor +# Arguments: Two boolean values +# Returns: Returns $(true) if exactly one of the booleans is true +# ---------------------------------------------------------------------------- +xor = $(__gmsl_tr2)$(if $1,$(if $2,$(false),$(true)),$(if $2,$(true),$(false))) + +# ---------------------------------------------------------------------------- +# Function: nand +# Arguments: Two boolean values +# Returns: Returns value of 'not and' +# ---------------------------------------------------------------------------- +nand = $(__gmsl_tr2)$(if $1,$(if $2,$(false),$(true)),$(true)) + +# ---------------------------------------------------------------------------- +# Function: nor +# Arguments: Two boolean values +# Returns: Returns value of 'not or' +# ---------------------------------------------------------------------------- +nor = $(__gmsl_tr2)$(if $1$2,$(false),$(true)) + +# ---------------------------------------------------------------------------- +# Function: xnor +# Arguments: Two boolean values +# Returns: Returns value of 'not xor' +# ---------------------------------------------------------------------------- +xnor =$(__gmsl_tr2)$(if $1,$(if $2,$(true),$(false)),$(if $2,$(false),$(true))) + +# ########################################################################### +# LIST MANIPULATION FUNCTIONS +# ########################################################################### + +# ---------------------------------------------------------------------------- +# Function: first (same as LISP's car, or head) +# Arguments: 1: A list +# Returns: Returns the first element of a list +# ---------------------------------------------------------------------------- +first = $(__gmsl_tr1)$(firstword $1) + +# ---------------------------------------------------------------------------- +# Function: last +# Arguments: 1: A list +# Returns: Returns the last element of a list +# ---------------------------------------------------------------------------- +ifeq ($(__gmsl_have_lastword),$(true)) +last = $(__gmsl_tr1)$(lastword $1) +else +last = $(__gmsl_tr1)$(if $1,$(word $(words $1),$1)) +endif + +# ---------------------------------------------------------------------------- +# Function: rest (same as LISP's cdr, or tail) +# Arguments: 1: A list +# Returns: Returns the list with the first element removed +# ---------------------------------------------------------------------------- +rest = $(__gmsl_tr1)$(wordlist 2,$(words $1),$1) + +# ---------------------------------------------------------------------------- +# Function: chop +# Arguments: 1: A list +# Returns: Returns the list with the last element removed +# ---------------------------------------------------------------------------- +chop = $(__gmsl_tr1)$(wordlist 2,$(words $1),x $1) + +# ---------------------------------------------------------------------------- +# Function: map +# Arguments: 1: Name of function to $(call) for each element of list +# 2: List to iterate over calling the function in 1 +# Returns: The list after calling the function on each element +# ---------------------------------------------------------------------------- +map = $(__gmsl_tr2)$(strip $(foreach a,$2,$(call $1,$a))) + +# ---------------------------------------------------------------------------- +# Function: pairmap +# Arguments: 1: Name of function to $(call) for each pair of elements +# 2: List to iterate over calling the function in 1 +# 3: Second list to iterate over calling the function in 1 +# Returns: The list after calling the function on each pair of elements +# ---------------------------------------------------------------------------- +pairmap = $(strip $(__gmsl_tr3)\ + $(if $2$3,$(call $1,$(call first,$2),$(call first,$3)) \ + $(call pairmap,$1,$(call rest,$2),$(call rest,$3)))) + +# ---------------------------------------------------------------------------- +# Function: leq +# Arguments: 1: A list to compare against... +# 2: ...this list +# Returns: Returns $(true) if the two lists are identical +# ---------------------------------------------------------------------------- +leq = $(__gmsl_tr2)$(strip $(if $(call seq,$(words $1),$(words $2)), \ + $(call __gmsl_list_equal,$1,$2),$(false))) + +__gmsl_list_equal = $(if $(strip $1), \ + $(if $(call seq,$(call first,$1),$(call first,$2)), \ + $(call __gmsl_list_equal, \ + $(call rest,$1), \ + $(call rest,$2)), \ + $(false)), \ + $(true)) + +# ---------------------------------------------------------------------------- +# Function: lne +# Arguments: 1: A list to compare against... +# 2: ...this list +# Returns: Returns $(true) if the two lists are different +# ---------------------------------------------------------------------------- +lne = $(__gmsl_tr2)$(call not,$(call leq,$1,$2)) + +# ---------------------------------------------------------------------------- +# Function: reverse +# Arguments: 1: A list to reverse +# Returns: The list with its elements in reverse order +# ---------------------------------------------------------------------------- +reverse =$(__gmsl_tr1)$(strip $(if $1,$(call reverse,$(call rest,$1)) \ + $(call first,$1))) + +# ---------------------------------------------------------------------------- +# Function: uniq +# Arguments: 1: A list from which to remove repeated elements +# Returns: The list with duplicate elements removed without reordering +# ---------------------------------------------------------------------------- +uniq = $(strip $(__gmsl_tr1) $(if $1,$(firstword $1) \ + $(call uniq,$(filter-out $(firstword $1),$1)))) + +# ---------------------------------------------------------------------------- +# Function: length +# Arguments: 1: A list +# Returns: The number of elements in the list +# ---------------------------------------------------------------------------- +length = $(__gmsl_tr1)$(words $1) + +# ########################################################################### +# STRING MANIPULATION FUNCTIONS +# ########################################################################### + +# Helper function that translates any GNU Make 'true' value (i.e. a +# non-empty string) to our $(true) + +__gmsl_make_bool = $(if $(strip $1),$(true),$(false)) + +# ---------------------------------------------------------------------------- +# Function: seq +# Arguments: 1: A string to compare against... +# 2: ...this string +# Returns: Returns $(true) if the two strings are identical +# ---------------------------------------------------------------------------- +seq = $(__gmsl_tr2)$(if $(subst x$1,,x$2)$(subst x$2,,x$1),$(false),$(true)) + +# ---------------------------------------------------------------------------- +# Function: sne +# Arguments: 1: A string to compare against... +# 2: ...this string +# Returns: Returns $(true) if the two strings are not the same +# ---------------------------------------------------------------------------- +sne = $(__gmsl_tr2)$(call not,$(call seq,$1,$2)) + +# ---------------------------------------------------------------------------- +# Function: split +# Arguments: 1: The character to split on +# 2: A string to split +# Returns: Splits a string into a list separated by spaces at the split +# character in the first argument +# ---------------------------------------------------------------------------- +split = $(__gmsl_tr2)$(strip $(subst $1, ,$2)) + +# ---------------------------------------------------------------------------- +# Function: merge +# Arguments: 1: The character to put between fields +# 2: A list to merge into a string +# Returns: Merges a list into a single string, list elements are separated +# by the character in the first argument +# ---------------------------------------------------------------------------- +merge = $(__gmsl_tr2)$(strip $(if $2, \ + $(if $(call seq,1,$(words $2)), \ + $2,$(call first,$2)$1$(call merge,$1,$(call rest,$2))))) + +ifdef __gmsl_have_eval +# ---------------------------------------------------------------------------- +# Function: tr +# Arguments: 1: The list of characters to translate from +# 2: The list of characters to translate to +# 3: The text to translate +# Returns: Returns the text after translating characters +# ---------------------------------------------------------------------------- +tr = $(strip $(__gmsl_tr3)$(call assert_no_dollar,$0,$1$2$3) \ + $(eval __gmsl_t := $3) \ + $(foreach c, \ + $(join $(addsuffix :,$1),$2), \ + $(eval __gmsl_t := \ + $(subst $(word 1,$(subst :, ,$c)),$(word 2,$(subst :, ,$c)), \ + $(__gmsl_t))))$(__gmsl_t)) + +# Common character classes for use with the tr function. Each of +# these is actually a variable declaration and must be wrapped with +# $() or ${} to be used. + +[A-Z] := A B C D E F G H I J K L M N O P Q R S T U V W X Y Z # +[a-z] := a b c d e f g h i j k l m n o p q r s t u v w x y z # +[0-9] := 0 1 2 3 4 5 6 7 8 9 # +[A-F] := A B C D E F # + +# ---------------------------------------------------------------------------- +# Function: uc +# Arguments: 1: Text to upper case +# Returns: Returns the text in upper case +# ---------------------------------------------------------------------------- +uc = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(call tr,$([a-z]),$([A-Z]),$1) + +# ---------------------------------------------------------------------------- +# Function: lc +# Arguments: 1: Text to lower case +# Returns: Returns the text in lower case +# ---------------------------------------------------------------------------- +lc = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(call tr,$([A-Z]),$([a-z]),$1) + +# ---------------------------------------------------------------------------- +# Function: strlen +# Arguments: 1: A string +# Returns: Returns the length of the string +# ---------------------------------------------------------------------------- + +# This results in __gmsl_tab containing a tab + +__gmsl_tab := # + +__gmsl_characters := A B C D E F G H I J K L M N O P Q R S T U V W X Y Z +__gmsl_characters += a b c d e f g h i j k l m n o p q r s t u v w x y z +__gmsl_characters += 0 1 2 3 4 5 6 7 8 9 +__gmsl_characters += ` ~ ! @ \# $$ % ^ & * ( ) - _ = + +__gmsl_characters += { } [ ] \ : ; ' " < > , . / ? | + +# This results in __gmsl_space containing just a space + +__gmsl_empty := +__gmsl_space := $(__gmsl_empty) $(__gmsl_empty) + +strlen = $(__gmsl_tr1)$(call assert_no_dollar,$0,$1)$(strip $(eval __temp := $(subst $(__gmsl_space),x,$1))$(foreach a,$(__gmsl_characters),$(eval __temp := $$(subst $$a,x,$(__temp))))$(eval __temp := $(subst x,x ,$(__temp)))$(words $(__temp))) + +# This results in __gmsl_newline containing just a newline + +define __gmsl_newline + + +endef + +# ---------------------------------------------------------------------------- +# Function: substr +# Arguments: 1: A string +# 2: Start position (first character is 1) +# 3: End position (inclusive) +# Returns: A substring. +# Note: The string in $1 must not contain a § +# ---------------------------------------------------------------------------- + +substr = $(if $2,$(__gmsl_tr3)$(call assert_no_dollar,$0,$1$2$3)$(strip $(eval __temp := $$(subst $$(__gmsl_space),§ ,$$1))$(foreach a,$(__gmsl_characters),$(eval __temp := $$(subst $$a,$$a$$(__gmsl_space),$(__temp))))$(eval __temp := $(wordlist $2,$3,$(__temp))))$(subst §,$(__gmsl_space),$(subst $(__gmsl_space),,$(__temp)))) + +endif # __gmsl_have_eval + +# ########################################################################### +# SET MANIPULATION FUNCTIONS +# ########################################################################### + +# Sets are represented by sorted, deduplicated lists. To create a set +# from a list use set_create, or start with the empty_set and +# set_insert individual elements + +# This is the empty set +empty_set := + +# ---------------------------------------------------------------------------- +# Function: set_create +# Arguments: 1: A list of set elements +# Returns: Returns the newly created set +# ---------------------------------------------------------------------------- +set_create = $(__gmsl_tr1)$(sort $1) + +# ---------------------------------------------------------------------------- +# Function: set_insert +# Arguments: 1: A single element to add to a set +# 2: A set +# Returns: Returns the set with the element added +# ---------------------------------------------------------------------------- +set_insert = $(__gmsl_tr2)$(sort $1 $2) + +# ---------------------------------------------------------------------------- +# Function: set_remove +# Arguments: 1: A single element to remove from a set +# 2: A set +# Returns: Returns the set with the element removed +# ---------------------------------------------------------------------------- +set_remove = $(__gmsl_tr2)$(filter-out $1,$2) + +# ---------------------------------------------------------------------------- +# Function: set_is_member, set_is_not_member +# Arguments: 1: A single element +# 2: A set +# Returns: (set_is_member) Returns $(true) if the element is in the set +# (set_is_not_member) Returns $(false) if the element is in the set +# ---------------------------------------------------------------------------- +set_is_member = $(__gmsl_tr2)$(if $(filter $1,$2),$(true),$(false)) +set_is_not_member = $(__gmsl_tr2)$(if $(filter $1,$2),$(false),$(true)) + +# ---------------------------------------------------------------------------- +# Function: set_union +# Arguments: 1: A set +# 2: Another set +# Returns: Returns the union of the two sets +# ---------------------------------------------------------------------------- +set_union = $(__gmsl_tr2)$(sort $1 $2) + +# ---------------------------------------------------------------------------- +# Function: set_intersection +# Arguments: 1: A set +# 2: Another set +# Returns: Returns the intersection of the two sets +# ---------------------------------------------------------------------------- +set_intersection = $(__gmsl_tr2)$(filter $1,$2) + +# ---------------------------------------------------------------------------- +# Function: set_is_subset +# Arguments: 1: A set +# 2: Another set +# Returns: Returns $(true) if the first set is a subset of the second +# ---------------------------------------------------------------------------- +set_is_subset = $(__gmsl_tr2)$(call set_equal,$(call set_intersection,$1,$2),$1) + +# ---------------------------------------------------------------------------- +# Function: set_equal +# Arguments: 1: A set +# 2: Another set +# Returns: Returns $(true) if the two sets are identical +# ---------------------------------------------------------------------------- +set_equal = $(__gmsl_tr2)$(call seq,$1,$2) + +# ########################################################################### +# ARITHMETIC LIBRARY +# ########################################################################### + +# Integers a represented by lists with the equivalent number of x's. +# For example the number 4 is x x x x. + +# ---------------------------------------------------------------------------- +# Function: int_decode +# Arguments: 1: A number of x's representation +# Returns: Returns the integer for human consumption that is represented +# by the string of x's +# ---------------------------------------------------------------------------- +int_decode = $(__gmsl_tr1)$(if $1,$(if $(call seq,$(word 1,$1),x),$(words $1),$1),0) + +# ---------------------------------------------------------------------------- +# Function: int_encode +# Arguments: 1: A number in human-readable integer form +# Returns: Returns the integer encoded as a string of x's +# ---------------------------------------------------------------------------- +__int_encode = $(if $1,$(if $(call seq,$(words $(wordlist 1,$1,$2)),$1),$(wordlist 1,$1,$2),$(call __int_encode,$1,$(if $2,$2 $2,x)))) +__strip_leading_zero = $(if $1,$(if $(call seq,$(patsubst 0%,%,$1),$1),$1,$(call __strip_leading_zero,$(patsubst 0%,%,$1))),0) +int_encode = $(__gmsl_tr1)$(call __int_encode,$(call __strip_leading_zero,$1)) + +# The arithmetic library functions come in two forms: one form of each +# function takes integers as arguments and the other form takes the +# encoded form (x's created by a call to int_encode). For example, +# there are two plus functions: +# +# plus Called with integer arguments and returns an integer +# int_plus Called with encoded arguments and returns an encoded result +# +# plus will be slower than int_plus because its arguments and result +# have to be translated between the x's format and integers. If doing +# a complex calculation use the int_* forms with a single encoding of +# inputs and single decoding of the output. For simple calculations +# the direct forms can be used. + +# Helper function used to wrap an int_* function into a function that +# takes a pair of integers, perhaps a function and returns an integer +# result +__gmsl_int_wrap = $(call int_decode,$(call $1,$(call int_encode,$2),$(call int_encode,$3))) +__gmsl_int_wrap1 = $(call int_decode,$(call $1,$(call int_encode,$2))) +__gmsl_int_wrap2 = $(call $1,$(call int_encode,$2),$(call int_encode,$3)) + +# ---------------------------------------------------------------------------- +# Function: int_plus +# Arguments: 1: A number in x's representation +# 2: Another number in x's represntation +# Returns: Returns the sum of the two numbers in x's representation +# ---------------------------------------------------------------------------- +int_plus = $(strip $(__gmsl_tr2)$1 $2) + +# ---------------------------------------------------------------------------- +# Function: plus (wrapped version of int_plus) +# Arguments: 1: An integer +# 2: Another integer +# Returns: Returns the sum of the two integers +# ---------------------------------------------------------------------------- +plus = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_plus,$1,$2) + +# ---------------------------------------------------------------------------- +# Function: int_subtract +# Arguments: 1: A number in x's representation +# 2: Another number in x's represntation +# Returns: Returns the difference of the two numbers in x's representation, +# or outputs an error on a numeric underflow +# ---------------------------------------------------------------------------- +int_subtract = $(strip $(__gmsl_tr2)$(if $(call int_gte,$1,$2), \ + $(filter-out xx,$(join $1,$2)), \ + $(call __gmsl_warning,Subtraction underflow))) + +# ---------------------------------------------------------------------------- +# Function: subtract (wrapped version of int_subtract) +# Arguments: 1: An integer +# 2: Another integer +# Returns: Returns the difference of the two integers, +# or outputs an error on a numeric underflow +# ---------------------------------------------------------------------------- +subtract = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_subtract,$1,$2) + +# ---------------------------------------------------------------------------- +# Function: int_multiply +# Arguments: 1: A number in x's representation +# 2: Another number in x's represntation +# Returns: Returns the product of the two numbers in x's representation +# ---------------------------------------------------------------------------- +int_multiply = $(strip $(__gmsl_tr2)$(foreach a,$1,$2)) + +# ---------------------------------------------------------------------------- +# Function: multiply (wrapped version of int_multiply) +# Arguments: 1: An integer +# 2: Another integer +# Returns: Returns the product of the two integers +# ---------------------------------------------------------------------------- +multiply = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_multiply,$1,$2) + +# ---------------------------------------------------------------------------- +# Function: int_divide +# Arguments: 1: A number in x's representation +# 2: Another number in x's represntation +# Returns: Returns the result of integer division of argument 1 divided +# by argument 2 in x's representation +# ---------------------------------------------------------------------------- +int_divide = $(__gmsl_tr2)$(strip $(if $1,$(if $2, \ + $(subst M,x,$(filter-out x,$(subst $2,M,$1))), \ + $(call __gmsl_error,Division by zero)))) + +# ---------------------------------------------------------------------------- +# Function: divide (wrapped version of int_divide) +# Arguments: 1: An integer +# 2: Another integer +# Returns: Returns the integer division of the first argument by the second +# ---------------------------------------------------------------------------- +divide = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_divide,$1,$2) + +# ---------------------------------------------------------------------------- +# Function: int_modulo +# Arguments: 1: A number in x's representation +# 2: Another number in x's represntation +# Returns: Returns the remainder of integer division of argument 1 divided +# by argument 2 in x's representation +# ---------------------------------------------------------------------------- +int_modulo = $(__gmsl_tr2)$(strip $(if $1,$(if $2, \ + $(filter-out M,$(subst $2,M,$1)), \ + $(call __gmsl_error,Division by zero)))) + +# ---------------------------------------------------------------------------- +# Function: modulo (wrapped version of int_modulo) +# Arguments: 1: An integer +# 2: Another integer +# Returns: Returns the remainder of integer division of the first argument +# by the second +# ---------------------------------------------------------------------------- +modulo = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_modulo,$1,$2) + +# ---------------------------------------------------------------------------- +# Function: int_max, int_min +# Arguments: 1: A number in x's representation +# 2: Another number in x's represntation +# Returns: Returns the maximum or minimum of its arguments in x's +# representation +# ---------------------------------------------------------------------------- +int_max = $(__gmsl_tr2)$(subst xx,x,$(join $1,$2)) +int_min = $(__gmsl_tr2)$(subst xx,x,$(filter xx,$(join $1,$2))) + +# ---------------------------------------------------------------------------- +# Function: max, min +# Arguments: 1: An integer +# 2: Another integer +# Returns: Returns the maximum or minimum of its integer arguments +# ---------------------------------------------------------------------------- +max = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_max,$1,$2) +min = $(__gmsl_tr2)$(call __gmsl_int_wrap,int_min,$1,$2) + +# ---------------------------------------------------------------------------- +# Function: int_gt, int_gte, int_lt, int_lte, int_eq, int_ne +# Arguments: Two x's representation numbers to be compared +# Returns: $(true) or $(false) +# +# int_gt First argument greater than second argument +# int_gte First argument greater than or equal to second argument +# int_lt First argument less than second argument +# int_lte First argument less than or equal to second argument +# int_eq First argument is numerically equal to the second argument +# int_ne First argument is not numerically equal to the second argument +# ---------------------------------------------------------------------------- +int_gt = $(__gmsl_tr2)$(call __gmsl_make_bool, \ + $(filter-out $(words $2), \ + $(words $(call int_max,$1,$2)))) +int_gte = $(__gmsl_tr2)$(call __gmsl_make_bool, \ + $(call int_gt,$1,$2)$(call int_eq,$1,$2)) +int_lt = $(__gmsl_tr2)$(call __gmsl_make_bool, \ + $(filter-out $(words $1), \ + $(words $(call int_max,$1,$2)))) +int_lte = $(__gmsl_tr2)$(call __gmsl_make_bool, \ + $(call int_lt,$1,$2)$(call int_eq,$1,$2)) +int_eq = $(__gmsl_tr2)$(call __gmsl_make_bool, \ + $(filter $(words $1),$(words $2))) +int_ne = $(__gmsl_tr2)$(call __gmsl_make_bool, \ + $(filter-out $(words $1),$(words $2))) + +# ---------------------------------------------------------------------------- +# Function: gt, gte, lt, lte, eq, ne +# Arguments: Two integers to be compared +# Returns: $(true) or $(false) +# +# gt First argument greater than second argument +# gte First argument greater than or equal to second argument +# lt First argument less than second argument +# lte First argument less than or equal to second argument +# eq First argument is numerically equal to the second argument +# ne First argument is not numerically equal to the second argument +# ---------------------------------------------------------------------------- +gt = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_gt,$1,$2) +gte = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_gte,$1,$2) +lt = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_lt,$1,$2) +lte = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_lte,$1,$2) +eq = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_eq,$1,$2) +ne = $(__gmsl_tr2)$(call __gmsl_int_wrap2,int_ne,$1,$2) + +# increment adds 1 to its argument, decrement subtracts 1. Note that +# decrement does not range check and hence will not underflow, but +# will incorrectly say that 0 - 1 = 0 + +# ---------------------------------------------------------------------------- +# Function: int_inc +# Arguments: 1: A number in x's representation +# Returns: The number incremented by 1 in x's representation +# ---------------------------------------------------------------------------- +int_inc = $(strip $(__gmsl_tr1)$1 x) + +# ---------------------------------------------------------------------------- +# Function: inc +# Arguments: 1: An integer +# Returns: The argument incremented by 1 +# ---------------------------------------------------------------------------- +inc = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_inc,$1) + +# ---------------------------------------------------------------------------- +# Function: int_dec +# Arguments: 1: A number in x's representation +# Returns: The number decremented by 1 in x's representation +# ---------------------------------------------------------------------------- +int_dec = $(__gmsl_tr1)$(strip \ + $(if $(call sne,0,$(words $1)), \ + $(wordlist 2,$(words $1),$1))) + +# ---------------------------------------------------------------------------- +# Function: dec +# Arguments: 1: An integer +# Returns: The argument decremented by 1 +# ---------------------------------------------------------------------------- +dec = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_dec,$1) + +# double doubles its argument, and halve halves it + +# ---------------------------------------------------------------------------- +# Function: int_double +# Arguments: 1: A number in x's representation +# Returns: The number doubled (i.e. * 2) and returned in x's representation +# ---------------------------------------------------------------------------- +int_double = $(strip $(__gmsl_tr1)$1 $1) + +# ---------------------------------------------------------------------------- +# Function: double +# Arguments: 1: An integer +# Returns: The integer times 2 +# ---------------------------------------------------------------------------- +double = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_double,$1) + +# ---------------------------------------------------------------------------- +# Function: int_halve +# Arguments: 1: A number in x's representation +# Returns: The number halved (i.e. / 2) and returned in x's representation +# ---------------------------------------------------------------------------- +int_halve = $(__gmsl_tr1)$(strip $(subst xx,x,$(filter-out xy x y, \ + $(join $1,$(foreach a,$1,y x))))) + +# ---------------------------------------------------------------------------- +# Function: halve +# Arguments: 1: An integer +# Returns: The integer divided by 2 +# ---------------------------------------------------------------------------- +halve = $(__gmsl_tr1)$(call __gmsl_int_wrap1,int_halve,$1) + +# ---------------------------------------------------------------------------- +# Function: sequence +# Arguments: 1: An integer +# 2: An integer +# Returns: The sequence [arg1, arg2] of integers if arg1 < arg2 or +# [arg2, arg1] if arg2 > arg1. If arg1 == arg1 return [arg1] +# ---------------------------------------------------------------------------- +sequence = $(__gmsl_tr2)$(strip $(if $(call lte,$1,$2), \ + $(call __gmsl_sequence_up,$1,$2), \ + $(call __gmsl_sequence_dn,$2,$1))) + +__gmsl_sequence_up = $(if $(call seq,$1,$2),$1,$1 $(call __gmsl_sequence_up,$(call inc,$1),$2)) +__gmsl_sequence_dn = $(if $(call seq,$1,$2),$1,$2 $(call __gmsl_sequence_dn,$1,$(call dec,$2))) + +# ---------------------------------------------------------------------------- +# Function: dec2hex, dec2bin, dec2oct +# Arguments: 1: An integer +# Returns: The decimal argument converted to hexadecimal, binary or +# octal +# ---------------------------------------------------------------------------- + +__gmsl_digit = $(subst 15,f,$(subst 14,e,$(subst 13,d,$(subst 12,c,$(subst 11,b,$(subst 10,a,$1)))))) + +dec2hex = $(call __gmsl_dec2base,$(call int_encode,$1),$(call int_encode,16)) +dec2bin = $(call __gmsl_dec2base,$(call int_encode,$1),$(call int_encode,2)) +dec2oct = $(call __gmsl_dec2base,$(call int_encode,$1),$(call int_encode,8)) + +__gmsl_base_divide = $(subst $2,X ,$1) +__gmsl_q = $(strip $(filter X,$1)) +__gmsl_r = $(words $(filter x,$1)) + +__gmsl_dec2base = $(eval __gmsl_temp := $(call __gmsl_base_divide,$1,$2))$(call __gmsl_dec2base_,$(call __gmsl_q,$(__gmsl_temp)),$(call __gmsl_r,$(__gmsl_temp)),$2) +__gmsl_dec2base_ = $(if $1,$(call __gmsl_dec2base,$(subst X,x,$1),$3))$(call __gmsl_digit,$2) + +ifdef __gmsl_have_eval +# ########################################################################### +# ASSOCIATIVE ARRAYS +# ########################################################################### + +# Magic string that is very unlikely to appear in a key or value + +__gmsl_aa_magic := faf192c8efbc25c27992c5bc5add390393d583c6 + +# ---------------------------------------------------------------------------- +# Function: set +# Arguments: 1: Name of associative array +# 2: The key value to associate +# 3: The value associated with the key +# Returns: Nothing +# ---------------------------------------------------------------------------- +set = $(__gmsl_tr3)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2$3)$(eval __gmsl_aa_$1_$(__gmsl_aa_magic)_$2_gmsl_aa_$1 := $3) + +# Only used internally by memoize function + +__gmsl_set = $(call set,$1,$2,$3)$3 + +# ---------------------------------------------------------------------------- +# Function: get +# Arguments: 1: Name of associative array +# 2: The key to retrieve +# Returns: The value stored in the array for that key +# ---------------------------------------------------------------------------- +get = $(strip $(__gmsl_tr2)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2)$(__gmsl_aa_$1_$(__gmsl_aa_magic)_$2_gmsl_aa_$1)) + +# ---------------------------------------------------------------------------- +# Function: keys +# Arguments: 1: Name of associative array +# Returns: Returns a list of all defined keys in the array +# ---------------------------------------------------------------------------- +keys = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(sort $(patsubst __gmsl_aa_$1_$(__gmsl_aa_magic)_%_gmsl_aa_$1,%, \ + $(filter __gmsl_aa_$1_$(__gmsl_aa_magic)_%_gmsl_aa_$1,$(.VARIABLES)))) + +# ---------------------------------------------------------------------------- +# Function: defined +# Arguments: 1: Name of associative array +# 2: The key to test +# Returns: Returns true if the key is defined (i.e. not empty) +# ---------------------------------------------------------------------------- +defined = $(__gmsl_tr2)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2)$(call sne,$(call get,$1,$2),) + +endif # __gmsl_have_eval + +ifdef __gmsl_have_eval +# ########################################################################### +# NAMED STACKS +# ########################################################################### + +# ---------------------------------------------------------------------------- +# Function: push +# Arguments: 1: Name of stack +# 2: Value to push onto the top of the stack (must not contain +# a space) +# Returns: None +# ---------------------------------------------------------------------------- +push = $(__gmsl_tr2)$(call assert_no_space,$0,$1$2)$(call assert_no_dollar,$0,$1$2)$(eval __gmsl_stack_$1 := $2 $(if $(filter-out undefined,\ + $(origin __gmsl_stack_$1)),$(__gmsl_stack_$1))) + +# ---------------------------------------------------------------------------- +# Function: pop +# Arguments: 1: Name of stack +# Returns: Top element from the stack after removing it +# ---------------------------------------------------------------------------- +pop = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(strip $(if $(filter-out undefined,$(origin __gmsl_stack_$1)), \ + $(call first,$(__gmsl_stack_$1)) \ + $(eval __gmsl_stack_$1 := $(call rest,$(__gmsl_stack_$1))))) + +# ---------------------------------------------------------------------------- +# Function: peek +# Arguments: 1: Name of stack +# Returns: Top element from the stack without removing it +# ---------------------------------------------------------------------------- +peek = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(call first,$(__gmsl_stack_$1)) + +# ---------------------------------------------------------------------------- +# Function: depth +# Arguments: 1: Name of stack +# Returns: Number of items on the stack +# ---------------------------------------------------------------------------- +depth = $(__gmsl_tr1)$(call assert_no_space,$0,$1)$(call assert_no_dollar,$0,$1)$(words $(__gmsl_stack_$1)) + +endif # __gmsl_have_eval + +ifdef __gmsl_have_eval +# ########################################################################### +# STRING CACHE +# ########################################################################### + +# ---------------------------------------------------------------------------- +# Function: memoize +# Arguments: 1. Name of the function to be called if the string +# has not been previously seen +# 2. A string +# Returns: Returns the result of a memo function (which the user must +# define) on the passed in string and remembers the result. +# +# Example: Set memo = $(shell echo "$1" | md5sum) to make a cache +# of MD5 hashes of strings. $(call memoize,memo,foo bar baz) +# ---------------------------------------------------------------------------- +__gmsl_memoize = $(subst $(__gmsl_space),§,$1)cc2af1bb7c4482f2ba75e338b963d3e7$(subst $(__gmsl_space),§,$2) +memoize = $(__gmsl_tr2)$(strip $(if $(call defined,__gmsl_m,$(__gmsl_memoize)),\ + $(call get,__gmsl_m,$(__gmsl_memoize)), \ + $(call __gmsl_set,__gmsl_m,$(__gmsl_memoize),$(call $1,$2)))) + +endif # __gmsl_have_eval + +# ########################################################################### +# DEBUGGING FACILITIES +# ########################################################################### + +# ---------------------------------------------------------------------------- +# Target: gmsl-echo-% +# Arguments: The % should be replaced by the name of a variable that you +# wish to print out. +# Action: Echos the value of the variable that matches the %. +# For example, 'make gmsl-echo-SHELL' will output the value of +# the SHELL variable. +# ---------------------------------------------------------------------------- +gmsl-echo-%: ; @echo $($*) + +# ---------------------------------------------------------------------------- +# Target: gmsl-print-% +# Arguments: The % should be replaced by the name of a variable that you +# wish to print out. +# Action: Echos the name of the variable that matches the % and its value. +# For example, 'make gmsl-print-SHELL' will output the value of +# the SHELL variable +# ---------------------------------------------------------------------------- +gmsl-print-%: ; @echo $* = $($*) + +# ---------------------------------------------------------------------------- +# Function: assert +# Arguments: 1: A boolean that must be true or the assertion will fail +# 2: The message to print with the assertion +# Returns: None +# ---------------------------------------------------------------------------- +assert = $(if $2,$(if $1,,$(call __gmsl_error,Assertion failure: $2))) + +# ---------------------------------------------------------------------------- +# Function: assert_exists +# Arguments: 1: Name of file that must exist, if it is missing an assertion +# will be generated +# Returns: None +# ---------------------------------------------------------------------------- +assert_exists = $(if $0,$(call assert,$(wildcard $1),file '$1' missing)) + +# ---------------------------------------------------------------------------- +# Function: assert_no_dollar +# Arguments: 1: Name of a function being executd +# 2: Arguments to check +# Returns: None +# ---------------------------------------------------------------------------- +assert_no_dollar = $(call __gmsl_tr2)$(call assert,$(call not,$(findstring $(__gmsl_dollar),$2)),$1 called with a dollar sign in argument) + +# ---------------------------------------------------------------------------- +# Function: assert_no_space +# Arguments: 1: Name of a function being executd +# 2: Arguments to check +# Returns: None +# ---------------------------------------------------------------------------- +ifeq ($(__gmsl_spaced_vars),$(false)) +assert_no_space = $(call assert,$(call not,$(findstring $(__gmsl_aa_magic),$(subst $(__gmsl_space),$(__gmsl_aa_magic),$2))),$1 called with a space in argument) +else +assert_no_space = +endif diff --git a/mk/gmsl b/mk/gmsl new file mode 100644 index 000000000..b22949a31 --- /dev/null +++ b/mk/gmsl @@ -0,0 +1,85 @@ +# ---------------------------------------------------------------------------- +# +# GNU Make Standard Library (GMSL) +# +# A library of functions to be used with GNU Make's $(call) that +# provides functionality not available in standard GNU Make. +# +# Copyright (c) 2005-2022 John Graham-Cumming +# +# This file is part of GMSL +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions +# are met: +# +# Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# Neither the name of the John Graham-Cumming nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +# LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +# CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +# ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +# POSSIBILITY OF SUCH DAMAGE. +# +# ---------------------------------------------------------------------------- + +# Determine if the library has already been included and if so don't +# bother including it again + +ifndef __gmsl_included + +# Standard definitions for true and false. true is any non-empty +# string, false is an empty string. These are intended for use with +# $(if). + +true := T +false := + +# ---------------------------------------------------------------------------- +# Function: not +# Arguments: 1: A boolean value +# Returns: Returns the opposite of the arg. (true -> false, false -> true) +# ---------------------------------------------------------------------------- +not = $(if $1,$(false),$(true)) + +# Prevent reinclusion of the library + +__gmsl_included := $(true) + +# Try to determine where this file is located. If the caller did +# include /foo/gmsl then extract the /foo/ so that __gmsl gets +# included transparently + +__gmsl_root := + +ifneq ($(MAKEFILE_LIST),) +__gmsl_root := $(word $(words $(MAKEFILE_LIST)),$(MAKEFILE_LIST)) + +# If there are any spaces in the path in __gmsl_root then give up + +ifeq (1,$(words $(__gmsl_root))) +__gmsl_root := $(patsubst %gmsl,%,$(__gmsl_root)) +endif + +endif + +include $(__gmsl_root)__gmsl + +endif # __gmsl_included + diff --git a/mk/gmsl.html b/mk/gmsl.html new file mode 100644 index 000000000..a8da46987 --- /dev/null +++ b/mk/gmsl.html @@ -0,0 +1,733 @@ + + + + GNU Make Standard Library + + +

GNU Make Standard Library

+The GNU Make Standard Library (GMSL) is a collection of functions +implemented using native GNU Make functionality that provide list and +string manipulation, integer arithmetic, associative arrays, stacks, +and debugging facilities.  The GMSL is released under the BSD License.
+
+[Project Page] +[Releases] +
+

Using GMSL

+The two files needed are gmsl +and __gmsl.  To +include the GMSL in your Makefile do
+
include gmsl
+gmsl automatically includes __gmsl.  To check that +you have the right version of gmsl +use the gmsl_compatible +function (see +below). The current version is 1 2 0.
+
+The GMSL package also includes a test suite for GMSL.  Just run make -f gmsl-tests.
+

Logical Operators

GMSL has boolean $(true) (a non-empty string) +and $(false) (an empty string).  The following operators can be +used with those variables.
+
+
not
+ +
+ +Arguments: A boolean value
+ +Returns:   Returns $(true) if the boolean is $(false) and vice versa
+ +
and
+
+Arguments: Two boolean values
+Returns:   Returns $(true) if both of the booleans are true
+
or
+
+Arguments: Two boolean values
+Returns:   Returns $(true) if either of the booleans is true
+
xor
+
+Arguments: Two boolean values
+Returns:   Returns $(true) if exactly one of the booleans is true
+
nand
+
+Arguments: Two boolean values
+Returns:   Returns value of 'not and'
+
nor
+
+Arguments: Two boolean values
+Returns:   Returns value of 'not or'
+
xnor
+
+Arguments: Two boolean values
+Returns:   Returns value of 'not xor'
+
+

List Manipulation Functions

+ A list is a string of characters; the list separator is a space.
+ +
+
first
+
+Arguments: 1: A list
+Returns:   Returns the first element of a list
+ +
last
+
+Arguments: 1: A list
+Returns:   Returns the last element of a list
+ +
rest
+
+Arguments: 1: A list
+Returns:   Returns the list with the first element +removed
+ +
chop
+
+Arguments: 1: A list
+Returns:   Returns the list with the last element removed
+ +
map
+
+Arguments: 1: Name of function to +$(call) for each element of list
+           2: List to +iterate over calling the function in 1
+Returns:   The list after calling the function on each +element
+ +
pairmap
+
+Arguments: 1: Name of function to +$(call) for each pair of elements
+           2: List to +iterate over calling the function in 1
+           3: Second +list to iterate over calling the function in 1
+Returns:   The list after calling the function on each +pair of elements
+ +
leq
+
+Arguments: 1: A list to compare +against...
+           2: ...this +list
+Returns:   Returns $(true) if the two lists are identical
+ +
lne
+
+Arguments: 1: A list to compare +against...
+           2: ...this +list
+Returns:   Returns $(true) if the two lists are different
+ +
reverse
+
+Arguments: 1: A list to reverse
+Returns:   The list with its elements in reverse order
+ +
uniq
+
+Arguments: 1: A list to deduplicate
+Returns:   The list with elements in order without duplicates
+ +
length
+
+Arguments: 1: A list
+Returns:   The number of elements in the list
+ +
+

String Manipulation Functions

+A string is any sequence of characters.
+
+
seq
+
+Arguments: 1: A string to compare +against...
+           2: ...this +string
+Returns:   Returns $(true) if the two strings are +identical
+ +
sne
+
+Arguments: 1: A string to compare +against...
+           2: ...this +string
+Returns:   Returns $(true) if the two strings are not +the same
+ +
strlen
+
+Arguments: 1: A string
+Returns:   Returns the length of the string
+ +
substr
+
+Arguments: 1: A string
+           2: Start offset (first character is 1)
+           3: Ending offset (inclusive)
Returns:   Returns a substring
+ +
split
+
+Arguments: 1: The character to +split on
+           2: A +string to split
+Returns:   Splits a string into a list separated by +spaces at the split
+           character +in the first argument
+ +
merge
+
+Arguments: 1: The character to +put between fields
+           2: A list +to merge into a string
+Returns:   Merges a list into a single string, list +elements are separated
+           by the +character in the first argument
+ +
tr
+
+Arguments: 1: The list of +characters to translate from
+           2: The +list of characters to translate to
+           3: The +text to translate
+Returns:   Returns the text after translating characters
+ +
uc
+
+Arguments: 1: Text to upper case
+Returns:   Returns the text in upper case
+ +
lc
+
+Arguments: 1: Text to lower case
+Returns:   Returns the text in lower case
+ +
+

Set Manipulation Functions

+Sets are represented by sorted, deduplicated lists. To create a set +from a list use set_create, or start with the empty_set and set_insert individual elements. +The empty set is defined as empty_set.

+ +

set_create
+
+Arguments: 1: A list of set elements
+Returns:   Returns the newly created set
+ + +
set_insert
+
+Arguments: 1: A single element to add to a set
+           2: A set
+Returns:   Returns the set with the element added
+ + +
set_remove
+
+Arguments: 1: A single element to remove from a set
+           2: A set
+Returns:   Returns the set with the element removed
+ + +
set_is_member
+
+Arguments: 1: A single element
+           2: A set
+Returns:   Returns $(true) if the element is in the set
+ + +
set_is_not_member
+
+Arguments: 1: A single element
+           2: A set
+Returns:   Returns $(false) if the element is in the set
+ + +
set_union
+
+Arguments: 1: A set
+           2: Another set
+Returns:   Returns the union of the two sets
+ + +
set_intersection
+
+Arguments: 1: A set
+           2: Another set
+Returns:   Returns the intersection of the two sets
+ + +
set_is_subset
+
+Arguments: 1: A set
+           2: Another set
+Returns:   Returns $(true) if the first set is a subset of the second
+ + +
set_equal
+
+Arguments: 1: A set
+           2: Another set
+Returns:   Returns $(true) if the two sets are identical
+ + +
+

Integer Arithmetic Functions

+Integers are represented by lists with the equivalent number of +x's.  For example the number 4 is x x x x.  The maximum +integer that the library can handle as input (i.e. as the argument to a +call to int_encode) is +65536. There is no limit on integer size for internal computations or +output.
+
+The arithmetic library functions come in two forms: one form of each +function takes integers as arguments and the other form takes the +encoded form (x's created by a call to int_encode).  For example, +there are two plus functions: plus +(called with integer arguments and returns an integer) and int_plus (called with encoded +arguments and returns an encoded result).
+
+plus will be slower than int_plus because its arguments +and result have to be translated between the x's format and +integers.  If doing a complex calculation use the int_* forms with a single +encoding of inputs and single decoding of the output.  For simple +calculations the direct forms can be used.
+
+
int_decode
+
+Arguments: 1: A number of x's +representation
+Returns:   Returns the integer for human consumption +that is represented
+           by the +string of x's
+ +
int_encode
+
+Arguments: 1: A number in +human-readable integer form
+Returns:   Returns the integer encoded as a string of x's
+ +
int_plus
+
+Arguments: 1: A number in x's +representation
+           2: Another +number in x's represntation
+Returns:   Returns the sum of the two numbers in x's +representation
+ +
plus (wrapped version of int_plus)
+
+Arguments: 1: An integer
+           2: Another +integer
+Returns:   Returns the sum of the two integers
+ +
int_subtract
+
+Arguments: 1: A number in x's +representation
+           2: Another +number in x's represntation
+Returns:   Returns the difference of the two numbers in +x's representation,
+           or outputs +an error on a numeric underflow
+ +
subtract (wrapped version of int_subtract)
+
+Arguments: 1: An integer
+           2: Another +integer
+Returns:   Returns the difference of the two integers,
+           or outputs +an error on a numeric underflow
+ +
int_multiply
+
+Arguments: 1: A number in x's +representation
+           2: Another +number in x's represntation
+Returns:   Returns the product of the two numbers in x's +representation
+ +
multiply (wrapped version of int_multiply)
+
+Arguments: 1: An integer
+           2: Another +integer
+Returns:   Returns the product of the two integers
+ +
int_divide
+
+Arguments: 1: A number in x's +representation
+           2: Another +number in x's represntation
+Returns:   Returns the result of integer division of +argument 1 divided
+           by +argument 2 in x's representation
+ +
divide (wrapped version of int_divide)
+
+Arguments: 1: An integer
+           2: Another +integer
+Returns:   Returns the integer division of the first +argument by the second
+ +
int_modulo
+
+Arguments: 1: A number in x's +representation
+           2: Another +number in x's represntation
+Returns:   Returns the remainder of integer division of +argument 1 divided
+           by +argument 2 in x's representation
+ +
modulo (wrapped version of int_modulo)
+
+Arguments: 1: An integer
+           2: Another +integer
+Returns:   Returns the remainder of integer division of the first +argument by the second
+ +
int_max, int_min
+
+Arguments: 1: A number in x's +representation
+           2: Another +number in x's represntation
+Returns:   Returns the maximum or minimum of its +arguments in x's
+           +representation
+ +
max, min
+
+Arguments: 1: An integer
+           2: Another +integer
+Returns:   Returns the maximum or minimum of its integer +arguments
+ +
int_gt, int_gte, int_lt, int_lte, int_eq, int_ne
+
+Arguments: Two x's representation +numbers to be compared
+Returns:   $(true) or $(false)
+
+int_gt First argument greater than second argument
+int_gte First argument greater than or equal to second argument
+int_lt First argument less than second argument
+int_lte First argument less than or equal to second argument
+int_eq First argument is numerically equal to the second argument
+int_ne First argument is not numerically equal to the second argument
+ +
gt, gte, lt, lte, eq, ne
+
+Arguments: Two integers to be +compared
+Returns:   $(true) or $(false)
+
+gt First argument greater than second argument
+gte First argument greater than or equal to second argument
+lt First argument less than second argument
+lte First argument less than or equal to second argument
+eq First argument is numerically equal to the second argument
+ne First argument is not numerically equal to the second argument
+ +increment adds 1 to its argument, decrement subtracts 1. Note that
+decrement does not range check and hence will not underflow, but
+will incorrectly say that 0 - 1 = 0
+
int_inc
+
+Arguments: 1: A number in x's +representation
+Returns:   The number incremented by 1 in x's +representation
+ +
inc
+
+Arguments: 1: An integer
+Returns:   The argument incremented by 1
+ +
int_dec
+
+Arguments: 1: A number in x's +representation
+Returns:   The number decremented by 1 in x's +representation
+ +
dec
+
+Arguments: 1: An integer
+Returns:   The argument decremented by 1
+ +
int_double
+
+Arguments: 1: A number in x's +representation
+Returns:   The number doubled (i.e. * 2) and returned in +x's representation
+ +
double
+
+Arguments: 1: An integer
+Returns:   The integer times 2
+ +
int_halve
+
+Arguments: 1: A number in x's +representation
+Returns:   The number halved (i.e. / 2) and returned in +x's representation
+ +
halve
+
+Arguments: 1: An integer
+Returns:   The integer divided by 2
+ +
sequence
+
+Arguments: 1: An integer
+           2: An integer
+Returns:   The sequence [arg1 arg2] if arg1 >= arg2 or [arg2 arg1] if arg2 > arg1
+ +
dec2hex, dec2bin, dec2oct
+
+Arguments: 1: An integer
+Returns:   The decimal argument converted to hexadecimal, binary or octal
+ +
+

Associative Arrays

+An associate array maps a key value (a string with no spaces in it) to +a single value (any string).   
+
+ +
set
+
+Arguments: 1: Name of associative +array
+           2: The key +value to associate
+           3: The +value associated with the key
+Returns:   Nothing
+ +
get
+
+Arguments: 1: Name of associative +array
+           2: The key +to retrieve
+Returns:   The value stored in the array for that key
+ +
keys
+
+Arguments: 1: Name of associative +array
+Returns:   Returns a list of all defined keys in the +array
+ +
defined
+
+Arguments: 1: Name of associative +array
+           2: The key +to test
+Returns:   Returns true if the key is defined (i.e. not +empty)
+ +
+

Named Stacks

+A stack is an ordered list of strings (with no spaces in them).
+
+
push
+
+Arguments: 1: Name of stack
+           2: Value +to push onto the top of the stack (must not contain
+           a space)
+Returns:   None
+ +
pop
+
+Arguments: 1: Name of stack
+Returns:   Top element from the stack after removing it
+ +
peek
+
+Arguments: 1: Name of stack
+Returns:   Top element from the stack without removing it
+ +
depth
+
+Arguments: 1: Name of stack
+Returns:   Number of items on the stack
+ +
+

Function memoization

+To reduce the number of calls to slow functions (such as $(shell) a single memoization function is provided.
+
+
memoize
+
+Arguments: 1: Name of function to memoize
+           2: String argument for the function
+Returns:   Result of $1 applied to $2 but only calls $1 once for each unique $2
+ + +
+

Miscellaneous and Debugging Facilities

+GMSL defines the following constants; all are accessed as normal GNU +Make variables by wrapping them in $() or ${}.
+
+ + + + + + + + + + + + + + + + + + + + + + + +
Constant
+ 		Value
+ 		Purpose
+
true
+ 		T
+ 		Boolean for $(if) +and return from  GMSL functions
+
false
+
+ 		Boolean for $(if) +and return from GMSL functions
+
gmsl_version
+ 		1 0 0
+ 		GMSL version number as list: major minor revision
+
+
+gmsl_compatible
+
+Arguments: List containing the desired library version number (maj min +rev)
+Returns:   +$(true) if this version of the library is compatible
+           +with the requested version number, otherwise $(false) +
gmsl-print-% (target not a function)
+
+Arguments: The % should be +replaced by the name of a variable that you
+           wish to +print out.
+Action:    Echos the name of the variable that matches +the % and its value.
+           For +example, 'make gmsl-print-SHELL' will output the value of
+           the SHELL +variable
+ +
gmsl-echo-% (target not a function)
+
+Arguments: The % should be +replaced by the name of a variable that you
+           wish to +print out.
+Action:    Echos the value of the variable that matches +the %.
+           For +example, 'make gmsl-echo-SHELL' will output the value of
+           the SHELL +variable
+ +
assert
+
+Arguments: 1: A boolean that must +be true or the assertion will fail
+           2: The +message to print with the assertion
+Returns:   None
+ +
assert_exists
+
+Arguments: 1: Name of file that +must exist, if it is missing an assertion
+           will be +generated
+Returns:   None
+ +

+GMSL has a number of environment variables (or command-line overrides) +that control various bits of functionality:
+
+ + + + + + + + + + + + + + + + + + + +
Variable
+ 		Purpose
+
GMSL_NO_WARNINGS
+ 		If set prevents GMSL from outputting warning messages: +artithmetic functions generate underflow warnings.
+
GMSL_NO_ERRORS
+ 		If set prevents GMSL from generating fatal errors: division +by zero or failed assertions are fatal.
+
GMSL_TRACE
+ 		Enables function tracing.  Calls to GMSL functions will +result in name and arguments being traced.
+
+
+
+Copyright (c) 2005-2022 John Graham-Cumming.
+
+ diff --git a/mk/platform/unix_common.mk b/mk/platform/unix_common.mk index f611693f4..8f06c9328 100644 --- a/mk/platform/unix_common.mk +++ b/mk/platform/unix_common.mk @@ -7,8 +7,14 @@ MKDIR=mkdir -p # Go should not be required to run functional tests GOOS ?= $(shell go env GOOS) -#Current versioning information from env +# Current versioning information from env BUILD_VERSION?=$(shell git describe --tags) BUILD_TIMESTAMP=$(shell date +%F"_"%T) DEFAULT_CONFIGDIR?=/etc/crowdsec DEFAULT_DATADIR?=/var/lib/crowdsec/data + +PKG_CONFIG:=$(shell command -v pkg-config 2>/dev/null) + +# See if we have libre2-dev installed for C++ optimizations. +# In fedora and other distros, we need to tell where to find re2.pc +RE2_CHECK := $(shell PKG_CONFIG_PATH=/usr/local/lib/pkgconfig:$(PKG_CONFIG_PATH) pkg-config --libs re2 2>/dev/null) diff --git a/mk/platform/windows.mk b/mk/platform/windows.mk index 8e2cdf19b..68abc6853 100644 --- a/mk/platform/windows.mk +++ b/mk/platform/windows.mk @@ -4,9 +4,9 @@ MAKE=make GOOS=windows PREFIX=$(shell $$env:TEMP) -#Current versioning information from env -#BUILD_VERSION?=$(shell (Invoke-WebRequest -UseBasicParsing -Uri https://api.github.com/repos/crowdsecurity/crowdsec/releases/latest).Content | jq -r '.tag_name') -#hardcode it till i find a workaround +# Current versioning information from env +# BUILD_VERSION?=$(shell (Invoke-WebRequest -UseBasicParsing -Uri https://api.github.com/repos/crowdsecurity/crowdsec/releases/latest).Content | jq -r '.tag_name') +# hardcode it till I find a workaround BUILD_VERSION?=$(shell git describe --tags $$(git rev-list --tags --max-count=1)) BUILD_TIMESTAMP?=$(shell Get-Date -Format "yyyy-MM-dd_HH:mm:ss") DEFAULT_CONFIGDIR?=C:\\ProgramData\\CrowdSec\\config @@ -18,3 +18,5 @@ CP=Copy-Item CPR=Copy-Item -Recurse MKDIR=New-Item -ItemType directory WIN_IGNORE_ERR=; exit 0 + +PKG_CONFIG:=$(shell Get-Command pkg-config -ErrorAction SilentlyContinue) diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 9fc8fc86f..a7916dc2b 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -35,6 +35,20 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) +type DataSourceUnavailableError struct { + Name string + Err error +} + +func (e *DataSourceUnavailableError) Error() string { + return fmt.Sprintf("datasource '%s' is not available: %v", e.Name, e.Err) +} + +func (e *DataSourceUnavailableError) Unwrap() error { + return e.Err +} + + // The interface each datasource must implement type DataSource interface { GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module @@ -75,6 +89,10 @@ func GetDataSourceIface(dataSourceType string) DataSource { return source() } +// DataSourceConfigure creates and returns a DataSource object from a configuration, +// if the configuration is not valid it returns an error. +// If the datasource can't be run (eg. journalctl not available), it still returns an error which +// can be checked for the appropriate action. func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataSource, error) { // we dump it back to []byte, because we want to decode the yaml blob twice: // once to DataSourceCommonCfg, and then later to the dedicated type of the datasource @@ -100,7 +118,7 @@ func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg) (*DataS subLogger := clog.WithFields(customLog) /* check eventual dependencies are satisfied (ie. journald will check journalctl availability) */ if err := dataSrc.CanRun(); err != nil { - return nil, fmt.Errorf("datasource %s cannot be run: %w", commonConfig.Source, err) + return nil, &DataSourceUnavailableError{Name: commonConfig.Source, Err: err} } /* configure the actual datasource */ if err := dataSrc.Configure(yamlConfig, subLogger); err != nil { @@ -173,10 +191,11 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, } dec := yaml.NewDecoder(yamlFile) dec.SetStrict(true) + idx := -1 for { var sub configuration.DataSourceCommonCfg - var idx int err = dec.Decode(&sub) + idx += 1 if err != nil { if !errors.Is(err, io.EOF) { return nil, fmt.Errorf("failed to yaml decode %s: %w", acquisFile, err) @@ -193,7 +212,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, if len(sub.Labels) == 0 { if sub.Source == "" { log.Debugf("skipping empty item in %s", acquisFile) - idx += 1 continue } return nil, fmt.Errorf("missing labels in %s (position: %d)", acquisFile, idx) @@ -208,6 +226,11 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, sub.UniqueId = uniqueId src, err := DataSourceConfigure(sub) if err != nil { + var dserr *DataSourceUnavailableError + if errors.As(err, &dserr) { + log.Error(err) + continue + } return nil, fmt.Errorf("while configuring datasource of type %s from %s (position: %d): %w", sub.Source, acquisFile, idx, err) } if sub.TransformExpr != "" { @@ -218,7 +241,6 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg) ([]DataSource, transformRuntimes[uniqueId] = vm } sources = append(sources, *src) - idx += 1 } } return sources, nil @@ -295,6 +317,11 @@ func transform(transformChan chan types.Event, output chan types.Event, AcquisTo } func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb *tomb.Tomb) error { + // Don't wait if we have no sources, as it will hang forever + if len(sources) == 0 { + return nil + } + for i := 0; i < len(sources); i++ { subsrc := sources[i] //ensure its a copy log.Debugf("starting one source %d/%d ->> %T", i, len(sources), subsrc) @@ -330,11 +357,8 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb return nil }) } - // Don't wait if we have no sources, as it will hang forever - if len(sources) > 0 { - /*return only when acquisition is over (cat) or never (tail)*/ - err := AcquisTomb.Wait() - return err - } - return nil + + /*return only when acquisition is over (cat) or never (tail)*/ + err := AcquisTomb.Wait() + return err } diff --git a/pkg/acquisition/acquisition_test.go b/pkg/acquisition/acquisition_test.go index 6b6d5ce71..548ecc04b 100644 --- a/pkg/acquisition/acquisition_test.go +++ b/pkg/acquisition/acquisition_test.go @@ -167,7 +167,7 @@ log_level: debug source: mock_cant_run wowo: ajsajasjas `, - ExpectedError: "datasource mock_cant_run cannot be run: can't run bro", + ExpectedError: "datasource 'mock_cant_run' is not available: can't run bro", }, } diff --git a/pkg/acquisition/modules/cloudwatch/cloudwatch.go b/pkg/acquisition/modules/cloudwatch/cloudwatch.go index 1abf04c5b..48bbe4217 100644 --- a/pkg/acquisition/modules/cloudwatch/cloudwatch.go +++ b/pkg/acquisition/modules/cloudwatch/cloudwatch.go @@ -13,7 +13,6 @@ import ( "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/cloudwatchlogs" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -201,7 +200,7 @@ func (cw *CloudwatchSource) Configure(yamlConfig []byte, logger *log.Entry) erro targetStream := "*" if cw.Config.StreamRegexp != nil { if _, err := regexp.Compile(*cw.Config.StreamRegexp); err != nil { - return errors.Wrapf(err, "error while compiling regexp '%s'", *cw.Config.StreamRegexp) + return fmt.Errorf("while compiling regexp '%s': %w", *cw.Config.StreamRegexp, err) } targetStream = *cw.Config.StreamRegexp } else if cw.Config.StreamName != nil { @@ -345,8 +344,7 @@ func (cw *CloudwatchSource) WatchLogGroupForStreams(out chan LogStreamTailConfig }, ) if err != nil { - newerr := errors.Wrapf(err, "while describing group %s", cw.Config.GroupName) - return newerr + return fmt.Errorf("while describing group %s: %w", cw.Config.GroupName, err) } cw.logger.Tracef("after DescribeLogStreamsPagesWithContext") } @@ -495,7 +493,7 @@ func (cw *CloudwatchSource) TailLogStream(cfg *LogStreamTailConfig, outChan chan }, ) if err != nil { - newerr := errors.Wrapf(err, "while reading %s/%s", cfg.GroupName, cfg.StreamName) + newerr := fmt.Errorf("while reading %s/%s: %w", cfg.GroupName, cfg.StreamName, err) cfg.logger.Warningf("err : %s", newerr) return newerr } @@ -532,7 +530,7 @@ func (cw *CloudwatchSource) ConfigureByDSN(dsn string, labels map[string]string, u, err := url.ParseQuery(args[1]) if err != nil { - return errors.Wrapf(err, "while parsing %s", dsn) + return fmt.Errorf("while parsing %s: %w", dsn, err) } for k, v := range u { @@ -543,7 +541,7 @@ func (cw *CloudwatchSource) ConfigureByDSN(dsn string, labels map[string]string, } lvl, err := log.ParseLevel(v[0]) if err != nil { - return errors.Wrapf(err, "unknown level %s", v[0]) + return fmt.Errorf("unknown level %s: %w", v[0], err) } cw.logger.Logger.SetLevel(lvl) @@ -577,7 +575,7 @@ func (cw *CloudwatchSource) ConfigureByDSN(dsn string, labels map[string]string, //let's reuse our parser helper so that a ton of date formats are supported duration, err := time.ParseDuration(v[0]) if err != nil { - return errors.Wrapf(err, "unable to parse '%s' as duration", v[0]) + return fmt.Errorf("unable to parse '%s' as duration: %w", v[0], err) } cw.logger.Debugf("parsed '%s' as '%s'", v[0], duration) start := time.Now().UTC().Add(-duration) @@ -674,7 +672,7 @@ func (cw *CloudwatchSource) CatLogStream(cfg *LogStreamTailConfig, outChan chan }, ) if err != nil { - return errors.Wrapf(err, "while reading logs from %s/%s", cfg.GroupName, cfg.StreamName) + return fmt.Errorf("while reading logs from %s/%s: %w", cfg.GroupName, cfg.StreamName, err) } cfg.logger.Tracef("after GetLogEventsPagesWithContext") case <-cw.t.Dying(): diff --git a/pkg/acquisition/modules/docker/docker.go b/pkg/acquisition/modules/docker/docker.go index b1808e446..929626974 100644 --- a/pkg/acquisition/modules/docker/docker.go +++ b/pkg/acquisition/modules/docker/docker.go @@ -12,7 +12,6 @@ import ( dockerTypes "github.com/docker/docker/api/types" "github.com/docker/docker/client" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -80,7 +79,7 @@ func (d *DockerSource) UnmarshalConfig(yamlConfig []byte) error { err := yaml.UnmarshalStrict(yamlConfig, &d.Config) if err != nil { - return errors.Wrap(err, "Cannot parse DockerAcquisition configuration") + return fmt.Errorf("while parsing DockerAcquisition configuration: %w", err) } if d.logger != nil { @@ -214,7 +213,7 @@ func (d *DockerSource) ConfigureByDSN(dsn string, labels map[string]string, logg parameters, err := url.ParseQuery(args[1]) if err != nil { - return errors.Wrapf(err, "while parsing parameters %s: %s", dsn, err) + return fmt.Errorf("while parsing parameters %s: %w", dsn, err) } for k, v := range parameters { @@ -225,7 +224,7 @@ func (d *DockerSource) ConfigureByDSN(dsn string, labels map[string]string, logg } lvl, err := log.ParseLevel(v[0]) if err != nil { - return errors.Wrapf(err, "unknown level %s", v[0]) + return fmt.Errorf("unknown level %s: %w", v[0], err) } d.logger.Logger.SetLevel(lvl) case "until": diff --git a/pkg/acquisition/modules/file/file.go b/pkg/acquisition/modules/file/file.go index c24b17332..799e78dca 100644 --- a/pkg/acquisition/modules/file/file.go +++ b/pkg/acquisition/modules/file/file.go @@ -14,8 +14,6 @@ import ( "strings" "time" - "github.com/crowdsecurity/go-cs-lib/pkg/trace" - "github.com/fsnotify/fsnotify" "github.com/nxadm/tail" "github.com/pkg/errors" @@ -24,6 +22,8 @@ import ( "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -110,7 +110,7 @@ func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry) error { f.watcher, err = fsnotify.NewWatcher() if err != nil { - return errors.Wrapf(err, "Could not create fsnotify watcher") + return fmt.Errorf("could not create fsnotify watcher: %w", err) } f.logger.Tracef("Actual FileAcquisition Configuration %+v", f.config) @@ -130,7 +130,7 @@ func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry) error { } files, err := filepath.Glob(pattern) if err != nil { - return errors.Wrap(err, "Glob failure") + return fmt.Errorf("glob failure: %w", err) } if len(files) == 0 { f.logger.Warnf("No matching files for pattern %s", pattern) @@ -191,7 +191,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger if len(args) == 2 && len(args[1]) != 0 { params, err := url.ParseQuery(args[1]) if err != nil { - return errors.Wrap(err, "could not parse file args") + return fmt.Errorf("could not parse file args: %w", err) } for key, value := range params { switch key { @@ -201,7 +201,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger } lvl, err := log.ParseLevel(value[0]) if err != nil { - return errors.Wrapf(err, "unknown level %s", value[0]) + return fmt.Errorf("unknown level %s: %w", value[0], err) } f.logger.Logger.SetLevel(lvl) case "max_buffer_size": @@ -210,7 +210,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger } maxBufferSize, err := strconv.Atoi(value[0]) if err != nil { - return errors.Wrapf(err, "could not parse max_buffer_size %s", value[0]) + return fmt.Errorf("could not parse max_buffer_size %s: %w", value[0], err) } f.config.MaxBufferSize = maxBufferSize default: @@ -226,7 +226,7 @@ func (f *FileSource) ConfigureByDSN(dsn string, labels map[string]string, logger f.logger.Debugf("Will try pattern %s", args[0]) files, err := filepath.Glob(args[0]) if err != nil { - return errors.Wrap(err, "Glob failure") + return fmt.Errorf("glob failure: %w", err) } if len(files) == 0 { @@ -433,7 +433,7 @@ func (f *FileSource) monitorNewFiles(out chan types.Event, t *tomb.Tomb) error { case <-t.Dying(): err := f.watcher.Close() if err != nil { - return errors.Wrapf(err, "could not remove all inotify watches") + return fmt.Errorf("could not remove all inotify watches: %w", err) } return nil } @@ -495,7 +495,7 @@ func (f *FileSource) readFile(filename string, out chan types.Event, t *tomb.Tom fd, err := os.Open(filename) if err != nil { - return errors.Wrapf(err, "failed opening %s", filename) + return fmt.Errorf("failed opening %s: %w", filename, err) } defer fd.Close() @@ -503,7 +503,7 @@ func (f *FileSource) readFile(filename string, out chan types.Event, t *tomb.Tom gz, err := gzip.NewReader(fd) if err != nil { logger.Errorf("Failed to read gz file: %s", err) - return errors.Wrapf(err, "failed to read gz %s", filename) + return fmt.Errorf("failed to read gz %s: %w", filename, err) } defer gz.Close() scanner = bufio.NewScanner(gz) diff --git a/pkg/acquisition/modules/file/file_test.go b/pkg/acquisition/modules/file/file_test.go index ff55bc413..8f80d050f 100644 --- a/pkg/acquisition/modules/file/file_test.go +++ b/pkg/acquisition/modules/file/file_test.go @@ -38,7 +38,7 @@ func TestBadConfiguration(t *testing.T) { { name: "glob syntax error", config: `filename: "[asd-.log"`, - expectedErr: "Glob failure: syntax error in pattern", + expectedErr: "glob failure: syntax error in pattern", }, { name: "bad exclude regexp", @@ -150,7 +150,7 @@ filename: /`, config: ` mode: cat filename: "[*-.log"`, - expectedConfigErr: "Glob failure: syntax error in pattern", + expectedConfigErr: "glob failure: syntax error in pattern", logLevel: log.WarnLevel, expectedLines: 0, }, diff --git a/pkg/acquisition/modules/journalctl/journalctl.go b/pkg/acquisition/modules/journalctl/journalctl.go index 7882cb7c2..b060ac364 100644 --- a/pkg/acquisition/modules/journalctl/journalctl.go +++ b/pkg/acquisition/modules/journalctl/journalctl.go @@ -9,7 +9,6 @@ import ( "strings" "time" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -237,7 +236,7 @@ func (j *JournalCtlSource) ConfigureByDSN(dsn string, labels map[string]string, } lvl, err := log.ParseLevel(value[0]) if err != nil { - return errors.Wrapf(err, "unknown level %s", value[0]) + return fmt.Errorf("unknown level %s: %w", value[0], err) } j.logger.Logger.SetLevel(lvl) case "since": diff --git a/pkg/acquisition/modules/kafka/kafka.go b/pkg/acquisition/modules/kafka/kafka.go index 085751cfc..dba8daf75 100644 --- a/pkg/acquisition/modules/kafka/kafka.go +++ b/pkg/acquisition/modules/kafka/kafka.go @@ -10,7 +10,6 @@ import ( "strconv" "time" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/segmentio/kafka-go" log "github.com/sirupsen/logrus" @@ -93,12 +92,12 @@ func (k *KafkaSource) Configure(yamlConfig []byte, logger *log.Entry) error { dialer, err := k.Config.NewDialer() if err != nil { - return errors.Wrapf(err, "cannot create %s dialer", dataSourceName) + return fmt.Errorf("cannot create %s dialer: %w", dataSourceName, err) } k.Reader, err = k.Config.NewReader(dialer) if err != nil { - return errors.Wrapf(err, "cannote create %s reader", dataSourceName) + return fmt.Errorf("cannote create %s reader: %w", dataSourceName, err) } if k.Reader == nil { @@ -149,7 +148,7 @@ func (k *KafkaSource) ReadMessage(out chan types.Event) error { if err == io.EOF { return nil } - k.logger.Errorln(errors.Wrapf(err, "while reading %s message", dataSourceName)) + k.logger.Errorln(fmt.Errorf("while reading %s message: %w", dataSourceName, err)) } l := types.Line{ Raw: string(m.Value), @@ -181,7 +180,7 @@ func (k *KafkaSource) RunReader(out chan types.Event, t *tomb.Tomb) error { case <-t.Dying(): k.logger.Infof("%s datasource topic %s stopping", dataSourceName, k.Config.Topic) if err := k.Reader.Close(); err != nil { - return errors.Wrapf(err, "while closing %s reader on topic '%s'", dataSourceName, k.Config.Topic) + return fmt.Errorf("while closing %s reader on topic '%s': %w", dataSourceName, k.Config.Topic, err) } return nil } @@ -264,7 +263,7 @@ func (kc *KafkaConfiguration) NewReader(dialer *kafka.Dialer) (*kafka.Reader, er rConf.GroupID = kc.GroupID } if err := rConf.Validate(); err != nil { - return &kafka.Reader{}, errors.Wrapf(err, "while validating reader configuration") + return &kafka.Reader{}, fmt.Errorf("while validating reader configuration: %w", err) } return kafka.NewReader(rConf), nil } diff --git a/pkg/acquisition/modules/kafka/kafka_test.go b/pkg/acquisition/modules/kafka/kafka_test.go index b37d0e7b7..950d33a62 100644 --- a/pkg/acquisition/modules/kafka/kafka_test.go +++ b/pkg/acquisition/modules/kafka/kafka_test.go @@ -8,13 +8,14 @@ import ( "testing" "time" + "github.com/segmentio/kafka-go" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "gopkg.in/tomb.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/cstest" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/segmentio/kafka-go" - log "github.com/sirupsen/logrus" - "gopkg.in/tomb.v2" - "gotest.tools/v3/assert" ) func TestConfigure(t *testing.T) { @@ -178,7 +179,7 @@ topic: crowdsecplaintext`), subLogger) break READLOOP } } - assert.Equal(t, ts.expectedLines, actualLines) + require.Equal(t, ts.expectedLines, actualLines) tomb.Kill(nil) tomb.Wait() }) @@ -254,7 +255,7 @@ tls: break READLOOP } } - assert.Equal(t, ts.expectedLines, actualLines) + require.Equal(t, ts.expectedLines, actualLines) tomb.Kill(nil) tomb.Wait() }) diff --git a/pkg/acquisition/modules/kinesis/kinesis.go b/pkg/acquisition/modules/kinesis/kinesis.go index 60cdc3751..cc263da4f 100644 --- a/pkg/acquisition/modules/kinesis/kinesis.go +++ b/pkg/acquisition/modules/kinesis/kinesis.go @@ -13,7 +13,6 @@ import ( "github.com/aws/aws-sdk-go/aws/arn" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/kinesis" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -124,7 +123,7 @@ func (k *KinesisSource) UnmarshalConfig(yamlConfig []byte) error { err := yaml.UnmarshalStrict(yamlConfig, &k.Config) if err != nil { - return errors.Wrap(err, "Cannot parse kinesis datasource configuration") + return fmt.Errorf("Cannot parse kinesis datasource configuration: %w", err) } if k.Config.Mode == "" { @@ -218,7 +217,7 @@ func (k *KinesisSource) WaitForConsumerDeregistration(consumerName string, strea return nil default: k.logger.Errorf("Error while waiting for consumer deregistration: %s", err) - return errors.Wrap(err, "Cannot describe stream consumer") + return fmt.Errorf("cannot describe stream consumer: %w", err) } } time.Sleep(time.Millisecond * 200 * time.Duration(i+1)) @@ -236,12 +235,12 @@ func (k *KinesisSource) DeregisterConsumer() error { switch err.(type) { case *kinesis.ResourceNotFoundException: default: - return errors.Wrap(err, "Cannot deregister stream consumer") + return fmt.Errorf("cannot deregister stream consumer: %w", err) } } err = k.WaitForConsumerDeregistration(k.Config.ConsumerName, k.Config.StreamARN) if err != nil { - return errors.Wrap(err, "Cannot wait for consumer deregistration") + return fmt.Errorf("cannot wait for consumer deregistration: %w", err) } return nil } @@ -253,7 +252,7 @@ func (k *KinesisSource) WaitForConsumerRegistration(consumerARN string) error { ConsumerARN: aws.String(consumerARN), }) if err != nil { - return errors.Wrap(err, "Cannot describe stream consumer") + return fmt.Errorf("cannot describe stream consumer: %w", err) } if *describeOutput.ConsumerDescription.ConsumerStatus == "ACTIVE" { k.logger.Debugf("Consumer %s is active", consumerARN) @@ -272,11 +271,11 @@ func (k *KinesisSource) RegisterConsumer() (*kinesis.RegisterStreamConsumerOutpu StreamARN: aws.String(k.Config.StreamARN), }) if err != nil { - return nil, errors.Wrap(err, "Cannot register stream consumer") + return nil, fmt.Errorf("cannot register stream consumer: %w", err) } err = k.WaitForConsumerRegistration(*streamConsumer.Consumer.ConsumerARN) if err != nil { - return nil, errors.Wrap(err, "Timeout while waiting for consumer to be active") + return nil, fmt.Errorf("timeout while waiting for consumer to be active: %w", err) } return streamConsumer, nil } @@ -339,7 +338,7 @@ func (k *KinesisSource) ReadFromSubscription(reader kinesis.SubscribeToShardEven logger.Infof("Subscribed shard reader is dying") err := reader.Close() if err != nil { - return errors.Wrap(err, "Cannot close kinesis subscribed shard reader") + return fmt.Errorf("cannot close kinesis subscribed shard reader: %w", err) } return nil case event, ok := <-reader.Events(): @@ -362,7 +361,7 @@ func (k *KinesisSource) SubscribeToShards(arn arn.ARN, streamConsumer *kinesis.R StreamName: aws.String(arn.Resource[7:]), }) if err != nil { - return errors.Wrap(err, "Cannot list shards for enhanced_read") + return fmt.Errorf("cannot list shards for enhanced_read: %w", err) } for _, shard := range shards.Shards { @@ -373,7 +372,7 @@ func (k *KinesisSource) SubscribeToShards(arn arn.ARN, streamConsumer *kinesis.R ConsumerARN: streamConsumer.Consumer.ConsumerARN, }) if err != nil { - return errors.Wrap(err, "Cannot subscribe to shard") + return fmt.Errorf("cannot subscribe to shard: %w", err) } k.shardReaderTomb.Go(func() error { return k.ReadFromSubscription(r.GetEventStream().Reader, out, shardId, arn.Resource[7:]) @@ -385,7 +384,7 @@ func (k *KinesisSource) SubscribeToShards(arn arn.ARN, streamConsumer *kinesis.R func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error { parsedARN, err := arn.Parse(k.Config.StreamARN) if err != nil { - return errors.Wrap(err, "Cannot parse stream ARN") + return fmt.Errorf("cannot parse stream ARN: %w", err) } if !strings.HasPrefix(parsedARN.Resource, "stream/") { return fmt.Errorf("resource part of stream ARN %s does not start with stream/", k.Config.StreamARN) @@ -395,12 +394,12 @@ func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error { k.logger.Info("starting kinesis acquisition with enhanced fan-out") err = k.DeregisterConsumer() if err != nil { - return errors.Wrap(err, "Cannot deregister consumer") + return fmt.Errorf("cannot deregister consumer: %w", err) } streamConsumer, err := k.RegisterConsumer() if err != nil { - return errors.Wrap(err, "Cannot register consumer") + return fmt.Errorf("cannot register consumer: %w", err) } for { @@ -408,7 +407,7 @@ func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error { err = k.SubscribeToShards(parsedARN, streamConsumer, out) if err != nil { - return errors.Wrap(err, "Cannot subscribe to shards") + return fmt.Errorf("cannot subscribe to shards: %w", err) } select { case <-t.Dying(): @@ -417,7 +416,7 @@ func (k *KinesisSource) EnhancedRead(out chan types.Event, t *tomb.Tomb) error { _ = k.shardReaderTomb.Wait() //we don't care about the error as we kill the tomb ourselves err = k.DeregisterConsumer() if err != nil { - return errors.Wrap(err, "Cannot deregister consumer") + return fmt.Errorf("cannot deregister consumer: %w", err) } return nil case <-k.shardReaderTomb.Dying(): @@ -440,7 +439,7 @@ func (k *KinesisSource) ReadFromShard(out chan types.Event, shardId string) erro ShardIteratorType: aws.String(kinesis.ShardIteratorTypeLatest)}) if err != nil { logger.Errorf("Cannot get shard iterator: %s", err) - return errors.Wrap(err, "Cannot get shard iterator") + return fmt.Errorf("cannot get shard iterator: %w", err) } it := sharIt.ShardIterator //AWS recommends to wait for a second between calls to GetRecords for a given shard @@ -461,7 +460,7 @@ func (k *KinesisSource) ReadFromShard(out chan types.Event, shardId string) erro continue default: logger.Error("Cannot get records") - return errors.Wrap(err, "Cannot get records") + return fmt.Errorf("cannot get records: %w", err) } } k.ParseAndPushRecords(records.Records, out, logger, shardId) @@ -486,7 +485,7 @@ func (k *KinesisSource) ReadFromStream(out chan types.Event, t *tomb.Tomb) error StreamName: aws.String(k.Config.StreamName), }) if err != nil { - return errors.Wrap(err, "Cannot list shards") + return fmt.Errorf("cannot list shards: %w", err) } k.shardReaderTomb = &tomb.Tomb{} for _, shard := range shards.Shards { diff --git a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go index f65a0aa57..243547381 100644 --- a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go +++ b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go @@ -8,16 +8,16 @@ import ( "net/http" "strings" - "github.com/crowdsecurity/go-cs-lib/pkg/trace" - - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" - "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" "k8s.io/apiserver/pkg/apis/audit" + + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/types" ) type KubernetesAuditConfiguration struct { @@ -66,7 +66,7 @@ func (ka *KubernetesAuditSource) UnmarshalConfig(yamlConfig []byte) error { k8sConfig := KubernetesAuditConfiguration{} err := yaml.UnmarshalStrict(yamlConfig, &k8sConfig) if err != nil { - return errors.Wrap(err, "Cannot parse k8s-audit configuration") + return fmt.Errorf("cannot parse k8s-audit configuration: %w", err) } ka.config = k8sConfig @@ -140,7 +140,7 @@ func (ka *KubernetesAuditSource) StreamingAcquisition(out chan types.Event, t *t t.Go(func() error { err := ka.server.ListenAndServe() if err != nil && err != http.ErrServerClosed { - return errors.Wrap(err, "k8s-audit server failed") + return fmt.Errorf("k8s-audit server failed: %w", err) } return nil }) diff --git a/pkg/acquisition/modules/s3/s3.go b/pkg/acquisition/modules/s3/s3.go index 4ba31f43b..96026f55e 100644 --- a/pkg/acquisition/modules/s3/s3.go +++ b/pkg/acquisition/modules/s3/s3.go @@ -6,6 +6,7 @@ import ( "compress/gzip" "context" "encoding/json" + "errors" "fmt" "io" "net/url" @@ -21,13 +22,13 @@ import ( "github.com/aws/aws-sdk-go/service/s3/s3iface" "github.com/aws/aws-sdk-go/service/sqs" "github.com/aws/aws-sdk-go/service/sqs/sqsiface" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" - "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" + + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/types" ) type S3Configuration struct { @@ -563,7 +564,7 @@ func (s *S3Source) ConfigureByDSN(dsn string, labels map[string]string, logger * if len(args) == 2 && len(args[1]) != 0 { params, err := url.ParseQuery(args[1]) if err != nil { - return errors.Wrap(err, "could not parse s3 args") + return fmt.Errorf("could not parse s3 args: %w", err) } for key, value := range params { switch key { @@ -573,7 +574,7 @@ func (s *S3Source) ConfigureByDSN(dsn string, labels map[string]string, logger * } lvl, err := log.ParseLevel(value[0]) if err != nil { - return errors.Wrapf(err, "unknown level %s", value[0]) + return fmt.Errorf("unknown level %s: %w", value[0], err) } s.logger.Logger.SetLevel(lvl) case "max_buffer_size": @@ -582,7 +583,7 @@ func (s *S3Source) ConfigureByDSN(dsn string, labels map[string]string, logger * } maxBufferSize, err := strconv.Atoi(value[0]) if err != nil { - return errors.Wrapf(err, "invalid value for 'max_buffer_size'") + return fmt.Errorf("invalid value for 'max_buffer_size': %w", err) } s.logger.Debugf("Setting max buffer size to %d", maxBufferSize) s.Config.MaxBufferSize = maxBufferSize diff --git a/pkg/acquisition/modules/syslog/internal/server/syslogserver.go b/pkg/acquisition/modules/syslog/internal/server/syslogserver.go index 088ab0d95..7118c295b 100644 --- a/pkg/acquisition/modules/syslog/internal/server/syslogserver.go +++ b/pkg/acquisition/modules/syslog/internal/server/syslogserver.go @@ -6,7 +6,6 @@ import ( "strings" "time" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" ) @@ -31,18 +30,18 @@ func (s *SyslogServer) Listen(listenAddr string, port int) error { s.port = port udpAddr, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", s.listenAddr, s.port)) if err != nil { - return errors.Wrapf(err, "could not resolve addr %s", s.listenAddr) + return fmt.Errorf("could not resolve addr %s: %w", s.listenAddr, err) } udpConn, err := net.ListenUDP("udp", udpAddr) if err != nil { - return errors.Wrapf(err, "could not listen on port %d", s.port) + return fmt.Errorf("could not listen on port %d: %w", s.port, err) } s.Logger.Debugf("listening on %s:%d", s.listenAddr, s.port) s.udpConn = udpConn err = s.udpConn.SetReadDeadline(time.Now().UTC().Add(100 * time.Millisecond)) if err != nil { - return errors.Wrap(err, "could not set read deadline on UDP socket") + return fmt.Errorf("could not set read deadline on UDP socket: %w", err) } return nil } @@ -87,7 +86,7 @@ func (s *SyslogServer) StartServer() *tomb.Tomb { func (s *SyslogServer) KillServer() error { err := s.udpConn.Close() if err != nil { - return errors.Wrap(err, "could not close UDP connection") + return fmt.Errorf("could not close UDP connection: %w", err) } close(s.channel) return nil diff --git a/pkg/acquisition/modules/syslog/syslog.go b/pkg/acquisition/modules/syslog/syslog.go index 948f3d005..840e37200 100644 --- a/pkg/acquisition/modules/syslog/syslog.go +++ b/pkg/acquisition/modules/syslog/syslog.go @@ -6,7 +6,6 @@ import ( "strings" "time" - "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -100,7 +99,7 @@ func (s *SyslogSource) UnmarshalConfig(yamlConfig []byte) error { err := yaml.UnmarshalStrict(yamlConfig, &s.config) if err != nil { - return errors.Wrap(err, "Cannot parse syslog configuration") + return fmt.Errorf("cannot parse syslog configuration: %w", err) } if s.config.Addr == "" { @@ -140,7 +139,7 @@ func (s *SyslogSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) s.server.SetChannel(c) err := s.server.Listen(s.config.Addr, s.config.Port) if err != nil { - return errors.Wrap(err, "could not start syslog server") + return fmt.Errorf("could not start syslog server: %w", err) } s.serverTomb = s.server.StartServer() t.Go(func() error { diff --git a/pkg/apiclient/alerts_service.go b/pkg/apiclient/alerts_service.go index b8d1b7fc1..dd2ba2975 100644 --- a/pkg/apiclient/alerts_service.go +++ b/pkg/apiclient/alerts_service.go @@ -5,9 +5,9 @@ import ( "fmt" "net/http" - "github.com/crowdsecurity/crowdsec/pkg/models" qs "github.com/google/go-querystring/query" - "github.com/pkg/errors" + + "github.com/crowdsecurity/crowdsec/pkg/models" ) // type ApiAlerts service @@ -72,7 +72,7 @@ func (s *AlertsService) List(ctx context.Context, opts AlertsListOpts) (*models. u := fmt.Sprintf("%s/alerts", s.client.URLPrefix) params, err := qs.Values(opts) if err != nil { - return nil, nil, errors.Wrap(err, "building query") + return nil, nil, fmt.Errorf("building query: %w", err) } if len(params) > 0 { URI = fmt.Sprintf("%s?%s", u, params.Encode()) @@ -82,12 +82,12 @@ func (s *AlertsService) List(ctx context.Context, opts AlertsListOpts) (*models. req, err := s.client.NewRequest(http.MethodGet, URI, nil) if err != nil { - return nil, nil, errors.Wrap(err, "building request") + return nil, nil, fmt.Errorf("building request: %w", err) } resp, err := s.client.Do(ctx, req, &alerts) if err != nil { - return nil, resp, errors.Wrap(err, "performing request") + return nil, resp, fmt.Errorf("performing request: %w", err) } return &alerts, resp, nil } diff --git a/pkg/apiclient/alerts_service_test.go b/pkg/apiclient/alerts_service_test.go index f4ec8cabe..aa5039f0b 100644 --- a/pkg/apiclient/alerts_service_test.go +++ b/pkg/apiclient/alerts_service_test.go @@ -8,12 +8,13 @@ import ( "reflect" "testing" - "github.com/crowdsecurity/go-cs-lib/pkg/version" - - "github.com/crowdsecurity/crowdsec/pkg/models" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/go-cs-lib/pkg/version" + + "github.com/crowdsecurity/crowdsec/pkg/models" ) func TestAlertsListAsMachine(t *testing.T) { diff --git a/pkg/apiclient/auth.go b/pkg/apiclient/auth.go index 48b971a06..84df74456 100644 --- a/pkg/apiclient/auth.go +++ b/pkg/apiclient/auth.go @@ -3,23 +3,21 @@ package apiclient import ( "bytes" "encoding/json" - "math/rand" - "sync" - "time" - - //"errors" "fmt" "io" + "math/rand" "net/http" "net/http/httputil" "net/url" + "sync" + "time" - "github.com/crowdsecurity/crowdsec/pkg/fflag" - "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/go-openapi/strfmt" "github.com/pkg/errors" log "github.com/sirupsen/logrus" - //"google.golang.org/appengine/log" + + "github.com/crowdsecurity/crowdsec/pkg/fflag" + "github.com/crowdsecurity/crowdsec/pkg/models" ) type APIKeyTransport struct { @@ -169,11 +167,11 @@ func (t *JWTTransport) refreshJwtToken() error { enc.SetEscapeHTML(false) err = enc.Encode(auth) if err != nil { - return errors.Wrap(err, "could not encode jwt auth body") + return fmt.Errorf("could not encode jwt auth body: %w", err) } req, err := http.NewRequest(http.MethodPost, fmt.Sprintf("%s%s/watchers/login", t.URL, t.VersionPrefix), buf) if err != nil { - return errors.Wrap(err, "could not create request") + return fmt.Errorf("could not create request: %w", err) } req.Header.Add("Content-Type", "application/json") client := &http.Client{ @@ -196,7 +194,7 @@ func (t *JWTTransport) refreshJwtToken() error { resp, err := client.Do(req) if err != nil { - return errors.Wrap(err, "could not get jwt token") + return fmt.Errorf("could not get jwt token: %w", err) } log.Debugf("auth-jwt : http %d", resp.StatusCode) @@ -217,10 +215,10 @@ func (t *JWTTransport) refreshJwtToken() error { } if err := json.NewDecoder(resp.Body).Decode(&response); err != nil { - return errors.Wrap(err, "unable to decode response") + return fmt.Errorf("unable to decode response: %w", err) } if err := t.Expiration.UnmarshalText([]byte(response.Expire)); err != nil { - return errors.Wrap(err, "unable to parse jwt expiration") + return fmt.Errorf("unable to parse jwt expiration: %w", err) } t.Token = response.Token @@ -263,7 +261,7 @@ func (t *JWTTransport) RoundTrip(req *http.Request) (*http.Response, error) { if err != nil { /*we had an error (network error for example, or 401 because token is refused), reset the token ?*/ t.Token = "" - return resp, errors.Wrapf(err, "performing jwt auth") + return resp, fmt.Errorf("performing jwt auth: %w", err) } log.Debugf("resp-jwt: %d", resp.StatusCode) diff --git a/pkg/apiclient/auth_service.go b/pkg/apiclient/auth_service.go index 26ad80c0c..64284902e 100644 --- a/pkg/apiclient/auth_service.go +++ b/pkg/apiclient/auth_service.go @@ -21,7 +21,6 @@ type enrollRequest struct { } func (s *AuthService) UnregisterWatcher(ctx context.Context) (*Response, error) { - u := fmt.Sprintf("%s/watchers", s.client.URLPrefix) req, err := s.client.NewRequest(http.MethodDelete, u, nil) if err != nil { @@ -36,7 +35,6 @@ func (s *AuthService) UnregisterWatcher(ctx context.Context) (*Response, error) } func (s *AuthService) RegisterWatcher(ctx context.Context, registration models.WatcherRegistrationRequest) (*Response, error) { - u := fmt.Sprintf("%s/watchers", s.client.URLPrefix) req, err := s.client.NewRequest(http.MethodPost, u, ®istration) diff --git a/pkg/apiclient/auth_service_test.go b/pkg/apiclient/auth_service_test.go index 6236cf041..32ba1890f 100644 --- a/pkg/apiclient/auth_service_test.go +++ b/pkg/apiclient/auth_service_test.go @@ -10,11 +10,12 @@ import ( "net/url" "testing" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/models" - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" ) type BasicMockPayload struct { diff --git a/pkg/apiclient/client.go b/pkg/apiclient/client.go index a7db1ed33..d95f77490 100644 --- a/pkg/apiclient/client.go +++ b/pkg/apiclient/client.go @@ -11,7 +11,6 @@ import ( "net/url" "github.com/crowdsecurity/crowdsec/pkg/models" - "github.com/pkg/errors" ) var ( @@ -125,9 +124,9 @@ func RegisterClient(config *Config, client *http.Client) (*ApiClient, error) { /*if we have http status, return it*/ if err != nil { if resp != nil && resp.Response != nil { - return nil, errors.Wrapf(err, "api register (%s) http %s : %s", c.BaseURL, resp.Response.Status, err) + return nil, fmt.Errorf("api register (%s) http %s: %w", c.BaseURL, resp.Response.Status, err) } - return nil, errors.Wrapf(err, "api register (%s) : %s", c.BaseURL, err) + return nil, fmt.Errorf("api register (%s): %w", c.BaseURL, err) } return c, nil @@ -166,7 +165,7 @@ func CheckResponse(r *http.Response) error { if err == nil && data != nil { err := json.Unmarshal(data, errorResponse) if err != nil { - return errors.Wrapf(err, "http code %d, invalid body", r.StatusCode) + return fmt.Errorf("http code %d, invalid body: %w", r.StatusCode, err) } } else { errorResponse.Message = new(string) diff --git a/pkg/apiclient/client_http_test.go b/pkg/apiclient/client_http_test.go index c50769041..9e082cf51 100644 --- a/pkg/apiclient/client_http_test.go +++ b/pkg/apiclient/client_http_test.go @@ -8,9 +8,9 @@ import ( "testing" "time" - "github.com/crowdsecurity/go-cs-lib/pkg/version" - "github.com/stretchr/testify/assert" + + "github.com/crowdsecurity/go-cs-lib/pkg/version" ) func TestNewRequestInvalid(t *testing.T) { diff --git a/pkg/apiclient/client_test.go b/pkg/apiclient/client_test.go index ef52a60ab..08f56730b 100644 --- a/pkg/apiclient/client_test.go +++ b/pkg/apiclient/client_test.go @@ -9,9 +9,8 @@ import ( "runtime" "testing" - "github.com/stretchr/testify/assert" - log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" "github.com/crowdsecurity/go-cs-lib/pkg/version" ) diff --git a/pkg/apiclient/decisions_service.go b/pkg/apiclient/decisions_service.go index 054c51a9c..e96394f56 100644 --- a/pkg/apiclient/decisions_service.go +++ b/pkg/apiclient/decisions_service.go @@ -6,14 +6,15 @@ import ( "fmt" "net/http" + qs "github.com/google/go-querystring/query" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/go-cs-lib/pkg/ptr" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/modelscapi" "github.com/crowdsecurity/crowdsec/pkg/types" - qs "github.com/google/go-querystring/query" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" ) type DecisionsService service diff --git a/pkg/apiclient/decisions_service_test.go b/pkg/apiclient/decisions_service_test.go index 935ddcea5..ab7e46e64 100644 --- a/pkg/apiclient/decisions_service_test.go +++ b/pkg/apiclient/decisions_service_test.go @@ -8,14 +8,15 @@ import ( "reflect" "testing" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/crowdsecurity/go-cs-lib/pkg/ptr" "github.com/crowdsecurity/go-cs-lib/pkg/version" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/modelscapi" - log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" ) func TestDecisionsList(t *testing.T) { diff --git a/pkg/apiclient/decisions_sync_service.go b/pkg/apiclient/decisions_sync_service.go index e5284f4d9..57999691f 100644 --- a/pkg/apiclient/decisions_sync_service.go +++ b/pkg/apiclient/decisions_sync_service.go @@ -5,9 +5,9 @@ import ( "fmt" "net/http" - "github.com/crowdsecurity/crowdsec/pkg/models" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/crowdsec/pkg/models" ) type DecisionDeleteService service @@ -18,12 +18,12 @@ func (d *DecisionDeleteService) Add(ctx context.Context, deletedDecisions *model u := fmt.Sprintf("%s/decisions/delete", d.client.URLPrefix) req, err := d.client.NewRequest(http.MethodPost, u, &deletedDecisions) if err != nil { - return nil, nil, errors.Wrap(err, "while building request") + return nil, nil, fmt.Errorf("while building request: %w", err) } resp, err := d.client.Do(ctx, req, &response) if err != nil { - return nil, resp, errors.Wrap(err, "while performing request") + return nil, resp, fmt.Errorf("while performing request: %w", err) } if resp.Response.StatusCode != http.StatusOK { log.Warnf("Decisions delete response : http %s", resp.Response.Status) diff --git a/pkg/apiclient/signal.go b/pkg/apiclient/signal.go index 27d2e3693..2dceb8157 100644 --- a/pkg/apiclient/signal.go +++ b/pkg/apiclient/signal.go @@ -8,7 +8,6 @@ import ( log "github.com/sirupsen/logrus" "github.com/crowdsecurity/crowdsec/pkg/models" - "github.com/pkg/errors" ) type SignalService service @@ -19,12 +18,12 @@ func (s *SignalService) Add(ctx context.Context, signals *models.AddSignalsReque u := fmt.Sprintf("%s/signals", s.client.URLPrefix) req, err := s.client.NewRequest(http.MethodPost, u, &signals) if err != nil { - return nil, nil, errors.Wrap(err, "while building request") + return nil, nil, fmt.Errorf("while building request: %w", err) } resp, err := s.client.Do(ctx, req, &response) if err != nil { - return nil, resp, errors.Wrap(err, "while performing request") + return nil, resp, fmt.Errorf("while performing request: %w", err) } if resp.Response.StatusCode != http.StatusOK { log.Warnf("Signal push response : http %s", resp.Response.Status) diff --git a/pkg/apiserver/apic.go b/pkg/apiserver/apic.go index 3791a9a3c..b15cf21d6 100644 --- a/pkg/apiserver/apic.go +++ b/pkg/apiserver/apic.go @@ -33,7 +33,8 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/types" ) -var ( +const ( + // delta values must be smaller than the interval pullIntervalDefault = time.Hour * 2 pullIntervalDelta = 5 * time.Minute pushIntervalDefault = time.Second * 10 @@ -71,7 +72,12 @@ type apic struct { // randomDuration returns a duration value between d-delta and d+delta func randomDuration(d time.Duration, delta time.Duration) time.Duration { - return time.Duration(float64(d) + float64(delta)*(-1.0+2.0*rand.Float64())) + ret := d + time.Duration(rand.Int63n(int64(2*delta))) - delta + // ticker interval must be > 0 (nanoseconds) + if ret <= 0 { + return 1 + } + return ret } func (a *apic) FetchScenariosListFromDB() ([]string, error) { @@ -822,80 +828,6 @@ func (a *apic) Pull() error { } } -func (a *apic) GetMetrics() (*models.Metrics, error) { - metric := &models.Metrics{ - ApilVersion: ptr.Of(version.String()), - Machines: make([]*models.MetricsAgentInfo, 0), - Bouncers: make([]*models.MetricsBouncerInfo, 0), - } - machines, err := a.dbClient.ListMachines() - if err != nil { - return metric, err - } - bouncers, err := a.dbClient.ListBouncers() - if err != nil { - return metric, err - } - var lastpush string - for _, machine := range machines { - if machine.LastPush == nil { - lastpush = time.Time{}.String() - } else { - lastpush = machine.LastPush.String() - } - m := &models.MetricsAgentInfo{ - Version: machine.Version, - Name: machine.MachineId, - LastUpdate: machine.UpdatedAt.String(), - LastPush: lastpush, - } - metric.Machines = append(metric.Machines, m) - } - - for _, bouncer := range bouncers { - m := &models.MetricsBouncerInfo{ - Version: bouncer.Version, - CustomName: bouncer.Name, - Name: bouncer.Type, - LastPull: bouncer.LastPull.String(), - } - metric.Bouncers = append(metric.Bouncers, m) - } - return metric, nil -} - -func (a *apic) SendMetrics(stop chan (bool)) { - defer trace.CatchPanic("lapi/metricsToAPIC") - - ticker := time.NewTicker(a.metricsIntervalFirst) - - log.Infof("Start send metrics to CrowdSec Central API (interval: %s once, then %s)", a.metricsIntervalFirst.Round(time.Second), a.metricsInterval) - - for { - metrics, err := a.GetMetrics() - if err != nil { - log.Errorf("unable to get metrics (%s), will retry", err) - } - _, _, err = a.apiClient.Metrics.Add(context.Background(), metrics) - if err != nil { - log.Errorf("capi metrics: failed: %s", err) - } else { - log.Infof("capi metrics: metrics sent successfully") - } - - select { - case <-stop: - return - case <-ticker.C: - ticker.Reset(a.metricsInterval) - case <-a.metricsTomb.Dying(): // if one apic routine is dying, do we kill the others? - a.pullTomb.Kill(nil) - a.pushTomb.Kill(nil) - return - } - } -} - func (a *apic) Shutdown() { a.pushTomb.Kill(nil) a.pullTomb.Kill(nil) diff --git a/pkg/apiserver/apic_metrics.go b/pkg/apiserver/apic_metrics.go new file mode 100644 index 000000000..4befcf50c --- /dev/null +++ b/pkg/apiserver/apic_metrics.go @@ -0,0 +1,145 @@ +package apiserver + +import ( + "context" + "time" + + log "github.com/sirupsen/logrus" + "golang.org/x/exp/slices" + + "github.com/crowdsecurity/go-cs-lib/pkg/ptr" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + + "github.com/crowdsecurity/crowdsec/pkg/models" +) + +func (a *apic) GetMetrics() (*models.Metrics, error) { + machines, err := a.dbClient.ListMachines() + if err != nil { + return nil, err + } + + machinesInfo := make([]*models.MetricsAgentInfo, len(machines)) + + for i, machine := range machines { + machinesInfo[i] = &models.MetricsAgentInfo{ + Version: machine.Version, + Name: machine.MachineId, + LastUpdate: machine.UpdatedAt.String(), + LastPush: ptr.OrEmpty(machine.LastPush).String(), + } + } + + bouncers, err := a.dbClient.ListBouncers() + if err != nil { + return nil, err + } + + bouncersInfo := make([]*models.MetricsBouncerInfo, len(bouncers)) + + for i, bouncer := range bouncers { + bouncersInfo[i] = &models.MetricsBouncerInfo{ + Version: bouncer.Version, + CustomName: bouncer.Name, + Name: bouncer.Type, + LastPull: bouncer.LastPull.String(), + } + } + + return &models.Metrics{ + ApilVersion: ptr.Of(version.String()), + Machines: machinesInfo, + Bouncers: bouncersInfo, + }, nil +} + +func (a *apic) fetchMachineIDs() ([]string, error) { + machines, err := a.dbClient.ListMachines() + if err != nil { + return nil, err + } + + ret := make([]string, len(machines)) + for i, machine := range machines { + ret[i] = machine.MachineId + } + // sorted slices are required for the slices.Equal comparison + slices.Sort(ret) + return ret, nil +} + +// SendMetrics sends metrics to the API server until it receives a stop signal. +// +// Metrics are sent at start, then at the randomized metricsIntervalFirst, +// then at regular metricsInterval. If a change is detected in the list +// of machines, the next metrics are sent immediately. +func (a *apic) SendMetrics(stop chan (bool)) { + defer trace.CatchPanic("lapi/metricsToAPIC") + + // verify the list of machines every interval + const checkInt = 20 * time.Second + + // intervals must always be > 0 + metInts := []time.Duration{1, a.metricsIntervalFirst, a.metricsInterval} + + log.Infof("Start send metrics to CrowdSec Central API (interval: %s once, then %s)", + metInts[1].Round(time.Second), metInts[2]) + + count := -1 + nextMetInt := func() time.Duration { + if count < len(metInts)-1 { + count++ + } + return metInts[count] + } + + // store the list of machine IDs to compare + // with the next list + machineIDs := []string{} + + reloadMachineIDs := func() { + ids, err := a.fetchMachineIDs() + if err != nil { + log.Debugf("unable to get machines (%s), will retry", err) + return + } + machineIDs = ids + } + + checkTicker := time.NewTicker(checkInt) + metTicker := time.NewTicker(nextMetInt()) + + for { + select { + case <-stop: + checkTicker.Stop() + metTicker.Stop() + return + case <-checkTicker.C: + oldIDs := machineIDs + reloadMachineIDs() + if !slices.Equal(oldIDs, machineIDs) { + log.Infof("capi metrics: machines changed, immediate send") + metTicker.Reset(1) + } + case <-metTicker.C: + metrics, err := a.GetMetrics() + if err != nil { + log.Errorf("unable to get metrics (%s), will retry", err) + } + log.Info("capi metrics: sending") + _, _, err = a.apiClient.Metrics.Add(context.Background(), metrics) + if err != nil { + log.Errorf("capi metrics: failed: %s", err) + } + metTicker.Reset(nextMetInt()) + case <-a.metricsTomb.Dying(): // if one apic routine is dying, do we kill the others? + checkTicker.Stop() + metTicker.Stop() + a.pullTomb.Kill(nil) + a.pushTomb.Kill(nil) + return + } + } +} diff --git a/pkg/apiserver/apic_metrics_test.go b/pkg/apiserver/apic_metrics_test.go new file mode 100644 index 000000000..7e37ea1e9 --- /dev/null +++ b/pkg/apiserver/apic_metrics_test.go @@ -0,0 +1,101 @@ +package apiserver + +import ( + "context" + "fmt" + "net/url" + "testing" + "time" + + "github.com/jarcoal/httpmock" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/go-cs-lib/pkg/version" + + "github.com/crowdsecurity/crowdsec/pkg/apiclient" +) + +func TestAPICSendMetrics(t *testing.T) { + tests := []struct { + name string + duration time.Duration + expectedCalls int + setUp func(*apic) + metricsInterval time.Duration + }{ + { + name: "basic", + duration: time.Millisecond * 30, + metricsInterval: time.Millisecond * 5, + expectedCalls: 5, + setUp: func(api *apic) {}, + }, + { + name: "with some metrics", + duration: time.Millisecond * 30, + metricsInterval: time.Millisecond * 5, + expectedCalls: 5, + setUp: func(api *apic) { + api.dbClient.Ent.Machine.Delete().ExecX(context.Background()) + api.dbClient.Ent.Machine.Create(). + SetMachineId("1234"). + SetPassword(testPassword.String()). + SetIpAddress("1.2.3.4"). + SetScenarios("crowdsecurity/test"). + SetLastPush(time.Time{}). + SetUpdatedAt(time.Time{}). + ExecX(context.Background()) + + api.dbClient.Ent.Bouncer.Delete().ExecX(context.Background()) + api.dbClient.Ent.Bouncer.Create(). + SetIPAddress("1.2.3.6"). + SetName("someBouncer"). + SetAPIKey("foobar"). + SetRevoked(false). + SetLastPull(time.Time{}). + ExecX(context.Background()) + }, + }, + } + + httpmock.RegisterResponder("POST", "http://api.crowdsec.net/api/metrics/", httpmock.NewBytesResponder(200, []byte{})) + httpmock.Activate() + defer httpmock.Deactivate() + + for _, tc := range tests { + tc := tc + t.Run(tc.name, func(t *testing.T) { + url, err := url.ParseRequestURI("http://api.crowdsec.net/") + require.NoError(t, err) + + apiClient, err := apiclient.NewDefaultClient( + url, + "/api", + fmt.Sprintf("crowdsec/%s", version.String()), + nil, + ) + require.NoError(t, err) + + api := getAPIC(t) + api.pushInterval = time.Millisecond + api.pushIntervalFirst = time.Millisecond + api.apiClient = apiClient + api.metricsInterval = tc.metricsInterval + api.metricsIntervalFirst = tc.metricsInterval + tc.setUp(api) + + stop := make(chan bool) + httpmock.ZeroCallCounters() + go api.SendMetrics(stop) + time.Sleep(tc.duration) + stop <- true + + info := httpmock.GetCallCountInfo() + noResponderCalls := info["NO_RESPONDER"] + responderCalls := info["POST http://api.crowdsec.net/api/metrics/"] + assert.LessOrEqual(t, absDiff(tc.expectedCalls, responderCalls), 2) + assert.Zero(t, noResponderCalls) + }) + } +} diff --git a/pkg/apiserver/apic_test.go b/pkg/apiserver/apic_test.go index 65ca29991..8aeb092cd 100644 --- a/pkg/apiserver/apic_test.go +++ b/pkg/apiserver/apic_test.go @@ -1057,90 +1057,6 @@ func TestAPICPush(t *testing.T) { } } -func TestAPICSendMetrics(t *testing.T) { - tests := []struct { - name string - duration time.Duration - expectedCalls int - setUp func(*apic) - metricsInterval time.Duration - }{ - { - name: "basic", - duration: time.Millisecond * 30, - metricsInterval: time.Millisecond * 5, - expectedCalls: 5, - setUp: func(api *apic) {}, - }, - { - name: "with some metrics", - duration: time.Millisecond * 30, - metricsInterval: time.Millisecond * 5, - expectedCalls: 5, - setUp: func(api *apic) { - api.dbClient.Ent.Machine.Delete().ExecX(context.Background()) - api.dbClient.Ent.Machine.Create(). - SetMachineId("1234"). - SetPassword(testPassword.String()). - SetIpAddress("1.2.3.4"). - SetScenarios("crowdsecurity/test"). - SetLastPush(time.Time{}). - SetUpdatedAt(time.Time{}). - ExecX(context.Background()) - - api.dbClient.Ent.Bouncer.Delete().ExecX(context.Background()) - api.dbClient.Ent.Bouncer.Create(). - SetIPAddress("1.2.3.6"). - SetName("someBouncer"). - SetAPIKey("foobar"). - SetRevoked(false). - SetLastPull(time.Time{}). - ExecX(context.Background()) - }, - }, - } - - httpmock.RegisterResponder("POST", "http://api.crowdsec.net/api/metrics/", httpmock.NewBytesResponder(200, []byte{})) - httpmock.Activate() - defer httpmock.Deactivate() - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - url, err := url.ParseRequestURI("http://api.crowdsec.net/") - require.NoError(t, err) - - apiClient, err := apiclient.NewDefaultClient( - url, - "/api", - fmt.Sprintf("crowdsec/%s", version.String()), - nil, - ) - require.NoError(t, err) - - api := getAPIC(t) - api.pushInterval = time.Millisecond - api.pushIntervalFirst = time.Millisecond - api.apiClient = apiClient - api.metricsInterval = tc.metricsInterval - api.metricsIntervalFirst = tc.metricsInterval - tc.setUp(api) - - stop := make(chan bool) - httpmock.ZeroCallCounters() - go api.SendMetrics(stop) - time.Sleep(tc.duration) - stop <- true - - info := httpmock.GetCallCountInfo() - noResponderCalls := info["NO_RESPONDER"] - responderCalls := info["POST http://api.crowdsec.net/api/metrics/"] - assert.LessOrEqual(t, absDiff(tc.expectedCalls, responderCalls), 2) - assert.Zero(t, noResponderCalls) - }) - } -} - func TestAPICPull(t *testing.T) { api := getAPIC(t) tests := []struct { diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index c65100573..3facd0f6f 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -9,9 +9,18 @@ import ( "net" "net/http" "os" + "path/filepath" "strings" "time" + "github.com/gin-gonic/gin" + "github.com/go-co-op/gocron" + "github.com/golang-jwt/jwt/v4" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "gopkg.in/natefinch/lumberjack.v2" + "gopkg.in/tomb.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" "github.com/crowdsecurity/crowdsec/pkg/apiclient" @@ -22,13 +31,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/fflag" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/gin-gonic/gin" - "github.com/go-co-op/gocron" - "github.com/golang-jwt/jwt/v4" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "gopkg.in/natefinch/lumberjack.v2" - "gopkg.in/tomb.v2" ) var ( @@ -116,7 +118,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) { logFile := "" if config.LogMedia == "file" { - logFile = fmt.Sprintf("%s/crowdsec_api.log", config.LogDir) + logFile = filepath.Join(config.LogDir, "crowdsec_api.log") } if log.GetLevel() < log.DebugLevel { @@ -162,15 +164,7 @@ func NewServer(config *csconfig.LocalApiServerCfg) (*APIServer, error) { if config.CompressLogs != nil { _compress = *config.CompressLogs } - /*cf. https://github.com/natefinch/lumberjack/issues/82 - let's create the file beforehand w/ the right perms */ - // check if file exists - _, err := os.Stat(logFile) - // create file if not exists, purposefully ignore errors - if os.IsNotExist(err) { - file, _ := os.OpenFile(logFile, os.O_RDWR|os.O_CREATE, 0600) - file.Close() - } + LogOutput := &lumberjack.Logger{ Filename: logFile, MaxSize: _maxsize, //megabytes diff --git a/pkg/apiserver/controllers/v1/controller.go b/pkg/apiserver/controllers/v1/controller.go index 3820bc10f..60da83d7d 100644 --- a/pkg/apiserver/controllers/v1/controller.go +++ b/pkg/apiserver/controllers/v1/controller.go @@ -2,17 +2,15 @@ package v1 import ( "context" + "fmt" "net" - //"github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers" - middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csplugin" "github.com/crowdsecurity/crowdsec/pkg/csprofiles" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/models" - "github.com/pkg/errors" ) type Controller struct { @@ -48,7 +46,7 @@ func New(cfg *ControllerV1Config) (*Controller, error) { profiles, err := csprofiles.NewProfile(cfg.ProfilesCfg) if err != nil { - return &Controller{}, errors.Wrapf(err, "failed to compile profiles") + return &Controller{}, fmt.Errorf("failed to compile profiles: %w", err) } v1 := &Controller{ diff --git a/pkg/apiserver/middlewares/v1/api_key.go b/pkg/apiserver/middlewares/v1/api_key.go index 503f4d43d..ce1bc8eee 100644 --- a/pkg/apiserver/middlewares/v1/api_key.go +++ b/pkg/apiserver/middlewares/v1/api_key.go @@ -3,7 +3,7 @@ package v1 import ( "crypto/rand" "crypto/sha512" - "encoding/hex" + "encoding/base64" "fmt" "net/http" "strings" @@ -15,9 +15,11 @@ import ( log "github.com/sirupsen/logrus" ) -var ( +const ( APIKeyHeader = "X-Api-Key" bouncerContextKey = "bouncer_info" + // max allowed by bcrypt 72 = 54 bytes in base64 + dummyAPIKeySize = 54 ) type APIKey struct { @@ -31,7 +33,7 @@ func GenerateAPIKey(n int) (string, error) { if _, err := rand.Read(bytes); err != nil { return "", err } - return hex.EncodeToString(bytes), nil + return base64.StdEncoding.EncodeToString(bytes), nil } func NewAPIKey(dbClient *database.Client) *APIKey { @@ -82,7 +84,7 @@ func (a *APIKey) MiddlewareFunc() gin.HandlerFunc { if err != nil && strings.Contains(err.Error(), "bouncer not found") { //Because we have a valid cert, automatically create the bouncer in the database if it does not exist //Set a random API key, but it will never be used - apiKey, err := GenerateAPIKey(64) + apiKey, err := GenerateAPIKey(dummyAPIKeySize) if err != nil { log.WithFields(log.Fields{ "ip": c.ClientIP(), diff --git a/pkg/apiserver/middlewares/v1/jwt.go b/pkg/apiserver/middlewares/v1/jwt.go index 9f69f332e..bbd33c544 100644 --- a/pkg/apiserver/middlewares/v1/jwt.go +++ b/pkg/apiserver/middlewares/v1/jwt.go @@ -81,7 +81,7 @@ func (j *JWT) Authenticator(c *gin.Context) (interface{}, error) { //Machine was not found, let's create it log.Printf("machine %s not found, create it", machineID) //let's use an apikey as the password, doesn't matter in this case (generatePassword is only available in cscli) - pwd, err := GenerateAPIKey(64) + pwd, err := GenerateAPIKey(dummyAPIKeySize) if err != nil { log.WithFields(log.Fields{ "ip": c.ClientIP(), diff --git a/pkg/apiserver/middlewares/v1/tls_auth.go b/pkg/apiserver/middlewares/v1/tls_auth.go index a0b837a41..87ca896a8 100644 --- a/pkg/apiserver/middlewares/v1/tls_auth.go +++ b/pkg/apiserver/middlewares/v1/tls_auth.go @@ -12,7 +12,6 @@ import ( "time" "github.com/gin-gonic/gin" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/crypto/ocsp" ) @@ -176,9 +175,9 @@ func (ta *TLSAuth) isInvalid(cert *x509.Certificate, issuer *x509.Certificate) ( } revoked, err := ta.isRevoked(cert, issuer) if err != nil { - //Fail securely, if we can't check the revokation status, let's consider the cert invalid + //Fail securely, if we can't check the revocation status, let's consider the cert invalid //We may change this in the future based on users feedback, but this seems the most sensible thing to do - return true, errors.Wrap(err, "could not check for client certification revokation status") + return true, fmt.Errorf("could not check for client certification revocation status: %w", err) } return revoked, nil @@ -231,7 +230,7 @@ func (ta *TLSAuth) ValidateCert(c *gin.Context) (bool, string, error) { revoked, err := ta.isInvalid(clientCert, c.Request.TLS.VerifiedChains[0][1]) if err != nil { ta.logger.Errorf("TLSAuth: error checking if client certificate is revoked: %s", err) - return false, "", errors.Wrap(err, "could not check for client certification revokation status") + return false, "", fmt.Errorf("could not check for client certification revokation status: %w", err) } if revoked { return false, "", fmt.Errorf("client certificate is revoked") diff --git a/pkg/csconfig/common.go b/pkg/csconfig/common.go index 6add00c05..9d80cd95a 100644 --- a/pkg/csconfig/common.go +++ b/pkg/csconfig/common.go @@ -4,7 +4,6 @@ import ( "fmt" "path/filepath" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" ) @@ -39,7 +38,7 @@ func (c *Config) LoadCommon() error { } *k, err = filepath.Abs(*k) if err != nil { - return errors.Wrapf(err, "failed to get absolute path of '%s'", *k) + return fmt.Errorf("failed to get absolute path of '%s': %w", *k, err) } } diff --git a/pkg/csconfig/config_paths.go b/pkg/csconfig/config_paths.go index 59be93ae6..24ff454b7 100644 --- a/pkg/csconfig/config_paths.go +++ b/pkg/csconfig/config_paths.go @@ -3,8 +3,6 @@ package csconfig import ( "fmt" "path/filepath" - - "github.com/pkg/errors" ) type ConfigurationPaths struct { @@ -50,7 +48,7 @@ func (c *Config) LoadConfigurationPaths() error { } *k, err = filepath.Abs(*k) if err != nil { - return errors.Wrapf(err, "failed to get absolute path of '%s'", *k) + return fmt.Errorf("failed to get absolute path of '%s': %w", *k, err) } } diff --git a/pkg/csconfig/profiles.go b/pkg/csconfig/profiles.go index 41725bcf2..ec70fb459 100644 --- a/pkg/csconfig/profiles.go +++ b/pkg/csconfig/profiles.go @@ -2,13 +2,13 @@ package csconfig import ( "bytes" + "errors" "fmt" "io" "github.com/crowdsecurity/go-cs-lib/pkg/yamlpatch" "github.com/crowdsecurity/crowdsec/pkg/models" - "github.com/pkg/errors" "gopkg.in/yaml.v2" ) @@ -53,7 +53,7 @@ func (c *LocalApiServerCfg) LoadProfiles() error { if errors.Is(err, io.EOF) { break } - return errors.Wrapf(err, "while decoding %s", c.ProfilesPath) + return fmt.Errorf("while decoding %s: %w", c.ProfilesPath, err) } c.Profiles = append(c.Profiles, &t) } diff --git a/pkg/csplugin/broker.go b/pkg/csplugin/broker.go index 6bc3b1296..b1bd54dfd 100644 --- a/pkg/csplugin/broker.go +++ b/pkg/csplugin/broker.go @@ -2,6 +2,7 @@ package csplugin import ( "context" + "errors" "fmt" "io" "os" @@ -14,7 +15,6 @@ import ( "github.com/Masterminds/sprig/v3" "github.com/google/uuid" plugin "github.com/hashicorp/go-plugin" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" @@ -83,10 +83,10 @@ func (pb *PluginBroker) Init(pluginCfg *csconfig.PluginCfg, profileConfigs []*cs pb.pluginProcConfig = pluginCfg pb.pluginsTypesToDispatch = make(map[string]struct{}) if err := pb.loadConfig(configPaths.NotificationDir); err != nil { - return errors.Wrap(err, "while loading plugin config") + return fmt.Errorf("while loading plugin config: %w", err) } if err := pb.loadPlugins(configPaths.PluginDir); err != nil { - return errors.Wrap(err, "while loading plugin") + return fmt.Errorf("while loading plugin: %w", err) } pb.watcher = PluginWatcher{} pb.watcher.Init(pb.pluginConfigByName, pb.alertsByPluginName) @@ -268,7 +268,7 @@ func (pb *PluginBroker) loadPlugins(path string) error { data = []byte(csstring.StrictExpand(string(data), os.LookupEnv)) _, err = pluginClient.Configure(context.Background(), &protobufs.Config{Config: data}) if err != nil { - return errors.Wrapf(err, "while configuring %s", pc.Name) + return fmt.Errorf("while configuring %s: %w", pc.Name, err) } log.Infof("registered plugin %s", pc.Name) pb.notificationPluginByName[pc.Name] = pluginClient @@ -354,7 +354,7 @@ func ParsePluginConfigFile(path string) ([]PluginConfig, error) { parsedConfigs := make([]PluginConfig, 0) yamlFile, err := os.Open(path) if err != nil { - return parsedConfigs, errors.Wrapf(err, "while opening %s", path) + return nil, fmt.Errorf("while opening %s: %w", path, err) } dec := yaml.NewDecoder(yamlFile) dec.SetStrict(true) @@ -365,7 +365,7 @@ func ParsePluginConfigFile(path string) ([]PluginConfig, error) { if errors.Is(err, io.EOF) { break } - return []PluginConfig{}, fmt.Errorf("while decoding %s got error %s", path, err) + return nil, fmt.Errorf("while decoding %s got error %s", path, err) } // if the yaml document is empty, skip if reflect.DeepEqual(pc, PluginConfig{}) { diff --git a/pkg/csplugin/broker_suite_test.go b/pkg/csplugin/broker_suite_test.go index 4c7cdd6eb..6e3e51407 100644 --- a/pkg/csplugin/broker_suite_test.go +++ b/pkg/csplugin/broker_suite_test.go @@ -14,7 +14,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) - type PluginSuite struct { suite.Suite @@ -23,21 +22,19 @@ type PluginSuite struct { // full path to the built plugin binary builtBinary string - runDir string // temporary directory for each test - pluginDir string // (config_paths.plugin_dir) - notifDir string // (config_paths.notification_dir) - pluginBinary string // full path to the plugin binary (unique for each test) - pluginConfig string // full path to the notification config (unique for each test) + runDir string // temporary directory for each test + pluginDir string // (config_paths.plugin_dir) + notifDir string // (config_paths.notification_dir) + pluginBinary string // full path to the plugin binary (unique for each test) + pluginConfig string // full path to the notification config (unique for each test) pluginBroker *PluginBroker } - func TestPluginSuite(t *testing.T) { suite.Run(t, new(PluginSuite)) } - func (s *PluginSuite) SetupSuite() { var err error @@ -57,14 +54,12 @@ func (s *PluginSuite) SetupSuite() { require.NoError(t, err, "while building dummy plugin") } - func (s *PluginSuite) TearDownSuite() { t := s.T() err := os.RemoveAll(s.buildDir) require.NoError(t, err) } - func copyFile(src string, dst string) error { s, err := os.Open(src) if err != nil { @@ -99,7 +94,6 @@ func (s *PluginSuite) TearDownTest() { s.TearDownSubTest() } - func (s *PluginSuite) SetupSubTest() { var err error t := s.T() @@ -125,7 +119,7 @@ func (s *PluginSuite) SetupSubTest() { require.NoError(t, err, "while copying built binary") err = os.Chmod(s.pluginBinary, 0o744) require.NoError(t, err, "chmod 0744 %s", s.pluginBinary) - + s.pluginConfig = path.Join(s.notifDir, "dummy.yaml") err = copyFile("testdata/dummy.yaml", s.pluginConfig) require.NoError(t, err, "while copying plugin config") diff --git a/pkg/csplugin/broker_test.go b/pkg/csplugin/broker_test.go index 467fadf45..991b89ed2 100644 --- a/pkg/csplugin/broker_test.go +++ b/pkg/csplugin/broker_test.go @@ -22,7 +22,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/models" ) - func (s *PluginSuite) permissionSetter(perm os.FileMode) func(*testing.T) { return func(t *testing.T) { err := os.Chmod(s.pluginBinary, perm) @@ -30,30 +29,28 @@ func (s *PluginSuite) permissionSetter(perm os.FileMode) func(*testing.T) { } } -func (s *PluginSuite) readconfig() (PluginConfig) { +func (s *PluginSuite) readconfig() PluginConfig { var config PluginConfig t := s.T() orig, err := os.ReadFile(s.pluginConfig) - require.NoError(t, err,"unable to read config file %s", s.pluginConfig) + require.NoError(t, err, "unable to read config file %s", s.pluginConfig) err = yaml.Unmarshal(orig, &config) - require.NoError(t, err,"unable to unmarshal config file") - + require.NoError(t, err, "unable to unmarshal config file") + return config } - func (s *PluginSuite) writeconfig(config PluginConfig) { t := s.T() data, err := yaml.Marshal(&config) - require.NoError(t, err,"unable to marshal config file") + require.NoError(t, err, "unable to marshal config file") err = os.WriteFile(s.pluginConfig, data, 0644) - require.NoError(t, err,"unable to write config file %s", s.pluginConfig) + require.NoError(t, err, "unable to write config file %s", s.pluginConfig) } - func (s *PluginSuite) TestBrokerInit() { tests := []struct { name string @@ -62,7 +59,7 @@ func (s *PluginSuite) TestBrokerInit() { expectedErr string }{ { - name: "valid config", + name: "valid config", }, { name: "group writable binary", @@ -349,7 +346,7 @@ func (s *PluginSuite) TestBrokerRunSimple() { DefaultEmptyTicker = 50 * time.Millisecond t := s.T() - + pb, err := s.InitBroker(nil) assert.NoError(t, err) diff --git a/pkg/csplugin/broker_win_test.go b/pkg/csplugin/broker_win_test.go index 3d7498c0d..01262b1fd 100644 --- a/pkg/csplugin/broker_win_test.go +++ b/pkg/csplugin/broker_win_test.go @@ -33,7 +33,7 @@ func (s *PluginSuite) TestBrokerInit() { expectedErr string }{ { - name: "valid config", + name: "valid config", }, { name: "no plugin dir", diff --git a/pkg/csplugin/listfiles.go b/pkg/csplugin/listfiles.go index 2dea44f4f..c91be03b4 100644 --- a/pkg/csplugin/listfiles.go +++ b/pkg/csplugin/listfiles.go @@ -13,10 +13,9 @@ func listFilesAtPath(path string) ([]string, error) { return nil, err } for _, file := range files { - if ! file.IsDir() { + if !file.IsDir() { filePaths = append(filePaths, filepath.Join(path, file.Name())) } } return filePaths, nil } - diff --git a/pkg/csplugin/listfiles_test.go b/pkg/csplugin/listfiles_test.go index 8bcedaa1f..09102ef0d 100644 --- a/pkg/csplugin/listfiles_test.go +++ b/pkg/csplugin/listfiles_test.go @@ -27,9 +27,9 @@ func TestListFilesAtPath(t *testing.T) { require.NoError(t, err) tests := []struct { - name string - path string - want []string + name string + path string + want []string expectedErr string }{ { @@ -41,8 +41,8 @@ func TestListFilesAtPath(t *testing.T) { }, }, { - name: "invalid directory", - path: "./foo/bar/", + name: "invalid directory", + path: "./foo/bar/", expectedErr: "open ./foo/bar/: " + cstest.PathNotFoundMessage, }, } diff --git a/pkg/csplugin/notifier.go b/pkg/csplugin/notifier.go index 64a1e6e71..8ab1aa923 100644 --- a/pkg/csplugin/notifier.go +++ b/pkg/csplugin/notifier.go @@ -4,9 +4,10 @@ import ( "context" "fmt" - "github.com/crowdsecurity/crowdsec/pkg/protobufs" plugin "github.com/hashicorp/go-plugin" "google.golang.org/grpc" + + "github.com/crowdsecurity/crowdsec/pkg/protobufs" ) type Notifier interface { diff --git a/pkg/csplugin/utils.go b/pkg/csplugin/utils.go index 67707c829..5e19dee38 100644 --- a/pkg/csplugin/utils.go +++ b/pkg/csplugin/utils.go @@ -3,6 +3,7 @@ package csplugin import ( + "errors" "fmt" "io/fs" "math" @@ -13,8 +14,6 @@ import ( "strconv" "strings" "syscall" - - "github.com/pkg/errors" ) func CheckCredential(uid int, gid int) *syscall.SysProcAttr { @@ -35,7 +34,7 @@ func (pb *PluginBroker) CreateCmd(binaryPath string) (*exec.Cmd, error) { } cmd.SysProcAttr, err = getProcessAttr(pb.pluginProcConfig.User, pb.pluginProcConfig.Group) if err != nil { - return nil, errors.Wrap(err, "while getting process attributes") + return nil, fmt.Errorf("while getting process attributes: %w", err) } cmd.SysProcAttr.Credential.NoSetGroups = true } @@ -105,17 +104,17 @@ func pluginIsValid(path string) error { // check if it exists if details, err = os.Stat(path); err != nil { - return errors.Wrap(err, fmt.Sprintf("plugin at %s does not exist", path)) + return fmt.Errorf("plugin at %s does not exist: %w", path, err) } // check if it is owned by current user currentUser, err := user.Current() if err != nil { - return errors.Wrap(err, "while getting current user") + return fmt.Errorf("while getting current user: %w", err) } currentUID, err := getUID(currentUser.Username) if err != nil { - return errors.Wrap(err, "while looking up the current uid") + return fmt.Errorf("while looking up the current uid: %w", err) } stat := details.Sys().(*syscall.Stat_t) if stat.Uid != currentUID { diff --git a/pkg/csplugin/utils_windows.go b/pkg/csplugin/utils_windows.go index 874e30021..dfb11aff5 100644 --- a/pkg/csplugin/utils_windows.go +++ b/pkg/csplugin/utils_windows.go @@ -13,7 +13,6 @@ import ( "syscall" "unsafe" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/sys/windows" ) @@ -54,38 +53,38 @@ func CheckPerms(path string) error { systemSid, err := windows.CreateWellKnownSid(windows.WELL_KNOWN_SID_TYPE(windows.WinLocalSystemSid)) if err != nil { - return errors.Wrap(err, "while creating SYSTEM well known sid") + return fmt.Errorf("while creating SYSTEM well known sid: %w", err) } adminSid, err := windows.CreateWellKnownSid(windows.WELL_KNOWN_SID_TYPE(windows.WinBuiltinAdministratorsSid)) if err != nil { - return errors.Wrap(err, "while creating built-in Administrators well known sid") + return fmt.Errorf("while creating built-in Administrators well known sid: %w", err) } currentUser, err := user.Current() if err != nil { - return errors.Wrap(err, "while getting current user") + return fmt.Errorf("while getting current user: %w", err) } currentUserSid, _, _, err := windows.LookupSID("", currentUser.Username) if err != nil { - return errors.Wrap(err, "while looking up current user sid") + return fmt.Errorf("while looking up current user sid: %w", err) } sd, err := windows.GetNamedSecurityInfo(path, windows.SE_FILE_OBJECT, windows.OWNER_SECURITY_INFORMATION|windows.DACL_SECURITY_INFORMATION) if err != nil { - return errors.Wrap(err, "while getting owner security info") + return fmt.Errorf("while getting owner security info: %w", err) } if !sd.IsValid() { - return errors.New("security descriptor is invalid") + return fmt.Errorf("security descriptor is invalid") } owner, _, err := sd.Owner() if err != nil { - return errors.Wrap(err, "while getting owner") + return fmt.Errorf("while getting owner: %w", err) } if !owner.IsValid() { - return errors.New("owner is invalid") + return fmt.Errorf("owner is invalid") } if !owner.Equals(systemSid) && !owner.Equals(currentUserSid) && !owner.Equals(adminSid) { @@ -94,7 +93,7 @@ func CheckPerms(path string) error { dacl, _, err := sd.DACL() if err != nil { - return errors.Wrap(err, "while getting DACL") + return fmt.Errorf("while getting DACL: %w", err) } if dacl == nil { @@ -102,7 +101,7 @@ func CheckPerms(path string) error { } if err != nil { - return errors.Wrap(err, "while looking up current user sid") + return fmt.Errorf("while looking up current user sid: %w", err) } rs := reflect.ValueOf(dacl).Elem() @@ -124,7 +123,7 @@ func CheckPerms(path string) error { ace := &AccessAllowedAce{} ret, _, _ := procGetAce.Call(uintptr(unsafe.Pointer(dacl)), uintptr(i), uintptr(unsafe.Pointer(&ace))) if ret == 0 { - return errors.Wrap(windows.GetLastError(), "while getting ACE") + return fmt.Errorf("while getting ACE: %w", windows.GetLastError()) } log.Debugf("ACE %d: %+v\n", i, ace) @@ -162,14 +161,14 @@ func getProcessAtr() (*syscall.SysProcAttr, error) { err := windows.OpenProcessToken(proc, windows.TOKEN_DUPLICATE|windows.TOKEN_ADJUST_DEFAULT| windows.TOKEN_QUERY|windows.TOKEN_ASSIGN_PRIMARY|windows.TOKEN_ADJUST_GROUPS|windows.TOKEN_ADJUST_PRIVILEGES, &procToken) if err != nil { - return nil, errors.Wrapf(err, "while opening process token") + return nil, fmt.Errorf("while opening process token: %w", err) } defer procToken.Close() err = windows.DuplicateTokenEx(procToken, 0, nil, windows.SecurityImpersonation, windows.TokenPrimary, &token) if err != nil { - return nil, errors.Wrapf(err, "while duplicating token") + return nil, fmt.Errorf("while duplicating token: %w", err) } //Remove all privileges from the token @@ -177,7 +176,7 @@ func getProcessAtr() (*syscall.SysProcAttr, error) { err = windows.AdjustTokenPrivileges(token, true, nil, 0, nil, nil) if err != nil { - return nil, errors.Wrapf(err, "while adjusting token privileges") + return nil, fmt.Errorf("while adjusting token privileges: %w", err) } //Run the plugin as a medium integrity level process @@ -195,7 +194,7 @@ func getProcessAtr() (*syscall.SysProcAttr, error) { (*byte)(unsafe.Pointer(tml)), tml.Size()) if err != nil { token.Close() - return nil, errors.Wrapf(err, "while setting token information") + return nil, fmt.Errorf("while setting token information: %w", err) } return &windows.SysProcAttr{ @@ -209,7 +208,7 @@ func (pb *PluginBroker) CreateCmd(binaryPath string) (*exec.Cmd, error) { cmd := exec.Command(binaryPath) cmd.SysProcAttr, err = getProcessAtr() if err != nil { - return nil, errors.Wrap(err, "while getting process attributes") + return nil, fmt.Errorf("while getting process attributes: %w", err) } return cmd, err } @@ -229,7 +228,7 @@ func pluginIsValid(path string) error { // check if it exists if _, err = os.Stat(path); err != nil { - return errors.Wrap(err, fmt.Sprintf("plugin at %s does not exist", path)) + return fmt.Errorf("plugin at %s does not exist", path) } // check if it is owned by root diff --git a/pkg/csplugin/watcher.go b/pkg/csplugin/watcher.go index 983a53c89..bec0302e4 100644 --- a/pkg/csplugin/watcher.go +++ b/pkg/csplugin/watcher.go @@ -4,9 +4,10 @@ import ( "sync" "time" - "github.com/crowdsecurity/crowdsec/pkg/models" log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" + + "github.com/crowdsecurity/crowdsec/pkg/models" ) /* diff --git a/pkg/csplugin/watcher_test.go b/pkg/csplugin/watcher_test.go index 94d8d0617..391a94810 100644 --- a/pkg/csplugin/watcher_test.go +++ b/pkg/csplugin/watcher_test.go @@ -7,9 +7,12 @@ import ( "testing" "time" - "github.com/crowdsecurity/crowdsec/pkg/models" + "github.com/stretchr/testify/require" "gopkg.in/tomb.v2" - "gotest.tools/v3/assert" + + "github.com/crowdsecurity/go-cs-lib/pkg/cstest" + + "github.com/crowdsecurity/crowdsec/pkg/models" ) var ctx = context.Background() @@ -64,7 +67,7 @@ func TestPluginWatcherInterval(t *testing.T) { ct, cancel := context.WithTimeout(ctx, time.Microsecond) defer cancel() err := listenChannelWithTimeout(ct, pw.PluginEvents) - assert.ErrorContains(t, err, "context deadline exceeded") + cstest.RequireErrorContains(t, err, "context deadline exceeded") resetTestTomb(&testTomb, &pw) testTomb = tomb.Tomb{} pw.Start(&testTomb) @@ -72,7 +75,7 @@ func TestPluginWatcherInterval(t *testing.T) { ct, cancel = context.WithTimeout(ctx, time.Millisecond*5) defer cancel() err = listenChannelWithTimeout(ct, pw.PluginEvents) - assert.NilError(t, err) + require.NoError(t, err) resetTestTomb(&testTomb, &pw) // This is to avoid the int complaining } @@ -96,7 +99,7 @@ func TestPluginAlertCountWatcher(t *testing.T) { ct, cancel := context.WithTimeout(ctx, time.Second) defer cancel() err := listenChannelWithTimeout(ct, pw.PluginEvents) - assert.ErrorContains(t, err, "context deadline exceeded") + cstest.RequireErrorContains(t, err, "context deadline exceeded") // Channel won't contain any events since threshold is not crossed. resetWatcherAlertCounter(&pw) @@ -104,7 +107,7 @@ func TestPluginAlertCountWatcher(t *testing.T) { ct, cancel = context.WithTimeout(ctx, time.Second) defer cancel() err = listenChannelWithTimeout(ct, pw.PluginEvents) - assert.ErrorContains(t, err, "context deadline exceeded") + cstest.RequireErrorContains(t, err, "context deadline exceeded") // Channel will contain an event since threshold is crossed. resetWatcherAlertCounter(&pw) @@ -112,6 +115,6 @@ func TestPluginAlertCountWatcher(t *testing.T) { ct, cancel = context.WithTimeout(ctx, time.Second) defer cancel() err = listenChannelWithTimeout(ct, pw.PluginEvents) - assert.NilError(t, err) + require.NoError(t, err) resetTestTomb(&testTomb, &pw) } diff --git a/pkg/csprofiles/csprofiles.go b/pkg/csprofiles/csprofiles.go index 29e6cdf36..70dea3d7a 100644 --- a/pkg/csprofiles/csprofiles.go +++ b/pkg/csprofiles/csprofiles.go @@ -6,12 +6,13 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" ) type Runtime struct { @@ -47,10 +48,10 @@ func NewProfile(profilesCfg []*csconfig.ProfileCfg) ([]*Runtime, error) { runtime.DebugFilters = make([]*exprhelpers.ExprDebugger, len(profile.Filters)) runtime.Cfg = profile if runtime.Cfg.OnSuccess != "" && runtime.Cfg.OnSuccess != "continue" && runtime.Cfg.OnSuccess != "break" { - return []*Runtime{}, errors.Wrapf(err, "invalid 'on_success' for '%s' : %s", profile.Name, runtime.Cfg.OnSuccess) + return []*Runtime{}, fmt.Errorf("invalid 'on_success' for '%s': %s", profile.Name, runtime.Cfg.OnSuccess) } if runtime.Cfg.OnFailure != "" && runtime.Cfg.OnFailure != "continue" && runtime.Cfg.OnFailure != "break" && runtime.Cfg.OnFailure != "apply" { - return []*Runtime{}, errors.Wrapf(err, "invalid 'on_failure' for '%s' : %s", profile.Name, runtime.Cfg.OnFailure) + return []*Runtime{}, fmt.Errorf("invalid 'on_failure' for '%s' : %s", profile.Name, runtime.Cfg.OnFailure) } for fIdx, filter := range profile.Filters { diff --git a/pkg/csprofiles/csprofiles_test.go b/pkg/csprofiles/csprofiles_test.go index 81b21e1fb..8adf68291 100644 --- a/pkg/csprofiles/csprofiles_test.go +++ b/pkg/csprofiles/csprofiles_test.go @@ -5,10 +5,11 @@ import ( "reflect" "testing" + "github.com/stretchr/testify/require" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/models" - "gotest.tools/v3/assert" ) var ( @@ -95,7 +96,7 @@ func TestNewProfile(t *testing.T) { } profile, _ := NewProfile(profilesCfg) fmt.Printf("expected : %+v | result : %+v", test.expectedNbProfile, len(profile)) - assert.Equal(t, test.expectedNbProfile, len(profile)) + require.Len(t, profile, test.expectedNbProfile) }) } } @@ -199,7 +200,7 @@ func TestEvaluateProfile(t *testing.T) { t.Errorf("EvaluateProfile() got1 = %v, want %v", got1, tt.expectedMatchStatus) } if tt.expectedDuration != "" { - assert.Equal(t, tt.expectedDuration, *got[0].Duration, "The two durations should be the same") + require.Equal(t, tt.expectedDuration, *got[0].Duration, "The two durations should be the same") } }) } diff --git a/pkg/cticlient/example/fire.go b/pkg/cticlient/example/fire.go index 7bcf814a0..e52922571 100644 --- a/pkg/cticlient/example/fire.go +++ b/pkg/cticlient/example/fire.go @@ -44,6 +44,9 @@ func main() { } for _, item := range items { + if item.State == "refused" { + continue + } banDuration := time.Until(item.Expiration.Time) allItems = append(allItems, []string{ item.Ip, diff --git a/pkg/cticlient/types.go b/pkg/cticlient/types.go index 1d6550d00..2ad0a6eb3 100644 --- a/pkg/cticlient/types.go +++ b/pkg/cticlient/types.go @@ -120,7 +120,7 @@ type FireItem struct { BackgroundNoiseScore *int `json:"background_noise_score"` Scores CTIScores `json:"scores"` References []CTIReferences `json:"references"` - Status string `json:"status"` + State string `json:"state"` Expiration CustomTime `json:"expiration"` } diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index 0e508d9f1..b8a09d4c1 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -148,7 +148,7 @@ func GetItemByPath(itemType string, itemPath string) (*Item, error) { finalName := "" f, err := os.Lstat(itemPath) if err != nil { - return nil, errors.Wrapf(err, "while performing lstat on %s", itemPath) + return nil, fmt.Errorf("while performing lstat on %s: %w", itemPath, err) } if f.Mode()&os.ModeSymlink == 0 { @@ -158,7 +158,7 @@ func GetItemByPath(itemType string, itemPath string) (*Item, error) { /*resolve the symlink to hub file*/ pathInHub, err := os.Readlink(itemPath) if err != nil { - return nil, errors.Wrapf(err, "while reading symlink of %s", itemPath) + return nil, fmt.Errorf("while reading symlink of %s: %w", itemPath, err) } //extract author from path fname := filepath.Base(pathInHub) @@ -240,7 +240,7 @@ func GetInstalledScenariosAsString() ([]string, error) { items, err := GetInstalledScenarios() if err != nil { - return nil, errors.Wrap(err, "while fetching scenarios") + return nil, fmt.Errorf("while fetching scenarios: %w", err) } for _, it := range items { retStr = append(retStr, it.Name) @@ -281,7 +281,7 @@ func GetInstalledParsersAsString() ([]string, error) { items, err := GetInstalledParsers() if err != nil { - return nil, errors.Wrap(err, "while fetching parsers") + return nil, fmt.Errorf("while fetching parsers: %w", err) } for _, it := range items { retStr = append(retStr, it.Name) @@ -308,7 +308,7 @@ func GetInstalledPostOverflowsAsString() ([]string, error) { items, err := GetInstalledPostOverflows() if err != nil { - return nil, errors.Wrap(err, "while fetching post overflows") + return nil, fmt.Errorf("while fetching post overflows: %w", err) } for _, it := range items { retStr = append(retStr, it.Name) @@ -321,8 +321,9 @@ func GetInstalledCollectionsAsString() ([]string, error) { items, err := GetInstalledCollections() if err != nil { - return nil, errors.Wrap(err, "while fetching collections") + return nil, fmt.Errorf("while fetching collections: %w", err) } + for _, it := range items { retStr = append(retStr, it.Name) } diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index b0e300aa7..f91b0dced 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -25,7 +25,7 @@ import ( var responseByPath map[string]string func TestItemStatus(t *testing.T) { - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) err := UpdateHubIdx(cfg.Hub) @@ -73,7 +73,7 @@ func TestItemStatus(t *testing.T) { } func TestGetters(t *testing.T) { - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) err := UpdateHubIdx(cfg.Hub) @@ -134,7 +134,7 @@ func TestGetters(t *testing.T) { } func TestIndexDownload(t *testing.T) { - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) err := UpdateHubIdx(cfg.Hub) @@ -155,10 +155,16 @@ func getTestCfg() (cfg *csconfig.Config) { return } -func envSetup() *csconfig.Config { +func envSetup(t *testing.T) *csconfig.Config { resetResponseByPath() log.SetLevel(log.DebugLevel) cfg := getTestCfg() + + defaultTransport := http.DefaultClient.Transport + t.Cleanup(func() { + http.DefaultClient.Transport = defaultTransport + }) + //Mock the http client http.DefaultClient.Transport = newMockTransport() @@ -321,7 +327,7 @@ func TestInstallParser(t *testing.T) { - check its status - remove it */ - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) getHubIdxOrFail(t) @@ -353,7 +359,7 @@ func TestInstallCollection(t *testing.T) { - check its status - remove it */ - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) getHubIdxOrFail(t) diff --git a/pkg/types/dataset.go b/pkg/cwhub/dataset.go similarity index 70% rename from pkg/types/dataset.go rename to pkg/cwhub/dataset.go index 2684342a9..848686be6 100644 --- a/pkg/types/dataset.go +++ b/pkg/cwhub/dataset.go @@ -1,4 +1,4 @@ -package types +package cwhub import ( "fmt" @@ -6,24 +6,14 @@ import ( "net/http" "os" "path" - "time" log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/crowdsec/pkg/types" ) -type DataSource struct { - SourceURL string `yaml:"source_url"` - DestPath string `yaml:"dest_file"` - Type string `yaml:"type"` - //Control cache strategy on expensive regexps - Cache *bool `yaml:"cache"` - Strategy *string `yaml:"strategy"` - Size *int `yaml:"size"` - TTL *time.Duration `yaml:"ttl"` -} - type DataSet struct { - Data []*DataSource `yaml:"data,omitempty"` + Data []*types.DataSource `yaml:"data,omitempty"` } func downloadFile(url string, destPath string) error { @@ -66,7 +56,7 @@ func downloadFile(url string, destPath string) error { return nil } -func GetData(data []*DataSource, dataDir string) error { +func GetData(data []*types.DataSource, dataDir string) error { for _, dataS := range data { destPath := path.Join(dataDir, dataS.DestPath) log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath) diff --git a/pkg/types/dataset_test.go b/pkg/cwhub/dataset_test.go similarity index 94% rename from pkg/types/dataset_test.go rename to pkg/cwhub/dataset_test.go index 956e3316f..106268c01 100644 --- a/pkg/types/dataset_test.go +++ b/pkg/cwhub/dataset_test.go @@ -1,4 +1,4 @@ -package types +package cwhub import ( "os" @@ -9,7 +9,7 @@ import ( "github.com/jarcoal/httpmock" ) -func TestDownladFile(t *testing.T) { +func TestDownloadFile(t *testing.T) { examplePath := "./example.txt" defer os.Remove(examplePath) diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go index 6923c6d05..17b5c0e00 100644 --- a/pkg/cwhub/download.go +++ b/pkg/cwhub/download.go @@ -12,7 +12,6 @@ import ( "strings" "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" @@ -256,7 +255,7 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error { dec := yaml.NewDecoder(reader) for { - data := &types.DataSet{} + data := &DataSet{} err = dec.Decode(data) if err != nil { if errors.Is(err, io.EOF) { @@ -272,7 +271,7 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error { } } if download || force { - err = types.GetData(data.Data, dataFolder) + err = GetData(data.Data, dataFolder) if err != nil { return errors.Wrap(err, "while getting data") } diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index af1e938d7..4133e2272 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -4,12 +4,12 @@ import ( "fmt" "path/filepath" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/enescakir/emoji" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/mod/semver" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/cwversion" ) // pick a hub branch corresponding to the current crowdsec version. @@ -79,11 +79,11 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo item, err := DownloadLatest(csConfig.Hub, item, force, true) if err != nil { - return errors.Wrapf(err, "while downloading %s", item.Name) + return fmt.Errorf("while downloading %s: %w", item.Name, err) } if err := AddItem(obtype, item); err != nil { - return errors.Wrapf(err, "while adding %s", item.Name) + return fmt.Errorf("while adding %s: %w", item.Name, err) } if downloadOnly { @@ -93,11 +93,11 @@ func InstallItem(csConfig *csconfig.Config, name string, obtype string, force bo item, err = EnableItem(csConfig.Hub, item) if err != nil { - return errors.Wrapf(err, "while enabling %s", item.Name) + return fmt.Errorf("while enabling %s: %w", item.Name, err) } if err := AddItem(obtype, item); err != nil { - return errors.Wrapf(err, "while adding %s", item.Name) + return fmt.Errorf("while adding %s: %w", item.Name, err) } log.Infof("Enabled %s", item.Name) diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index 143967002..bf6e84fb3 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -9,7 +9,7 @@ import ( //Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection // We expect the new scenario to be installed func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) // fresh install of collection @@ -55,7 +55,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { // Install a collection, disable a scenario. // Upgrade should install should not enable/download the disabled scenario. func TestUpgradeConfigInDisabledSceanarioShouldNotBeInstalled(t *testing.T) { - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) // fresh install of collection @@ -103,7 +103,7 @@ func getHubIdxOrFail(t *testing.T) { // Upgrade should not enable/download the disabled scenario. // Upgrade should install and enable the newly added scenario. func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) { - cfg := envSetup() + cfg := envSetup(t) defer envTearDown(cfg) // fresh install of collection diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index 961e9a02e..ecb5d0a3a 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -2,16 +2,17 @@ package cwhub import ( "encoding/json" + "errors" "fmt" "os" "path/filepath" "sort" "strings" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "golang.org/x/mod/semver" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) /*the walk/parser_visit function can't receive extra args*/ @@ -368,7 +369,7 @@ func GetHubIdx(hub *csconfig.Hub) error { log.Debugf("loading hub idx %s", hub.HubIndexFile) bidx, err := os.ReadFile(hub.HubIndexFile) if err != nil { - return errors.Wrap(err, "unable to read index file") + return fmt.Errorf("unable to read index file: %w", err) } ret, err := LoadPkgIndex(bidx) if err != nil { diff --git a/pkg/database/alerts.go b/pkg/database/alerts.go index ad0117236..688288cee 100644 --- a/pkg/database/alerts.go +++ b/pkg/database/alerts.go @@ -9,6 +9,10 @@ import ( "strings" "time" + "github.com/davecgh/go-spew/spew" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/alert" @@ -20,9 +24,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/database/ent/predicate" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/davecgh/go-spew/spew" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" ) const ( @@ -201,7 +202,7 @@ func (c *Client) CreateOrUpdateAlert(machineID string, alertItem *models.Alert) if strings.ToLower(*decisionItem.Scope) == "ip" || strings.ToLower(*decisionItem.Scope) == "range" { sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(*decisionItem.Value) if err != nil { - return "", errors.Wrapf(ParseDurationFail, "invalid addr/range %s : %s", *decisionItem.Value, err) + return "", errors.Wrapf(InvalidIPOrRange, "invalid addr/range %s : %s", *decisionItem.Value, err) } } decisionDuration, err := time.ParseDuration(*decisionItem.Duration) @@ -391,7 +392,7 @@ func (c *Client) UpdateCommunityBlocklist(alertItem *models.Alert) (int, int, in if rollbackErr != nil { log.Errorf("rollback error: %s", rollbackErr) } - return 0, 0, 0, errors.Wrapf(ParseDurationFail, "invalid addr/range %s : %s", *decisionItem.Value, err) + return 0, 0, 0, errors.Wrapf(InvalidIPOrRange, "invalid addr/range %s : %s", *decisionItem.Value, err) } } /*bulk insert some new decisions*/ @@ -564,7 +565,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ } marshallMetas, err := json.Marshal(eventItem.Meta) if err != nil { - return []string{}, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) + return nil, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) } //the serialized field is too big, let's try to progressively strip it @@ -582,7 +583,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ marshallMetas, err = json.Marshal(eventItem.Meta) if err != nil { - return []string{}, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) + return nil, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err) } if event.SerializedValidator(string(marshallMetas)) == nil { valid = true @@ -611,7 +612,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ } events, err = c.Ent.Event.CreateBulk(eventBulk...).Save(c.CTX) if err != nil { - return []string{}, errors.Wrapf(BulkError, "creating alert events: %s", err) + return nil, errors.Wrapf(BulkError, "creating alert events: %s", err) } } @@ -624,7 +625,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ } metas, err = c.Ent.Meta.CreateBulk(metaBulk...).Save(c.CTX) if err != nil { - return []string{}, errors.Wrapf(BulkError, "creating alert meta: %s", err) + return nil, errors.Wrapf(BulkError, "creating alert meta: %s", err) } } @@ -637,14 +638,14 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ duration, err := time.ParseDuration(*decisionItem.Duration) if err != nil { - return []string{}, errors.Wrapf(ParseDurationFail, "decision duration '%+v' : %s", *decisionItem.Duration, err) + return nil, errors.Wrapf(ParseDurationFail, "decision duration '%+v' : %s", *decisionItem.Duration, err) } /*if the scope is IP or Range, convert the value to integers */ if strings.ToLower(*decisionItem.Scope) == "ip" || strings.ToLower(*decisionItem.Scope) == "range" { sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(*decisionItem.Value) if err != nil { - return []string{}, errors.Wrapf(ParseDurationFail, "invalid addr/range %s : %s", *decisionItem.Value, err) + return nil, fmt.Errorf("%s: %w", *decisionItem.Value, InvalidIPOrRange) } } @@ -667,7 +668,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ if len(decisionBulk) == decisionBulkSize { decisionsCreateRet, err := c.Ent.Decision.CreateBulk(decisionBulk...).Save(c.CTX) if err != nil { - return []string{}, errors.Wrapf(BulkError, "creating alert decisions: %s", err) + return nil, errors.Wrapf(BulkError, "creating alert decisions: %s", err) } decisions = append(decisions, decisionsCreateRet...) @@ -680,7 +681,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ } decisionsCreateRet, err := c.Ent.Decision.CreateBulk(decisionBulk...).Save(c.CTX) if err != nil { - return []string{}, errors.Wrapf(BulkError, "creating alert decisions: %s", err) + return nil, errors.Wrapf(BulkError, "creating alert decisions: %s", err) } decisions = append(decisions, decisionsCreateRet...) } @@ -719,7 +720,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ if len(bulk) == bulkSize { alerts, err := c.Ent.Alert.CreateBulk(bulk...).Save(c.CTX) if err != nil { - return []string{}, errors.Wrapf(BulkError, "bulk creating alert : %s", err) + return nil, errors.Wrapf(BulkError, "bulk creating alert : %s", err) } for alertIndex, a := range alerts { ret = append(ret, strconv.Itoa(a.ID)) @@ -728,7 +729,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ for _, d2 := range decisionsChunk { _, err := c.Ent.Alert.Update().Where(alert.IDEQ(a.ID)).AddDecisions(d2...).Save(c.CTX) if err != nil { - return []string{}, fmt.Errorf("error while updating decisions: %s", err) + return nil, fmt.Errorf("error while updating decisions: %s", err) } } } @@ -744,7 +745,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ alerts, err := c.Ent.Alert.CreateBulk(bulk...).Save(c.CTX) if err != nil { - return []string{}, errors.Wrapf(BulkError, "leftovers creating alert : %s", err) + return nil, errors.Wrapf(BulkError, "leftovers creating alert : %s", err) } for alertIndex, a := range alerts { @@ -754,7 +755,7 @@ func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([ for _, d2 := range decisionsChunk { _, err := c.Ent.Alert.Update().Where(alert.IDEQ(a.ID)).AddDecisions(d2...).Save(c.CTX) if err != nil { - return []string{}, fmt.Errorf("error while updating decisions: %s", err) + return nil, fmt.Errorf("error while updating decisions: %s", err) } } } diff --git a/pkg/database/bouncers.go b/pkg/database/bouncers.go index 4cd32d839..98bfd4587 100644 --- a/pkg/database/bouncers.go +++ b/pkg/database/bouncers.go @@ -4,9 +4,10 @@ import ( "fmt" "time" + "github.com/pkg/errors" + "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/bouncer" - "github.com/pkg/errors" ) func (c *Client) SelectBouncer(apiKeyHash string) (*ent.Bouncer, error) { diff --git a/pkg/database/config.go b/pkg/database/config.go index 90a85490f..8c3578ad5 100644 --- a/pkg/database/config.go +++ b/pkg/database/config.go @@ -1,9 +1,10 @@ package database import ( + "github.com/pkg/errors" + "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/configitem" - "github.com/pkg/errors" ) func (c *Client) GetConfigItem(key string) (*string, error) { diff --git a/pkg/database/decisions.go b/pkg/database/decisions.go index 61483f9fd..5dacc23d4 100644 --- a/pkg/database/decisions.go +++ b/pkg/database/decisions.go @@ -2,17 +2,17 @@ package database import ( "fmt" + "strconv" "strings" "time" - "strconv" - "entgo.io/ent/dialect/sql" + "github.com/pkg/errors" + "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/decision" "github.com/crowdsecurity/crowdsec/pkg/database/ent/predicate" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/pkg/errors" ) type DecisionsByScenario struct { diff --git a/pkg/database/machines.go b/pkg/database/machines.go index 48243324d..7a010fbfb 100644 --- a/pkg/database/machines.go +++ b/pkg/database/machines.go @@ -5,12 +5,12 @@ import ( "time" "github.com/go-openapi/strfmt" + "github.com/pkg/errors" + "golang.org/x/crypto/bcrypt" "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/machine" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/pkg/errors" - "golang.org/x/crypto/bcrypt" ) const CapiMachineID = types.CAPIOrigin diff --git a/pkg/exprhelpers/jsonextract.go b/pkg/exprhelpers/jsonextract.go index a874122ff..a616588a7 100644 --- a/pkg/exprhelpers/jsonextract.go +++ b/pkg/exprhelpers/jsonextract.go @@ -175,8 +175,8 @@ func UnmarshalJSON(params ...any) (any, error) { err := json.Unmarshal([]byte(jsonBlob), &out) if err != nil { log.Errorf("UnmarshalJSON : %s", err) - return "", err + return nil, err } target[key] = out - return "", nil + return nil, nil } diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index 36415f746..c1aa4251c 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -7,7 +7,6 @@ import ( "path/filepath" "github.com/crowdsecurity/crowdsec/pkg/cwhub" - "github.com/pkg/errors" ) type HubTest struct { @@ -105,7 +104,7 @@ func (h *HubTest) LoadAllTests() error { for _, f := range testsFolder { if f.IsDir() { if _, err := h.LoadTestItem(f.Name()); err != nil { - return errors.Wrapf(err, "while loading %s", f.Name()) + return fmt.Errorf("while loading %s: %w", f.Name(), err) } } } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index c3e842bbb..1ec7c5f44 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -10,7 +10,6 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/parser" - "github.com/crowdsecurity/crowdsec/pkg/types" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" ) @@ -23,7 +22,7 @@ type HubTestItemConfig struct { LogType string `yaml:"log_type"` Labels map[string]string `yaml:"labels"` IgnoreParsers bool `yaml:"ignore_parsers"` // if we test a scenario, we don't want to assert on Parser - OverrideStatics []types.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00 + OverrideStatics []parser.ExtraField `yaml:"override_statics"` //Allow to override statics. Executed before s00 } type HubIndex struct { diff --git a/pkg/hubtest/parser_assert.go b/pkg/hubtest/parser_assert.go index 3d52f37e5..95400b50d 100644 --- a/pkg/hubtest/parser_assert.go +++ b/pkg/hubtest/parser_assert.go @@ -12,14 +12,14 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" - "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" - "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/enescakir/emoji" "github.com/fatih/color" - "github.com/pkg/errors" diff "github.com/r3labs/diff/v2" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" + + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/types" ) type AssertFail struct { @@ -164,7 +164,7 @@ func (p *ParserAssert) RunExpression(expression string) (interface{}, error) { if err != nil { log.Warningf("running : %s", expression) log.Warningf("runtime error : %s", err) - return output, errors.Wrapf(err, "while running expression %s", expression) + return output, fmt.Errorf("while running expression %s: %w", expression, err) } return output, nil } diff --git a/pkg/hubtest/scenario_assert.go b/pkg/hubtest/scenario_assert.go index d9ec4dddc..2e2a4e9c8 100644 --- a/pkg/hubtest/scenario_assert.go +++ b/pkg/hubtest/scenario_assert.go @@ -11,11 +11,11 @@ import ( "github.com/antonmedv/expr" "github.com/antonmedv/expr/vm" - "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" - "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/pkg/errors" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" + + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/types" ) type ScenarioAssert struct { @@ -149,7 +149,7 @@ func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) { env := map[string]interface{}{"results": *s.TestData} if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil { - return output, err + return nil, err } // if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(env)); err != nil { // log.Warningf("Failed building debugher for %s : %s", assert, err) @@ -162,7 +162,7 @@ func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) { if err != nil { log.Warningf("running : %s", expression) log.Warningf("runtime error : %s", err) - return output, errors.Wrapf(err, "while running expression %s", expression) + return nil, fmt.Errorf("while running expression %s: %w", expression, err) } return output, nil } diff --git a/pkg/leakybucket/README.md b/pkg/leakybucket/README.md index 5254f33b2..4614eddba 100644 --- a/pkg/leakybucket/README.md +++ b/pkg/leakybucket/README.md @@ -2,86 +2,102 @@ ## Bucket concepts -Leakybucket is used for decision making. Under certain conditions -enriched events are poured in these buckets. When these buckets are +The Leakybucket is used for decision making. Under certain conditions, +enriched events are poured into these buckets. When these buckets are full, we raise a new event. After this event is raised the bucket is destroyed. There are many types of buckets, and we welcome any new useful design of buckets. -Usually the bucket configuration generates the creation of many -buckets. They are differenciated by a field called stackkey. When two -events arrives with the same stackkey they go in the same matching +Usually, the bucket configuration generates the creation of many +buckets. They are differentiated by a field called stackkey. When two +events arrive with the same stackkey they go in the same matching bucket. The very purpose of these buckets is to detect clients that exceed a -certain rate of attemps to do something (ssh connection, http -authentication failure, etc...). Thus, the most use stackkey field is +certain rate of attempts to do something (ssh connection, http +authentication failure, etc...). Thus, the most used stackkey field is often the source_ip. ## Standard leaky buckets Default buckets have two main configuration options: + * capacity: number of events the bucket can hold. When the capacity is reached and a new event is poured, a new event is raised. We call this type of event overflow. This is an int. + * leakspeed: duration needed for an event to leak. When an event - leaks, it disappear from the bucket. + leaks, it disappears from the bucket. ## Trigger -It's a special type of bucket with a zero capacity. Thus, when an -event is poured in a trigger, it always raises an overflow. +A Trigger is a special type of bucket with a capacity of zero. Thus, when an +event is poured into a trigger, it always raises an overflow. ## Uniq -It's a bucket working as the standard leaky bucket except for one +A Uniq is a bucket working like the standard leaky bucket except for one thing: a filter returns a property for each event and only one occurrence of this property is allowed in the bucket, thus the bucket is called uniq. ## Counter -It's a special type of bucket with an infinite capacity and an -infinite leakspeed (it never overflows, neither leaks). Nevertheless, +A Counter is a special type of bucket with an infinite capacity and an +infinite leakspeed (it never overflows, nor leaks). Nevertheless, the event is raised after a fixed duration. The option is called duration. +## Bayesian + +A Bayesian is a special bucket that runs bayesian inference instead of +counting events. Each event must have its likelihoods specified in the +yaml file under `prob_given_benign` and `prob_given_evil`. The bucket +will continue evaluating events until the posterior goes above the +threshold (triggering the overflow) or the duration (specified by leakspeed) +expires. + ## Available configuration options for buckets ### Fields for standard buckets * type: mandatory field. Must be one of "leaky", "trigger", "uniq" or "counter" -* name: mandatory field, but the value is totally open. Nevertheless + +* name: mandatory field, but the value is totally open. Nevertheless, this value will tag the events raised by the bucket. -* filter: mandatory field. It's a filter that is run when the decision - to make an event match the bucket or not. The filter have to return + +* filter: mandatory field. It's a filter that is run to decide whether + an event matches the bucket or not. The filter has to return a boolean. As a filter implementation we use https://github.com/antonmedv/expr + * capacity: [mandatory for now, shouldn't be mandatory in the final version] it's the size of the bucket. When pouring in a bucket already with size events, it overflows. -* leakspeed: leakspeed is a time duration (has to be parseable by - https://golang.org/pkg/time/#ParseDuration). After each interval an + +* leakspeed: leakspeed is a time duration (it has to be parsed by + https://golang.org/pkg/time/#ParseDuration). After each interval, an event is leaked from the bucket. + * stackkey: mandatory field. This field is used to differentiate on - which bucket ongoing events will be poured. When an unknown stackkey - is seen in an event a new bucket is created. -* on_overflow: optional field, that tells the what to do when the - bucket is returning the overflow event. As of today, the possibility - are these: "ban,1h", "Reprocess", "Delete". - Reprocess is used to send the raised event back in the event pool to - be matched agains buckets + which instance of the bucket the matching events will be poured. + When an unknown stackkey is seen in an event, a new bucket is created. + +* on_overflow: optional field, that tells what to do when the + bucket is returning the overflow event. As of today, the possibilities + are "ban,1h", "Reprocess" or "Delete". + Reprocess is used to send the raised event back to the event pool to + be matched against buckets ### Fields for special buckets #### Uniq -Uniq has an extra field uniq_filter which is too use the filter -implementation from https://github.com/antonmedv/expr. The filter must -return a string. All strins returned by this filter in the same -buckets have to be different. Thus, if a string is seen twice it is -dismissed. + * uniq_filter: an expression that must comply with the syntax defined + in https://github.com/antonmedv/expr and must return a string. + All strings returned by this filter in the same buckets have to be different. + Thus if a string is seen twice, the event is dismissed. #### Trigger @@ -89,11 +105,27 @@ Capacity and leakspeed are not relevant for this kind of bucket. #### Counter -It's a special kind of bucket that raise an event and is destroyed -after a fixed duration. The configuration field used is duration and -must be parseable by https://golang.org/pkg/time/#ParseDuration. -Nevertheless, this kind of bucket is often used with an infinite -leakspeed and an infinite capacity [capacity set to -1 for now]. + * duration: the Counter will be destroyed after this interval + has elapsed since its creation. The duration must be parsed + by https://golang.org/pkg/time/#ParseDuration. + Nevertheless, this kind of bucket is often used with an infinite + leakspeed and an infinite capacity [capacity set to -1 for now]. + +#### Bayesian + + * bayesian_prior: The prior to start with + * bayesian_threshold: The threshold for the posterior to trigger the overflow. + * bayesian_conditions: List of Bayesian conditions with likelihoods + +Bayesian Conditions are built from: + * condition: The expr for this specific condition to be true + * prob_given_evil: The likelihood an IP satisfies the condition given the fact + that it is a maliscious IP + * prob_given_benign: The likelihood an IP satisfies the condition given the fact + that it is a benign IP + * guillotine: Bool to stop the condition from getting evaluated if it has + evaluated to true once. This should be used if evaluating the condition is + computationally expensive. ## Add examples here @@ -126,17 +158,17 @@ leakspeed and an infinite capacity [capacity set to -1 for now]. [This is not dry enough to have many details here, but:] -The bucket code is triggered by `InfiniBucketify` in main.go. -There's one struct called buckets which is for now a +The bucket code is triggered by runPour in pour.go, by calling the `leaky.PourItemToHolders` function. +There is one struct called buckets which is for now a `map[string]interface{}` that holds all buckets. The key of this map -is derivated from the filter configured for the bucket and its -stackkey. This looks like complicated, but in fact it allows us to use -only one structs. This is done in buckets.go. +is derived from the filter configured for the bucket and its +stackkey. This looks complicated, but it allows us to use +only one struct. This is done in buckets.go. -On top of that the implementation define only the standard leaky -bucket. A goroutine is launched for every buckets (bucket.go). This +On top of that the implementation defines only the standard leaky +bucket. A goroutine is launched for every bucket (`bucket.go`). This goroutine manages the life of the bucket. For special buckets, hooks are defined at initialization time in -manager.go. Hooks are called when relevant by the bucket gorourine -when events are poured and/or when bucket overflows. \ No newline at end of file +manager.go. Hooks are called when relevant by the bucket goroutine +when events are poured and/or when a bucket overflows. diff --git a/pkg/leakybucket/bayesian.go b/pkg/leakybucket/bayesian.go new file mode 100644 index 000000000..bd9aaed96 --- /dev/null +++ b/pkg/leakybucket/bayesian.go @@ -0,0 +1,163 @@ +package leakybucket + +import ( + "fmt" + + "github.com/antonmedv/expr" + "github.com/antonmedv/expr/vm" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/types" +) + +type RawBayesianCondition struct { + ConditionalFilterName string `yaml:"condition"` + ProbGivenEvil float32 `yaml:"prob_given_evil"` + ProbGivenBenign float32 `yaml:"prob_given_benign"` + Guillotine bool `yaml:"guillotine,omitempty"` +} + +type BayesianEvent struct { + rawCondition RawBayesianCondition + conditionalFilterRuntime *vm.Program + guillotineState bool +} + +type BayesianBucket struct { + bayesianEventArray []*BayesianEvent + prior float32 + threshold float32 + posterior float32 + DumbProcessor +} + +func updateProbability(prior, probGivenEvil, ProbGivenBenign float32) float32 { + numerator := probGivenEvil * prior + denominator := numerator + ProbGivenBenign*(1-prior) + + return numerator / denominator +} + +func (c *BayesianBucket) OnBucketInit(g *BucketFactory) error { + var err error + BayesianEventArray := make([]*BayesianEvent, len(g.BayesianConditions)) + + if conditionalExprCache == nil { + conditionalExprCache = make(map[string]vm.Program) + } + conditionalExprCacheLock.Lock() + + for index, bcond := range g.BayesianConditions { + var bayesianEvent BayesianEvent + bayesianEvent.rawCondition = bcond + err = bayesianEvent.compileCondition() + if err != nil { + return err + } + BayesianEventArray[index] = &bayesianEvent + } + conditionalExprCacheLock.Unlock() + c.bayesianEventArray = BayesianEventArray + + c.prior = g.BayesianPrior + c.threshold = g.BayesianThreshold + + return err +} + +func (c *BayesianBucket) AfterBucketPour(b *BucketFactory) func(types.Event, *Leaky) *types.Event { + return func(msg types.Event, l *Leaky) *types.Event { + c.posterior = c.prior + l.logger.Debugf("starting bayesian evaluation with prior: %v", c.posterior) + + for _, bevent := range c.bayesianEventArray { + err := bevent.bayesianUpdate(c, msg, l) + if err != nil { + l.logger.Errorf("bayesian update failed for %s with %s", bevent.rawCondition.ConditionalFilterName, err) + } + } + + l.logger.Debugf("value of posterior after events : %v", c.posterior) + + if c.posterior > c.threshold { + l.logger.Debugf("Bayesian bucket overflow") + l.Ovflw_ts = l.Last_ts + l.Out <- l.Queue + return nil + } + + return &msg + } +} + +func (b *BayesianEvent) bayesianUpdate(c *BayesianBucket, msg types.Event, l *Leaky) error { + var condition, ok bool + + if b.conditionalFilterRuntime == nil { + l.logger.Tracef("empty conditional filter runtime for %s", b.rawCondition.ConditionalFilterName) + return nil + } + + l.logger.Tracef("guillotine value for %s : %v", b.rawCondition.ConditionalFilterName, b.getGuillotineState()) + if b.getGuillotineState() { + l.logger.Tracef("guillotine already triggered for %s", b.rawCondition.ConditionalFilterName) + l.logger.Tracef("condition true updating prior for: %s", b.rawCondition.ConditionalFilterName) + c.posterior = updateProbability(c.posterior, b.rawCondition.ProbGivenEvil, b.rawCondition.ProbGivenBenign) + l.logger.Tracef("new value of posterior : %v", c.posterior) + return nil + } + + l.logger.Debugf("running condition expression: %s", b.rawCondition.ConditionalFilterName) + ret, err := expr.Run(b.conditionalFilterRuntime, map[string]interface{}{"evt": &msg, "queue": l.Queue, "leaky": l}) + if err != nil { + return fmt.Errorf("unable to run conditional filter: %s", err) + } + + l.logger.Tracef("bayesian bucket expression %s returned : %v", b.rawCondition.ConditionalFilterName, ret) + if condition, ok = ret.(bool); !ok { + return fmt.Errorf("bayesian condition unexpected non-bool return: %T", ret) + } + + l.logger.Tracef("condition %T updating prior for: %s", condition, b.rawCondition.ConditionalFilterName) + if condition { + c.posterior = updateProbability(c.posterior, b.rawCondition.ProbGivenEvil, b.rawCondition.ProbGivenBenign) + b.triggerGuillotine() + } else { + c.posterior = updateProbability(c.posterior, 1-b.rawCondition.ProbGivenEvil, 1-b.rawCondition.ProbGivenBenign) + } + l.logger.Tracef("new value of posterior: %v", c.posterior) + + return nil +} + +func (b *BayesianEvent) getGuillotineState() bool { + if b.rawCondition.Guillotine { + return b.guillotineState + } + return false +} + +func (b *BayesianEvent) triggerGuillotine() { + b.guillotineState = true +} + +func (b *BayesianEvent) compileCondition() error { + var err error + var compiledExpr *vm.Program + + if compiled, ok := conditionalExprCache[b.rawCondition.ConditionalFilterName]; ok { + b.conditionalFilterRuntime = &compiled + return nil + } + + conditionalExprCacheLock.Unlock() + //release the lock during compile same as coditional bucket + compiledExpr, err = expr.Compile(b.rawCondition.ConditionalFilterName, exprhelpers.GetExprOptions(map[string]interface{}{"queue": &Queue{}, "leaky": &Leaky{}, "evt": &types.Event{}})...) + if err != nil { + return fmt.Errorf("bayesian condition compile error: %w", err) + } + b.conditionalFilterRuntime = compiledExpr + conditionalExprCacheLock.Lock() + conditionalExprCache[b.rawCondition.ConditionalFilterName] = *compiledExpr + + return nil +} diff --git a/pkg/leakybucket/bucket.go b/pkg/leakybucket/bucket.go index 004d5b9d8..286c51f11 100644 --- a/pkg/leakybucket/bucket.go +++ b/pkg/leakybucket/bucket.go @@ -191,6 +191,10 @@ func FromFactory(bucketFactory BucketFactory) *Leaky { l.conditionalOverflow = true l.Duration = l.BucketConfig.leakspeed } + + if l.BucketConfig.Type == "bayesian" { + l.Duration = l.BucketConfig.leakspeed + } return l } diff --git a/pkg/leakybucket/manager_load.go b/pkg/leakybucket/manager_load.go index 1e212f815..dc1f4ed51 100644 --- a/pkg/leakybucket/manager_load.go +++ b/pkg/leakybucket/manager_load.go @@ -51,6 +51,9 @@ type BucketFactory struct { Profiling bool `yaml:"profiling"` //Profiling, if true, will make the bucket record pours/overflows/etc. OverflowFilter string `yaml:"overflow_filter"` //OverflowFilter if present, is a filter that must return true for the overflow to go through ConditionalOverflow string `yaml:"condition"` //condition if present, is an expression that must return true for the bucket to overflow + BayesianPrior float32 `yaml:"bayesian_prior"` + BayesianThreshold float32 `yaml:"bayesian_threshold"` + BayesianConditions []RawBayesianCondition `yaml:"bayesian_conditions"` //conditions for the bayesian bucket ScopeType types.ScopeType `yaml:"scope,omitempty"` //to enforce a different remediation than blocking an IP. Will default this to IP BucketName string `yaml:"-"` Filename string `yaml:"-"` @@ -120,6 +123,25 @@ func ValidateFactory(bucketFactory *BucketFactory) error { if bucketFactory.leakspeed == 0 { return fmt.Errorf("bad leakspeed for conditional bucket '%s'", bucketFactory.LeakSpeed) } + } else if bucketFactory.Type == "bayesian" { + if bucketFactory.BayesianConditions == nil { + return fmt.Errorf("bayesian bucket must have bayesian conditions") + } + if bucketFactory.BayesianPrior == 0 { + return fmt.Errorf("bayesian bucket must have a valid, non-zero prior") + } + if bucketFactory.BayesianThreshold == 0 { + return fmt.Errorf("bayesian bucket must have a valid, non-zero threshold") + } + if bucketFactory.BayesianPrior > 1 { + return fmt.Errorf("bayesian bucket must have a valid, non-zero prior") + } + if bucketFactory.BayesianThreshold > 1 { + return fmt.Errorf("bayesian bucket must have a valid, non-zero threshold") + } + if bucketFactory.Capacity != -1 { + return fmt.Errorf("bayesian bucket must have capacity -1") + } } else { return fmt.Errorf("unknown bucket type '%s'", bucketFactory.Type) } @@ -316,6 +338,8 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error { bucketFactory.processors = append(bucketFactory.processors, &DumbProcessor{}) case "conditional": bucketFactory.processors = append(bucketFactory.processors, &DumbProcessor{}) + case "bayesian": + bucketFactory.processors = append(bucketFactory.processors, &DumbProcessor{}) default: return fmt.Errorf("invalid type '%s' in %s : %v", bucketFactory.Type, bucketFactory.Filename, err) } @@ -355,6 +379,11 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error { bucketFactory.processors = append(bucketFactory.processors, &ConditionalOverflow{}) } + if bucketFactory.BayesianThreshold != 0 { + bucketFactory.logger.Tracef("Adding bayesian processor") + bucketFactory.processors = append(bucketFactory.processors, &BayesianBucket{}) + } + if len(bucketFactory.Data) > 0 { for _, data := range bucketFactory.Data { if data.DestPath == "" { diff --git a/pkg/leakybucket/manager_load_test.go b/pkg/leakybucket/manager_load_test.go index bb3df75cd..513f11ff3 100644 --- a/pkg/leakybucket/manager_load_test.go +++ b/pkg/leakybucket/manager_load_test.go @@ -119,3 +119,25 @@ func TestCounterBucketsConfig(t *testing.T) { } } + +func TestBayesianBucketsConfig(t *testing.T) { + var CfgTests = []cfgTest{ + + //basic valid counter + {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianPrior: 0.5, BayesianThreshold: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, true, true}, + //bad capacity + {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: 1, Filter: "true", BayesianPrior: 0.5, BayesianThreshold: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, + //missing prior + {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianThreshold: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, + //missing threshold + {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianPrior: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, + //bad prior + {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianPrior: 1.5, BayesianThreshold: 0.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, + //bad threshold + {BucketFactory{Name: "test", Description: "test1", Type: "bayesian", Capacity: -1, Filter: "true", BayesianPrior: 0.5, BayesianThreshold: 1.5, BayesianConditions: []RawBayesianCondition{{ConditionalFilterName: "true", ProbGivenEvil: 0.5, ProbGivenBenign: 0.5}}}, false, false}, + } + if err := runTest(CfgTests); err != nil { + t.Fatalf("%s", err) + } + +} diff --git a/pkg/leakybucket/overflows.go b/pkg/leakybucket/overflows.go index d6131cd26..45446b8f6 100644 --- a/pkg/leakybucket/overflows.go +++ b/pkg/leakybucket/overflows.go @@ -6,15 +6,14 @@ import ( "sort" "strconv" + "github.com/antonmedv/expr" + "github.com/davecgh/go-spew/spew" + "github.com/go-openapi/strfmt" + log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/crowdsec/pkg/alertcontext" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/davecgh/go-spew/spew" - "github.com/go-openapi/strfmt" - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - - "github.com/antonmedv/expr" ) // SourceFromEvent extracts and formats a valid models.Source object from an Event @@ -53,7 +52,7 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e if leaky.scopeType.RunTimeFilter != nil { retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, map[string]interface{}{"evt": &evt}) if err != nil { - return srcs, errors.Wrapf(err, "while running scope filter") + return srcs, fmt.Errorf("while running scope filter: %w", err) } value, ok := retValue.(string) if !ok { @@ -128,7 +127,7 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e if leaky.scopeType.RunTimeFilter != nil { retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, map[string]interface{}{"evt": &evt}) if err != nil { - return srcs, errors.Wrapf(err, "while running scope filter") + return srcs, fmt.Errorf("while running scope filter: %w", err) } value, ok := retValue.(string) @@ -145,7 +144,7 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e } retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, map[string]interface{}{"evt": &evt}) if err != nil { - return srcs, errors.Wrapf(err, "while running scope filter") + return srcs, fmt.Errorf("while running scope filter: %w", err) } value, ok := retValue.(string) @@ -217,7 +216,7 @@ func alertFormatSource(leaky *Leaky, queue *Queue) (map[string]models.Source, st for _, evt := range queue.Queue { srcs, err := SourceFromEvent(evt, leaky) if err != nil { - return nil, "", errors.Wrapf(err, "while extracting scope from bucket %s", leaky.Name) + return nil, "", fmt.Errorf("while extracting scope from bucket %s: %w", leaky.Name, err) } for key, src := range srcs { if source_type == types.Undefined { @@ -276,7 +275,7 @@ func NewAlert(leaky *Leaky, queue *Queue) (types.RuntimeAlert, error) { //Get the sources from Leaky/Queue sources, source_scope, err := alertFormatSource(leaky, queue) if err != nil { - return runtimeAlert, errors.Wrap(err, "unable to collect sources from bucket") + return runtimeAlert, fmt.Errorf("unable to collect sources from bucket: %w", err) } runtimeAlert.Sources = sources //Include source info in format string diff --git a/pkg/leakybucket/tests/guillotine-bayesian-bucket/bucket.yaml b/pkg/leakybucket/tests/guillotine-bayesian-bucket/bucket.yaml new file mode 100644 index 000000000..8e8c26e6f --- /dev/null +++ b/pkg/leakybucket/tests/guillotine-bayesian-bucket/bucket.yaml @@ -0,0 +1,21 @@ +type: bayesian +name: test/guillotine-bayesian +debug: true +description: "bayesian bucket" +filter: "evt.Meta.log_type == 'http_access-log' || evt.Meta.log_type == 'ssh_access-log'" +groupby: evt.Meta.source_ip +bayesian_prior: 0.5 +bayesian_threshold: 0.8 +bayesian_conditions: +- condition: evt.Meta.http_path == "/" + prob_given_evil: 0.8 + prob_given_benign: 0.2 + guillotine : true +- condition: evt.Meta.ssh_user == "admin" + prob_given_evil: 0.9 + prob_given_benign: 0.5 + guillotine : true +leakspeed: 30s +capacity: -1 +labels: + type: overflow_1 \ No newline at end of file diff --git a/pkg/leakybucket/tests/guillotine-bayesian-bucket/scenarios.yaml b/pkg/leakybucket/tests/guillotine-bayesian-bucket/scenarios.yaml new file mode 100644 index 000000000..05e1557cf --- /dev/null +++ b/pkg/leakybucket/tests/guillotine-bayesian-bucket/scenarios.yaml @@ -0,0 +1 @@ + - filename: {{.TestDirectory}}/bucket.yaml \ No newline at end of file diff --git a/pkg/leakybucket/tests/guillotine-bayesian-bucket/test.json b/pkg/leakybucket/tests/guillotine-bayesian-bucket/test.json new file mode 100644 index 000000000..07b7b6a6e --- /dev/null +++ b/pkg/leakybucket/tests/guillotine-bayesian-bucket/test.json @@ -0,0 +1,50 @@ +{ + "lines": [ + { + "Line": { + "Labels": { + "type": "nginx" + }, + "Raw": "don't care" + }, + "MarshaledTime": "2020-01-01T10:00:00.000Z", + "Meta": { + "source_ip": "2a00:1450:4007:816::200e", + "log_type": "http_access-log", + "http_path": "/" + } + }, + { + "Line": { + "Labels": { + "type": "nginx" + }, + "Raw": "don't care" + }, + "MarshaledTime": "2020-01-01T10:00:00.000Z", + "Meta": { + "source_ip": "2a00:1450:4007:816::200e", + "log_type": "ssh_access-log", + "ssh_user": "admin" + } + } + ], + "results": [ + { + "Type" : 1, + "Alert": { + "sources" : { + "2a00:1450:4007:816::200e": { + "ip": "2a00:1450:4007:816::200e", + "scope": "Ip", + "value": "2a00:1450:4007:816::200e" + } + }, + "Alert" : { + "scenario": "test/guillotine-bayesian", + "events_count": 2 + } + } + } + ] + } \ No newline at end of file diff --git a/pkg/leakybucket/tests/multiple-bayesian-bucket/bucket.yaml b/pkg/leakybucket/tests/multiple-bayesian-bucket/bucket.yaml new file mode 100644 index 000000000..1110fb7b8 --- /dev/null +++ b/pkg/leakybucket/tests/multiple-bayesian-bucket/bucket.yaml @@ -0,0 +1,21 @@ +type: bayesian +name: test/multiple-bayesian +debug: true +description: "bayesian bucket" +filter: "evt.Meta.log_type == 'http_access-log' || evt.Meta.log_type == 'ssh_access-log'" +groupby: evt.Meta.source_ip +bayesian_prior: 0.5 +bayesian_threshold: 0.8 +bayesian_conditions: +- condition: evt.Meta.http_path == "/" + prob_given_evil: 0.8 + prob_given_benign: 0.2 + guillotine : true +- condition: evt.Meta.ssh_user == "admin" + prob_given_evil: 0.9 + prob_given_benign: 0.5 + guillotine : true +leakspeed: 30s +capacity: -1 +labels: + type: overflow_1 \ No newline at end of file diff --git a/pkg/leakybucket/tests/multiple-bayesian-bucket/scenarios.yaml b/pkg/leakybucket/tests/multiple-bayesian-bucket/scenarios.yaml new file mode 100644 index 000000000..05e1557cf --- /dev/null +++ b/pkg/leakybucket/tests/multiple-bayesian-bucket/scenarios.yaml @@ -0,0 +1 @@ + - filename: {{.TestDirectory}}/bucket.yaml \ No newline at end of file diff --git a/pkg/leakybucket/tests/multiple-bayesian-bucket/test.json b/pkg/leakybucket/tests/multiple-bayesian-bucket/test.json new file mode 100644 index 000000000..69454a6ed --- /dev/null +++ b/pkg/leakybucket/tests/multiple-bayesian-bucket/test.json @@ -0,0 +1,64 @@ +{ + "lines": [ + { + "Line": { + "Labels": { + "type": "nginx" + }, + "Raw": "don't care" + }, + "MarshaledTime": "2020-01-01T10:00:00.000Z", + "Meta": { + "source_ip": "2a00:1450:4007:816::200e", + "log_type": "http_access-log", + "http_path": "/" + } + }, + { + "Line": { + "Labels": { + "type": "nginx" + }, + "Raw": "don't care" + }, + "MarshaledTime": "2020-01-01T10:00:00.000Z", + "Meta": { + "source_ip": "1.2.3.4", + "log_type": "ssh_access-log", + "ssh_user": "admin" + } + }, + { + "Line": { + "Labels": { + "type": "nginx" + }, + "Raw": "don't care" + }, + "MarshaledTime": "2020-01-01T10:00:00.000Z", + "Meta": { + "source_ip": "2a00:1450:4007:816::200e", + "log_type": "ssh_access-log", + "ssh_user": "admin" + } + } + ], + "results": [ + { + "Type" : 1, + "Alert": { + "sources" : { + "2a00:1450:4007:816::200e": { + "ip": "2a00:1450:4007:816::200e", + "scope": "Ip", + "value": "2a00:1450:4007:816::200e" + } + }, + "Alert" : { + "scenario": "test/multiple-bayesian", + "events_count": 2 + } + } + } + ] + } \ No newline at end of file diff --git a/pkg/leakybucket/tests/simple-bayesian-bucket/bucket.yaml b/pkg/leakybucket/tests/simple-bayesian-bucket/bucket.yaml new file mode 100644 index 000000000..21a4ab074 --- /dev/null +++ b/pkg/leakybucket/tests/simple-bayesian-bucket/bucket.yaml @@ -0,0 +1,19 @@ +type: bayesian +name: test/simple-bayesian +debug: true +description: "bayesian bucket" +filter: "evt.Meta.log_type == 'http_access-log' || evt.Meta.log_type == 'ssh_access-log'" +groupby: evt.Meta.source_ip +bayesian_prior: 0.5 +bayesian_threshold: 0.8 +bayesian_conditions: +- condition: any(queue.Queue, {.Meta.http_path == "/"}) + prob_given_evil: 0.8 + prob_given_benign: 0.2 +- condition: any(queue.Queue, {.Meta.ssh_user == "admin"}) + prob_given_evil: 0.9 + prob_given_benign: 0.5 +leakspeed: 30s +capacity: -1 +labels: + type: overflow_1 \ No newline at end of file diff --git a/pkg/leakybucket/tests/simple-bayesian-bucket/scenarios.yaml b/pkg/leakybucket/tests/simple-bayesian-bucket/scenarios.yaml new file mode 100644 index 000000000..05e1557cf --- /dev/null +++ b/pkg/leakybucket/tests/simple-bayesian-bucket/scenarios.yaml @@ -0,0 +1 @@ + - filename: {{.TestDirectory}}/bucket.yaml \ No newline at end of file diff --git a/pkg/leakybucket/tests/simple-bayesian-bucket/test.json b/pkg/leakybucket/tests/simple-bayesian-bucket/test.json new file mode 100644 index 000000000..a5807c4b6 --- /dev/null +++ b/pkg/leakybucket/tests/simple-bayesian-bucket/test.json @@ -0,0 +1,50 @@ +{ + "lines": [ + { + "Line": { + "Labels": { + "type": "nginx" + }, + "Raw": "don't care" + }, + "MarshaledTime": "2020-01-01T10:00:00.000Z", + "Meta": { + "source_ip": "2a00:1450:4007:816::200e", + "log_type": "http_access-log", + "http_path": "/" + } + }, + { + "Line": { + "Labels": { + "type": "nginx" + }, + "Raw": "don't care" + }, + "MarshaledTime": "2020-01-01T10:00:00.000Z", + "Meta": { + "source_ip": "2a00:1450:4007:816::200e", + "log_type": "ssh_access-log", + "ssh_user": "admin" + } + } + ], + "results": [ + { + "Type" : 1, + "Alert": { + "sources" : { + "2a00:1450:4007:816::200e": { + "ip": "2a00:1450:4007:816::200e", + "scope": "Ip", + "value": "2a00:1450:4007:816::200e" + } + }, + "Alert" : { + "scenario": "test/simple-bayesian", + "events_count": 2 + } + } + } + ] + } \ No newline at end of file diff --git a/pkg/metabase/api.go b/pkg/metabase/api.go index 99cbf9ec7..7235ff7f1 100644 --- a/pkg/metabase/api.go +++ b/pkg/metabase/api.go @@ -12,7 +12,7 @@ import ( log "github.com/sirupsen/logrus" ) -type APIClient struct { +type MBClient struct { CTX *sling.Sling Client *http.Client } @@ -35,15 +35,15 @@ var ( } ) -func NewAPIClient(url string) (*APIClient, error) { +func NewMBClient(url string) (*MBClient, error) { httpClient := &http.Client{Timeout: 20 * time.Second} - return &APIClient{ + return &MBClient{ CTX: sling.New().Client(httpClient).Base(url).Set("User-Agent", fmt.Sprintf("crowdsec/%s", version.String())), Client: httpClient, }, nil } -func (h *APIClient) Do(method string, route string, body interface{}) (interface{}, interface{}, error) { +func (h *MBClient) Do(method string, route string, body interface{}) (interface{}, interface{}, error) { var Success interface{} var Error interface{} var resp *http.Response @@ -80,6 +80,6 @@ func (h *APIClient) Do(method string, route string, body interface{}) (interface } // Set set headers as key:value -func (h *APIClient) Set(key string, value string) { +func (h *MBClient) Set(key string, value string) { h.CTX = h.CTX.Set(key, value) } diff --git a/pkg/metabase/database.go b/pkg/metabase/database.go index 0a7890fac..273d06dee 100644 --- a/pkg/metabase/database.go +++ b/pkg/metabase/database.go @@ -7,14 +7,13 @@ import ( "strings" "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/pkg/errors" ) type Database struct { DBUrl string Model *Model Config *csconfig.DatabaseCfg - Client *APIClient + Client *MBClient Details *Details // in case mysql host is 127.0.0.1 the ip address of mysql/pgsql host will be the docker gateway since metabase run in a container } @@ -41,7 +40,7 @@ type Model struct { Schedules map[string]interface{} `json:"schedules"` } -func NewDatabase(config *csconfig.DatabaseCfg, client *APIClient, remoteDBAddr string) (*Database, error) { +func NewDatabase(config *csconfig.DatabaseCfg, client *MBClient, remoteDBAddr string) (*Database, error) { var details *Details database := Database{} @@ -80,13 +79,13 @@ func (d *Database) Update() error { data, err := json.Marshal(success) if err != nil { - return errors.Wrap(err, "update sqlite db response (marshal)") + return fmt.Errorf("update sqlite db response (marshal): %w", err) } model := Model{} if err := json.Unmarshal(data, &model); err != nil { - return errors.Wrap(err, "update sqlite db response (unmarshal)") + return fmt.Errorf("update sqlite db response (unmarshal): %w", err) } model.Details = d.Details _, errormsg, err = d.Client.Do("PUT", routes[databaseEndpoint], model) diff --git a/pkg/metabase/metabase.go b/pkg/metabase/metabase.go index c12d9166f..cdbe65ec8 100644 --- a/pkg/metabase/metabase.go +++ b/pkg/metabase/metabase.go @@ -4,6 +4,7 @@ import ( "archive/zip" "bytes" "context" + "errors" "fmt" "io" "net/http" @@ -15,15 +16,14 @@ import ( "github.com/docker/docker/client" log "github.com/sirupsen/logrus" + "gopkg.in/yaml.v2" "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/pkg/errors" - "gopkg.in/yaml.v2" ) type Metabase struct { Config *Config - Client *APIClient + Client *MBClient Container *Container Database *Database InternalDBURL string @@ -80,7 +80,7 @@ func (m *Metabase) Init(containerName string) error { return fmt.Errorf("database '%s' not supported", m.Config.Database.Type) } - m.Client, err = NewAPIClient(m.Config.ListenURL) + m.Client, err = NewMBClient(m.Config.ListenURL) if err != nil { return err } @@ -90,7 +90,7 @@ func (m *Metabase) Init(containerName string) error { } m.Container, err = NewContainer(m.Config.ListenAddr, m.Config.ListenPort, m.Config.DBPath, containerName, metabaseImage, DBConnectionURI, m.Config.DockerGroupID) if err != nil { - return errors.Wrap(err, "container init") + return fmt.Errorf("container init: %w", err) } return nil @@ -151,36 +151,36 @@ func SetupMetabase(dbConfig *csconfig.DatabaseCfg, listenAddr string, listenPort }, } if err := metabase.Init(containerName); err != nil { - return nil, errors.Wrap(err, "metabase setup init") + return nil, fmt.Errorf("metabase setup init: %w", err) } if err := metabase.DownloadDatabase(false); err != nil { - return nil, errors.Wrap(err, "metabase db download") + return nil, fmt.Errorf("metabase db download: %w", err) } if err := metabase.Container.Create(); err != nil { - return nil, errors.Wrap(err, "container create") + return nil, fmt.Errorf("container create: %w", err) } if err := metabase.Container.Start(); err != nil { - return nil, errors.Wrap(err, "container start") + return nil, fmt.Errorf("container start: %w", err) } log.Infof("waiting for metabase to be up (can take up to a minute)") if err := metabase.WaitAlive(); err != nil { - return nil, errors.Wrap(err, "wait alive") + return nil, fmt.Errorf("wait alive: %w", err) } if err := metabase.Database.Update(); err != nil { - return nil, errors.Wrap(err, "update database") + return nil, fmt.Errorf("update database: %w", err) } if err := metabase.Scan(); err != nil { - return nil, errors.Wrap(err, "db scan") + return nil, fmt.Errorf("db scan: %w", err) } if err := metabase.ResetCredentials(); err != nil { - return nil, errors.Wrap(err, "reset creds") + return nil, fmt.Errorf("reset creds: %w", err) } return metabase, nil @@ -193,7 +193,7 @@ func (m *Metabase) WaitAlive() error { if err != nil { if strings.Contains(err.Error(), "password:did not match stored password") { log.Errorf("Password mismatch error, is your dashboard already setup ? Run 'cscli dashboard remove' to reset it.") - return errors.Wrapf(err, "Password mismatch error") + return fmt.Errorf("password mismatch error: %w", err) } log.Debugf("%+v", err) } else { @@ -215,7 +215,7 @@ func (m *Metabase) Login(username string, password string) error { } if errormsg != nil { - return errors.Wrap(err, "http login") + return fmt.Errorf("http login: %s", errormsg) } resp, ok := successmsg.(map[string]interface{}) if !ok { @@ -238,7 +238,7 @@ func (m *Metabase) Scan() error { return err } if errormsg != nil { - return errors.Wrap(err, "http scan") + return fmt.Errorf("http scan: %s", errormsg) } return nil @@ -252,10 +252,10 @@ func (m *Metabase) ResetPassword(current string, newPassword string) error { } _, errormsg, err := m.Client.Do("PUT", routes[resetPasswordEndpoint], body) if err != nil { - return errors.Wrap(err, "reset username") + return fmt.Errorf("reset username: %w", err) } if errormsg != nil { - return errors.Wrap(err, "http reset password") + return fmt.Errorf("http reset password: %s", errormsg) } return nil } @@ -275,11 +275,11 @@ func (m *Metabase) ResetUsername(username string) error { _, errormsg, err := m.Client.Do("PUT", routes[userEndpoint], body) if err != nil { - return errors.Wrap(err, "reset username") + return fmt.Errorf("reset username: %w", err) } if errormsg != nil { - return errors.Wrap(err, "http reset username") + return fmt.Errorf("http reset username: %s", errormsg) } return nil diff --git a/pkg/parser/enrich.go b/pkg/parser/enrich.go index 04652407d..5180b9a5f 100644 --- a/pkg/parser/enrich.go +++ b/pkg/parser/enrich.go @@ -1,11 +1,12 @@ package parser import ( - "github.com/crowdsecurity/crowdsec/pkg/types" log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/crowdsec/pkg/types" ) -/* should be part of a packaged shared with enrich/geoip.go */ +/* should be part of a package shared with enrich/geoip.go */ type EnrichFunc func(string, *types.Event, interface{}, *log.Entry) (map[string]string, error) type InitFunc func(map[string]string) (interface{}, error) diff --git a/pkg/parser/enrich_date.go b/pkg/parser/enrich_date.go index a1dd994be..20828af90 100644 --- a/pkg/parser/enrich_date.go +++ b/pkg/parser/enrich_date.go @@ -3,9 +3,10 @@ package parser import ( "time" + log "github.com/sirupsen/logrus" + expr "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/types" - log "github.com/sirupsen/logrus" ) func parseDateWithFormat(date, format string) (string, time.Time) { diff --git a/pkg/parser/enrich_dns.go b/pkg/parser/enrich_dns.go index 5bbc0e94d..f622e6c35 100644 --- a/pkg/parser/enrich_dns.go +++ b/pkg/parser/enrich_dns.go @@ -3,9 +3,9 @@ package parser import ( "net" - "github.com/crowdsecurity/crowdsec/pkg/types" log "github.com/sirupsen/logrus" - //"github.com/crowdsecurity/crowdsec/pkg/parser" + + "github.com/crowdsecurity/crowdsec/pkg/types" ) /* All plugins must export a list of function pointers for exported symbols */ diff --git a/pkg/parser/enrich_geoip.go b/pkg/parser/enrich_geoip.go index 8c25bef44..0a263c827 100644 --- a/pkg/parser/enrich_geoip.go +++ b/pkg/parser/enrich_geoip.go @@ -5,11 +5,11 @@ import ( "net" "strconv" - "github.com/crowdsecurity/crowdsec/pkg/types" - log "github.com/sirupsen/logrus" - "github.com/oschwald/geoip2-golang" "github.com/oschwald/maxminddb-golang" + log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/crowdsec/pkg/types" ) func IpToRange(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) { diff --git a/pkg/parser/enrich_unmarshal.go b/pkg/parser/enrich_unmarshal.go index 404340401..dce9c75d4 100644 --- a/pkg/parser/enrich_unmarshal.go +++ b/pkg/parser/enrich_unmarshal.go @@ -3,8 +3,9 @@ package parser import ( "encoding/json" - "github.com/crowdsecurity/crowdsec/pkg/types" log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/crowdsec/pkg/types" ) func unmarshalJSON(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) { diff --git a/pkg/types/grok_pattern.go b/pkg/parser/grok_pattern.go similarity index 99% rename from pkg/types/grok_pattern.go rename to pkg/parser/grok_pattern.go index bb51d2d9b..5b3204a42 100644 --- a/pkg/types/grok_pattern.go +++ b/pkg/parser/grok_pattern.go @@ -1,9 +1,10 @@ -package types +package parser import ( "time" "github.com/antonmedv/expr/vm" + "github.com/crowdsecurity/grokky" ) diff --git a/pkg/parser/node.go b/pkg/parser/node.go index 2370d3796..036a244cc 100644 --- a/pkg/parser/node.go +++ b/pkg/parser/node.go @@ -1,24 +1,24 @@ package parser import ( + "errors" "fmt" "net" "strings" "time" "github.com/antonmedv/expr" - "github.com/crowdsecurity/grokky" - "github.com/pkg/errors" + "github.com/antonmedv/expr/vm" + "github.com/davecgh/go-spew/spew" + "github.com/prometheus/client_golang/prometheus" + log "github.com/sirupsen/logrus" yaml "gopkg.in/yaml.v2" - "github.com/antonmedv/expr/vm" + "github.com/crowdsecurity/grokky" + "github.com/crowdsecurity/crowdsec/pkg/cache" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/davecgh/go-spew/spew" - "github.com/prometheus/client_golang/prometheus" - "github.com/sirupsen/logrus" - log "github.com/sirupsen/logrus" ) type Node struct { @@ -56,11 +56,11 @@ type Node struct { SubGroks yaml.MapSlice `yaml:"pattern_syntax,omitempty"` //Holds a grok pattern - Grok types.GrokPattern `yaml:"grok,omitempty"` + Grok GrokPattern `yaml:"grok,omitempty"` //Statics can be present in any type of node and is executed last - Statics []types.ExtraField `yaml:"statics,omitempty"` + Statics []ExtraField `yaml:"statics,omitempty"` //Stash allows to capture data from the log line and store it in an accessible cache - Stash []types.DataCapture `yaml:"stash,omitempty"` + Stash []DataCapture `yaml:"stash,omitempty"` //Whitelists Whitelist Whitelist `yaml:"whitelist,omitempty"` Data []*types.DataSource `yaml:"data,omitempty"` @@ -360,29 +360,27 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri } //Iterate on leafs - if len(n.LeavesNodes) > 0 { - for _, leaf := range n.LeavesNodes { - ret, err := leaf.process(p, ctx, cachedExprEnv) - if err != nil { - clog.Tracef("\tNode (%s) failed : %v", leaf.rn, err) - clog.Debugf("Event leaving node : ko") - return false, err - } - clog.Tracef("\tsub-node (%s) ret : %v (strategy:%s)", leaf.rn, ret, n.OnSuccess) - if ret { - NodeState = true - /* if child is successful, stop processing */ - if n.OnSuccess == "next_stage" { - clog.Debugf("child is success, OnSuccess=next_stage, skip") - break - } - } else if !NodeHasOKGrok { - /* - If the parent node has a successful grok pattern, it's state will stay successful even if one or more chil fails. - If the parent node is a skeleton node (no grok pattern), then at least one child must be successful for it to be a success. - */ - NodeState = false + for _, leaf := range n.LeavesNodes { + ret, err := leaf.process(p, ctx, cachedExprEnv) + if err != nil { + clog.Tracef("\tNode (%s) failed : %v", leaf.rn, err) + clog.Debugf("Event leaving node : ko") + return false, err + } + clog.Tracef("\tsub-node (%s) ret : %v (strategy:%s)", leaf.rn, ret, n.OnSuccess) + if ret { + NodeState = true + /* if child is successful, stop processing */ + if n.OnSuccess == "next_stage" { + clog.Debugf("child is success, OnSuccess=next_stage, skip") + break } + } else if !NodeHasOKGrok { + /* + If the parent node has a successful grok pattern, it's state will stay successful even if one or more chil fails. + If the parent node is a skeleton node (no grok pattern), then at least one child must be successful for it to be a success. + */ + NodeState = false } } /*todo : check if a node made the state change ?*/ @@ -402,10 +400,11 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri if n.Name != "" { NodesHitsOk.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc() } + /* - Please kill me. this is to apply statics when the node *has* whitelists that successfully matched the node. + This is to apply statics when the node *has* whitelists that successfully matched the node. */ - if hasWhitelist && isWhitelisted && len(n.Statics) > 0 || len(n.Statics) > 0 && !hasWhitelist { + if len(n.Statics) > 0 && (isWhitelisted || !hasWhitelist) { clog.Debugf("+ Processing %d statics", len(n.Statics)) // if all else is good in whitelist, process node's statics err := n.ProcessStatics(n.Statics, p) @@ -453,8 +452,8 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { /* if the node has debugging enabled, create a specific logger with debug that will be used only for processing this node ;) */ if n.Debug { - var clog = logrus.New() - if err := types.ConfigureLogger(clog); err != nil { + var clog = log.New() + if err = types.ConfigureLogger(clog); err != nil { log.Fatalf("While creating bucket-specific logger : %s", err) } clog.SetLevel(log.DebugLevel) @@ -493,7 +492,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { /* handle pattern_syntax and groks */ for _, pattern := range n.SubGroks { n.Logger.Tracef("Adding subpattern '%s' : '%s'", pattern.Key, pattern.Value) - if err := pctx.Grok.Add(pattern.Key.(string), pattern.Value.(string)); err != nil { + if err = pctx.Grok.Add(pattern.Key.(string), pattern.Value.(string)); err != nil { if errors.Is(err, grokky.ErrAlreadyExist) { n.Logger.Warningf("grok '%s' already registred", pattern.Key) continue @@ -536,20 +535,18 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { n.Grok.RunTimeValue, err = expr.Compile(n.Grok.ExpValue, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) if err != nil { - return errors.Wrap(err, "while compiling grok's expression") + return fmt.Errorf("while compiling grok's expression: %w", err) } } /* load grok statics */ - if len(n.Grok.Statics) > 0 { - //compile expr statics if present - for idx := range n.Grok.Statics { - if n.Grok.Statics[idx].ExpValue != "" { - n.Grok.Statics[idx].RunTimeValue, err = expr.Compile(n.Grok.Statics[idx].ExpValue, - exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) - if err != nil { - return err - } + //compile expr statics if present + for idx := range n.Grok.Statics { + if n.Grok.Statics[idx].ExpValue != "" { + n.Grok.Statics[idx].RunTimeValue, err = expr.Compile(n.Grok.Statics[idx].ExpValue, + exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) + if err != nil { + return err } } valid = true @@ -560,54 +557,53 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { n.Stash[i].ValueExpression, err = expr.Compile(stash.Value, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) if err != nil { - return errors.Wrap(err, "while compiling stash value expression") + return fmt.Errorf("while compiling stash value expression: %w", err) } n.Stash[i].KeyExpression, err = expr.Compile(stash.Key, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) if err != nil { - return errors.Wrap(err, "while compiling stash key expression") + return fmt.Errorf("while compiling stash key expression: %w", err) } n.Stash[i].TTLVal, err = time.ParseDuration(stash.TTL) if err != nil { - return errors.Wrap(err, "while parsing stash ttl") + return fmt.Errorf("while parsing stash ttl: %w", err) } logLvl := n.Logger.Logger.GetLevel() //init the cache, does it make sense to create it here just to be sure everything is fine ? - if err := cache.CacheInit(cache.CacheCfg{ + if err = cache.CacheInit(cache.CacheCfg{ Size: n.Stash[i].MaxMapSize, TTL: n.Stash[i].TTLVal, Name: n.Stash[i].Name, Strategy: n.Stash[i].Strategy, LogLevel: &logLvl, }); err != nil { - return errors.Wrap(err, "while initializing cache") + return fmt.Errorf("while initializing cache: %w", err) } } /* compile leafs if present */ - if len(n.LeavesNodes) > 0 { - for idx := range n.LeavesNodes { - if n.LeavesNodes[idx].Name == "" { - n.LeavesNodes[idx].Name = fmt.Sprintf("child-%s", n.Name) - } - /*propagate debug/stats to child nodes*/ - if !n.LeavesNodes[idx].Debug && n.Debug { - n.LeavesNodes[idx].Debug = true - } - if !n.LeavesNodes[idx].Profiling && n.Profiling { - n.LeavesNodes[idx].Profiling = true - } - n.LeavesNodes[idx].Stage = n.Stage - err = n.LeavesNodes[idx].compile(pctx, ectx) - if err != nil { - return err - } + for idx := range n.LeavesNodes { + if n.LeavesNodes[idx].Name == "" { + n.LeavesNodes[idx].Name = fmt.Sprintf("child-%s", n.Name) + } + /*propagate debug/stats to child nodes*/ + if !n.LeavesNodes[idx].Debug && n.Debug { + n.LeavesNodes[idx].Debug = true + } + if !n.LeavesNodes[idx].Profiling && n.Profiling { + n.LeavesNodes[idx].Profiling = true + } + n.LeavesNodes[idx].Stage = n.Stage + err = n.LeavesNodes[idx].compile(pctx, ectx) + if err != nil { + return err } valid = true } + /* load statics if present */ for idx := range n.Statics { if n.Statics[idx].ExpValue != "" { @@ -626,6 +622,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { n.Logger.Debugf("adding ip %s to whitelists", net.ParseIP(v)) valid = true } + for _, v := range n.Whitelist.Cidrs { _, tnet, err := net.ParseCIDR(v) if err != nil { @@ -635,6 +632,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { n.Logger.Debugf("adding cidr %s to whitelists", tnet) valid = true } + for _, filter := range n.Whitelist.Exprs { expression := &ExprWhitelist{} expression.Filter, err = expr.Compile(filter, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...) @@ -660,5 +658,6 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error { if err := n.validate(pctx, ectx); err != nil { return err } + return nil } diff --git a/pkg/parser/node_test.go b/pkg/parser/node_test.go index f529cb4d4..d85aa82a8 100644 --- a/pkg/parser/node_test.go +++ b/pkg/parser/node_test.go @@ -3,7 +3,6 @@ package parser import ( "testing" - "github.com/crowdsecurity/crowdsec/pkg/types" yaml "gopkg.in/yaml.v2" ) @@ -20,7 +19,7 @@ func TestParserConfigs(t *testing.T) { Valid bool }{ //valid node with grok pattern - {&Node{Debug: true, Stage: "s00", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true}, + {&Node{Debug: true, Stage: "s00", Grok: GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true}, //bad filter {&Node{Debug: true, Stage: "s00", Filter: "ratata"}, false, false}, //empty node @@ -28,25 +27,25 @@ func TestParserConfigs(t *testing.T) { //bad subgrok {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-$")}}}, false, true}, //valid node with grok pattern - {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-z]")}}, Grok: types.GrokPattern{RegexpValue: "^x%{FOOBAR:extr}$", TargetField: "t"}}, true, true}, + {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBAR"), Value: string("[a-z]")}}, Grok: GrokPattern{RegexpValue: "^x%{FOOBAR:extr}$", TargetField: "t"}}, true, true}, //bad node success - {&Node{Debug: true, Stage: "s00", OnSuccess: "ratat", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, false, false}, + {&Node{Debug: true, Stage: "s00", OnSuccess: "ratat", Grok: GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, false, false}, //ok node success - {&Node{Debug: true, Stage: "s00", OnSuccess: "continue", Grok: types.GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true}, + {&Node{Debug: true, Stage: "s00", OnSuccess: "continue", Grok: GrokPattern{RegexpValue: "^x%{DATA:extr}$", TargetField: "t"}}, true, true}, //valid node with grok sub-pattern used by name - {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBARx"), Value: string("[a-z] %{DATA:lol}$")}}, Grok: types.GrokPattern{RegexpName: "FOOBARx", TargetField: "t"}}, true, true}, + {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{{Key: string("FOOBARx"), Value: string("[a-z] %{DATA:lol}$")}}, Grok: GrokPattern{RegexpName: "FOOBARx", TargetField: "t"}}, true, true}, //node with unexisting grok pattern - {&Node{Debug: true, Stage: "s00", Grok: types.GrokPattern{RegexpName: "RATATA", TargetField: "t"}}, false, true}, + {&Node{Debug: true, Stage: "s00", Grok: GrokPattern{RegexpName: "RATATA", TargetField: "t"}}, false, true}, //node with grok pattern dependencies {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{ {Key: string("SUBGROK"), Value: string("[a-z]")}, {Key: string("MYGROK"), Value: string("[a-z]%{SUBGROK}")}, - }, Grok: types.GrokPattern{RegexpValue: "^x%{MYGROK:extr}$", TargetField: "t"}}, true, true}, + }, Grok: GrokPattern{RegexpValue: "^x%{MYGROK:extr}$", TargetField: "t"}}, true, true}, //node with broken grok pattern dependencies {&Node{Debug: true, Stage: "s00", SubGroks: yaml.MapSlice{ {Key: string("SUBGROKBIS"), Value: string("[a-z]%{MYGROKBIS}")}, {Key: string("MYGROKBIS"), Value: string("[a-z]")}, - }, Grok: types.GrokPattern{RegexpValue: "^x%{MYGROKBIS:extr}$", TargetField: "t"}}, false, true}, + }, Grok: GrokPattern{RegexpValue: "^x%{MYGROKBIS:extr}$", TargetField: "t"}}, false, true}, } for idx := range CfgTests { err := CfgTests[idx].NodeCfg.compile(pctx, EnricherCtx{}) @@ -64,6 +63,5 @@ func TestParserConfigs(t *testing.T) { if CfgTests[idx].Valid == false && err == nil { t.Fatalf("Valid: (%d/%d) expected error", idx+1, len(CfgTests)) } - } } diff --git a/pkg/parser/parsing_test.go b/pkg/parser/parsing_test.go index 089b95835..04d08cc27 100644 --- a/pkg/parser/parsing_test.go +++ b/pkg/parser/parsing_test.go @@ -11,11 +11,12 @@ import ( "strings" "testing" - "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" - "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/davecgh/go-spew/spew" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" + + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/types" ) type TestFile struct { @@ -54,7 +55,6 @@ func TestParser(t *testing.T) { } } } - } func BenchmarkParser(t *testing.B) { @@ -90,7 +90,6 @@ func BenchmarkParser(t *testing.B) { } func testOneParser(pctx *UnixParserCtx, ectx EnricherCtx, dir string, b *testing.B) error { - var ( err error pnodes []Node @@ -112,7 +111,7 @@ func testOneParser(pctx *UnixParserCtx, ectx EnricherCtx, dir string, b *testing if err != nil { panic(err) } - if err := yaml.UnmarshalStrict(out.Bytes(), &parser_configs); err != nil { + if err = yaml.UnmarshalStrict(out.Bytes(), &parser_configs); err != nil { return fmt.Errorf("failed unmarshaling %s : %s", parser_cfg_file, err) } @@ -399,7 +398,7 @@ func TestGeneratePatternsDoc(t *testing.T) { t.Fatal("failed to write to file") } for _, k := range p { - if _, err := f.WriteString(fmt.Sprintf("## %s\n\nPattern :\n```\n%s\n```\n\n", k.Key, k.Value)); err != nil { + if _, err := fmt.Fprintf(f, "## %s\n\nPattern :\n```\n%s\n```\n\n", k.Key, k.Value); err != nil { t.Fatal("failed to write to file") } fmt.Printf("%v\t%v\n", k.Key, k.Value) @@ -414,5 +413,4 @@ func TestGeneratePatternsDoc(t *testing.T) { t.Fatal("failed to write to file") } f.Close() - } diff --git a/pkg/parser/runtime.go b/pkg/parser/runtime.go index 3c7d382d5..e6ee07ea7 100644 --- a/pkg/parser/runtime.go +++ b/pkg/parser/runtime.go @@ -9,24 +9,21 @@ import ( "errors" "fmt" "reflect" + "strconv" "strings" "sync" "time" - "github.com/crowdsecurity/crowdsec/pkg/types" - - "strconv" - + "github.com/antonmedv/expr" "github.com/mohae/deepcopy" "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" - "github.com/antonmedv/expr" + "github.com/crowdsecurity/crowdsec/pkg/types" ) /* ok, this is kinda experimental, I don't know how bad of an idea it is .. */ func SetTargetByName(target string, value string, evt *types.Event) bool { - if evt == nil { return false } @@ -71,8 +68,6 @@ func SetTargetByName(target string, value string, evt *types.Event) bool { tmp = reflect.Indirect(tmp) } iter = tmp - //nolint: gosimple - break case reflect.Ptr: tmp := iter.Elem() iter = reflect.Indirect(tmp.FieldByName(f)) @@ -94,24 +89,24 @@ func SetTargetByName(target string, value string, evt *types.Event) bool { return true } -func printStaticTarget(static types.ExtraField) string { - - if static.Method != "" { +func printStaticTarget(static ExtraField) string { + switch { + case static.Method != "": return static.Method - } else if static.Parsed != "" { + case static.Parsed != "": return fmt.Sprintf(".Parsed[%s]", static.Parsed) - } else if static.Meta != "" { + case static.Meta != "": return fmt.Sprintf(".Meta[%s]", static.Meta) - } else if static.Enriched != "" { + case static.Enriched != "": return fmt.Sprintf(".Enriched[%s]", static.Enriched) - } else if static.TargetByName != "" { + case static.TargetByName != "": return static.TargetByName - } else { + default: return "?" } } -func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) error { +func (n *Node) ProcessStatics(statics []ExtraField, event *types.Event) error { //we have a few cases : //(meta||key) + (static||reference||expr) var value string @@ -197,7 +192,6 @@ func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) er } else { clog.Fatal("unable to process static : unknown target") } - } return nil } @@ -357,10 +351,8 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error) event.Process = false return event, nil } - } event.Process = true return event, nil - } diff --git a/pkg/parser/stage.go b/pkg/parser/stage.go index 3bdaed1f0..37d43fbfe 100644 --- a/pkg/parser/stage.go +++ b/pkg/parser/stage.go @@ -109,17 +109,16 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx EnricherCtx) ( continue } - if len(node.Data) > 0 { - for _, data := range node.Data { - err = exprhelpers.FileInit(pctx.DataFolder, data.DestPath, data.Type) - if err != nil { - log.Error(err) - } - if data.Type == "regexp" { //cache only makes sense for regexp - exprhelpers.RegexpCacheInit(data.DestPath, *data) - } + for _, data := range node.Data { + err = exprhelpers.FileInit(pctx.DataFolder, data.DestPath, data.Type) + if err != nil { + log.Error(err) + } + if data.Type == "regexp" { //cache only makes sense for regexp + exprhelpers.RegexpCacheInit(data.DestPath, *data) } } + nodes = append(nodes, node) nodesCount++ } diff --git a/pkg/parser/unix_parser.go b/pkg/parser/unix_parser.go index b0f38f81e..d5d91f932 100644 --- a/pkg/parser/unix_parser.go +++ b/pkg/parser/unix_parser.go @@ -7,12 +7,13 @@ import ( "sort" "strings" + log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/grokky" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/fflag" - - "github.com/crowdsecurity/grokky" - log "github.com/sirupsen/logrus" ) type UnixParserCtx struct { @@ -117,7 +118,7 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) { parsers.EnricherCtx, err = Loadplugin(cConfig.Crowdsec.DataDir) if err != nil { - return parsers, fmt.Errorf("Failed to load enrich plugin : %v", err) + return parsers, fmt.Errorf("failed to load enrich plugin : %v", err) } /* @@ -135,8 +136,8 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) { log.Infof("Loading postoverflow parsers") parsers.Povfwnodes, err = LoadStages(parsers.PovfwStageFiles, parsers.Povfwctx, parsers.EnricherCtx) } else { - parsers.Povfwnodes = []Node{} log.Infof("No postoverflow parsers to load") + parsers.Povfwnodes = []Node{} } if err != nil { diff --git a/pkg/parser/whitelist.go b/pkg/parser/whitelist.go index c56ad40a7..e2f179fb3 100644 --- a/pkg/parser/whitelist.go +++ b/pkg/parser/whitelist.go @@ -4,6 +4,7 @@ import ( "net" "github.com/antonmedv/expr/vm" + "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" ) diff --git a/pkg/setup/detect.go b/pkg/setup/detect.go index 957f70a25..b345c0d6f 100644 --- a/pkg/setup/detect.go +++ b/pkg/setup/detect.go @@ -7,15 +7,12 @@ import ( "os/exec" "sort" - "github.com/Masterminds/semver" + "github.com/Masterminds/semver/v3" "github.com/antonmedv/expr" "github.com/blackfireio/osinfo" "github.com/shirou/gopsutil/v3/process" log "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" - // goccyyaml "github.com/goccy/go-yaml" - - // "github.com/k0kubun/pp" "github.com/crowdsecurity/crowdsec/pkg/acquisition" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" @@ -52,7 +49,6 @@ func validateDataSource(opaqueDS DataSourceItem) error { return nil } - // formally validate YAML commonDS := configuration.DataSourceCommonCfg{} @@ -72,7 +68,6 @@ func validateDataSource(opaqueDS DataSourceItem) error { return fmt.Errorf("source is empty") } - // source must be known ds := acquisition.GetDataSourceIface(commonDS.Source) diff --git a/pkg/setup/detect_test.go b/pkg/setup/detect_test.go index 4f6ef0c33..adb5f1d43 100644 --- a/pkg/setup/detect_test.go +++ b/pkg/setup/detect_test.go @@ -10,6 +10,7 @@ import ( "github.com/lithammer/dedent" "github.com/stretchr/testify/require" + "github.com/crowdsecurity/go-cs-lib/pkg/csstring" "github.com/crowdsecurity/go-cs-lib/pkg/cstest" "github.com/crowdsecurity/crowdsec/pkg/setup" @@ -117,16 +118,16 @@ func TestVersionCheck(t *testing.T) { {"1", ">1", false, ""}, {"1", ">=1", true, ""}, {"1.0", "<1.0", false, ""}, - {"1", "<1", true, ""}, // XXX why? - {"1.3.5", "1.3", false, ""}, // XXX ok? + {"1", "<1", false, ""}, + {"1.3.5", "1.3", true, ""}, {"1.0", "<1.0", false, ""}, {"1.0", "<=1.0", true, ""}, {"2", ">1, <3", true, ""}, {"2", "<=2, >=2.2", false, ""}, {"2.3", "~2", true, ""}, {"2.3", "=2", true, ""}, - {"1.1.1", "=1.1", false, ""}, - {"1.1.1", "1.1", false, ""}, + {"1.1.1", "=1.1", true, ""}, + {"1.1.1", "1.1", true, ""}, {"1.1", "!=1.1.1", true, ""}, {"1.1", "~1.1.1", false, ""}, {"1.1.1", "~1.1", true, ""}, @@ -136,7 +137,7 @@ func TestVersionCheck(t *testing.T) { {"19.04", "=19.4", true, ""}, {"19.04", "~19.4", true, ""}, {"1.2.3", "~1.2", true, ""}, - {"1.2.3", "!=1.2", true, ""}, + {"1.2.3", "!=1.2", false, ""}, {"1.2.3", "1.1.1 - 1.3.4", true, ""}, {"1.3.5", "1.1.1 - 1.3.4", false, ""}, {"1.3.5", "=1", true, ""}, @@ -1007,7 +1008,7 @@ func TestDetectDatasourceValidation(t *testing.T) { "DetectYaml": detectYaml, } - expectedErr, err := cstest.Interpolate(tc.expectedErr, data) + expectedErr, err := csstring.Interpolate(tc.expectedErr, data) require.NoError(err) detected, err := setup.Detect(detectYaml, setup.DetectOptions{}) diff --git a/pkg/types/datasource.go b/pkg/types/datasource.go new file mode 100644 index 000000000..39b83fbaf --- /dev/null +++ b/pkg/types/datasource.go @@ -0,0 +1,16 @@ +package types + +import ( + "time" +) + +type DataSource struct { + SourceURL string `yaml:"source_url"` + DestPath string `yaml:"dest_file"` + Type string `yaml:"type"` + //Control cache strategy on expensive regexps + Cache *bool `yaml:"cache"` + Strategy *string `yaml:"strategy"` + Size *int `yaml:"size"` + TTL *time.Duration `yaml:"ttl"` +} diff --git a/pkg/types/ip.go b/pkg/types/ip.go index 647fb4a63..5e4d7734f 100644 --- a/pkg/types/ip.go +++ b/pkg/types/ip.go @@ -6,13 +6,14 @@ import ( "math" "net" "strings" - - "github.com/pkg/errors" ) +// LastAddress returns the last address of a network func LastAddress(n net.IPNet) net.IP { + // get the last address by ORing the hostmask and the IP ip := n.IP.To4() if ip == nil { + // IPv6 ip = n.IP return net.IP{ ip[0] | ^n.Mask[0], ip[1] | ^n.Mask[1], ip[2] | ^n.Mask[2], @@ -35,7 +36,7 @@ func Addr2Ints(anyIP string) (int, int64, int64, int64, int64, error) { if strings.Contains(anyIP, "/") { _, net, err := net.ParseCIDR(anyIP) if err != nil { - return -1, 0, 0, 0, 0, errors.Wrapf(err, "while parsing range %s", anyIP) + return -1, 0, 0, 0, 0, fmt.Errorf("while parsing range %s: %w", anyIP, err) } return Range2Ints(*net) } @@ -47,7 +48,7 @@ func Addr2Ints(anyIP string) (int, int64, int64, int64, int64, error) { sz, start, end, err := IP2Ints(ip) if err != nil { - return -1, 0, 0, 0, 0, errors.Wrapf(err, "while parsing ip %s", anyIP) + return -1, 0, 0, 0, 0, fmt.Errorf("while parsing ip %s: %w", anyIP, err) } return sz, start, end, start, end, nil @@ -58,12 +59,12 @@ func Range2Ints(network net.IPNet) (int, int64, int64, int64, int64, error) { szStart, nwStart, sfxStart, err := IP2Ints(network.IP) if err != nil { - return -1, 0, 0, 0, 0, errors.Wrap(err, "converting first ip in range") + return -1, 0, 0, 0, 0, fmt.Errorf("converting first ip in range: %w", err) } lastAddr := LastAddress(network) szEnd, nwEnd, sfxEnd, err := IP2Ints(lastAddr) if err != nil { - return -1, 0, 0, 0, 0, errors.Wrap(err, "transforming last address of range") + return -1, 0, 0, 0, 0, fmt.Errorf("transforming last address of range: %w", err) } if szEnd != szStart { return -1, 0, 0, 0, 0, fmt.Errorf("inconsistent size for range first(%d) and last(%d) ip", szStart, szEnd) diff --git a/pkg/types/utils.go b/pkg/types/utils.go index caed6961e..0485db59e 100644 --- a/pkg/types/utils.go +++ b/pkg/types/utils.go @@ -2,13 +2,9 @@ package types import ( "bufio" - "bytes" - "encoding/gob" "fmt" - "io" "os" "path/filepath" - "regexp" "strconv" "strings" "time" @@ -40,19 +36,9 @@ func SetDefaultLoggerConfig(cfgMode string, cfgFolder string, cfgLevel log.Level if compress != nil { _compress = *compress } - /*cf. https://github.com/natefinch/lumberjack/issues/82 - let's create the file beforehand w/ the right perms */ - fname := cfgFolder + "/crowdsec.log" - // check if file exists - _, err := os.Stat(fname) - // create file if not exists, purposefully ignore errors - if os.IsNotExist(err) { - file, _ := os.OpenFile(fname, os.O_RDWR|os.O_CREATE, 0600) - file.Close() - } LogOutput = &lumberjack.Logger{ - Filename: fname, + Filename: filepath.Join(cfgFolder, "crowdsec.log"), MaxSize: _maxsize, MaxBackups: _maxfiles, MaxAge: _maxage, @@ -82,19 +68,6 @@ func ConfigureLogger(clog *log.Logger) error { return nil } -func Clone(a, b interface{}) error { - buff := new(bytes.Buffer) - enc := gob.NewEncoder(buff) - dec := gob.NewDecoder(buff) - if err := enc.Encode(a); err != nil { - return fmt.Errorf("failed cloning %T", a) - } - if err := dec.Decode(b); err != nil { - return fmt.Errorf("failed cloning %T", b) - } - return nil -} - func ParseDuration(d string) (time.Duration, error) { durationStr := d if strings.HasSuffix(d, "d") { @@ -115,67 +88,6 @@ func ParseDuration(d string) (time.Duration, error) { return duration, nil } -/*help to copy the file, ioutil doesn't offer the feature*/ - -func copyFileContents(src, dst string) (err error) { - in, err := os.Open(src) - if err != nil { - return - } - defer in.Close() - out, err := os.Create(dst) - if err != nil { - return - } - defer func() { - cerr := out.Close() - if err == nil { - err = cerr - } - }() - if _, err = io.Copy(out, in); err != nil { - return - } - err = out.Sync() - return -} - -/*copy the file, ioutile doesn't offer the feature*/ -func CopyFile(sourceSymLink, destinationFile string) (err error) { - sourceFile, err := filepath.EvalSymlinks(sourceSymLink) - if err != nil { - log.Infof("Not a symlink : %s", err) - sourceFile = sourceSymLink - } - - sourceFileStat, err := os.Stat(sourceFile) - if err != nil { - return - } - if !sourceFileStat.Mode().IsRegular() { - // cannot copy non-regular files (e.g., directories, - // symlinks, devices, etc.) - return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String()) - } - destinationFileStat, err := os.Stat(destinationFile) - if err != nil { - if !os.IsNotExist(err) { - return - } - } else { - if !(destinationFileStat.Mode().IsRegular()) { - return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String()) - } - if os.SameFile(sourceFileStat, destinationFileStat) { - return - } - } - if err = os.Link(sourceFile, destinationFile); err != nil { - err = copyFileContents(sourceFile, destinationFile) - } - return -} - func UtcNow() time.Time { return time.Now().UTC() } @@ -193,11 +105,3 @@ func GetLineCountForFile(filepath string) int { } return lc } - -// from https://github.com/acarl005/stripansi -var reStripAnsi = regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))") - -func StripAnsiString(str string) string { - // the byte version doesn't strip correctly - return reStripAnsi.ReplaceAllString(str, "") -} diff --git a/plugins/notifications/dummy/Makefile b/plugins/notifications/dummy/Makefile index 612ec6c86..d45d6f198 100644 --- a/plugins/notifications/dummy/Makefile +++ b/plugins/notifications/dummy/Makefile @@ -16,8 +16,3 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) - -.PHONY: vendor -vendor: - @echo "vendoring $(PLUGIN) plugin..." - @$(GOCMD) mod vendor diff --git a/plugins/notifications/email/Makefile b/plugins/notifications/email/Makefile index a386625ac..ae548af0a 100644 --- a/plugins/notifications/email/Makefile +++ b/plugins/notifications/email/Makefile @@ -16,8 +16,3 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) - -.PHONY: vendor -vendor: - @echo "vendoring $(PLUGIN) plugin..." - @$(GOCMD) mod vendor diff --git a/plugins/notifications/email/go.mod b/plugins/notifications/email/go.mod index 549249467..bfeb5b023 100644 --- a/plugins/notifications/email/go.mod +++ b/plugins/notifications/email/go.mod @@ -13,16 +13,16 @@ require ( ) require ( - github.com/fatih/color v1.13.0 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect - github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/text v0.7.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/email/go.sum b/plugins/notifications/email/go.sum index ace03e73b..8077850de 100644 --- a/plugins/notifications/email/go.sum +++ b/plugins/notifications/email/go.sum @@ -21,8 +21,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -62,14 +62,13 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -84,7 +83,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= github.com/xhit/go-simple-mail/v2 v2.10.0 h1:nib6RaJ4qVh5HD9UE9QJqnUZyWp3upv+Z6CFxaMj0V8= github.com/xhit/go-simple-mail/v2 v2.10.0/go.mod h1:kA1XbQfCI4JxQ9ccSN6VFyIEkkugOm7YiPkA5hKiQn4= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -117,18 +116,15 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/plugins/notifications/http/Makefile b/plugins/notifications/http/Makefile index 44ee8c58f..56f490772 100644 --- a/plugins/notifications/http/Makefile +++ b/plugins/notifications/http/Makefile @@ -16,8 +16,3 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) - -.PHONY: vendor -vendor: - @echo "vendoring $(PLUGIN) plugin..." - @$(GOCMD) mod vendor diff --git a/plugins/notifications/http/go.mod b/plugins/notifications/http/go.mod index a09f74276..5f599674e 100644 --- a/plugins/notifications/http/go.mod +++ b/plugins/notifications/http/go.mod @@ -12,16 +12,16 @@ require ( ) require ( - github.com/fatih/color v1.13.0 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect - github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/text v0.7.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/http/go.sum b/plugins/notifications/http/go.sum index e160060b1..c7ba83755 100644 --- a/plugins/notifications/http/go.sum +++ b/plugins/notifications/http/go.sum @@ -21,8 +21,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -62,14 +62,13 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -84,7 +83,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -115,18 +114,15 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/plugins/notifications/slack/Makefile b/plugins/notifications/slack/Makefile index e950eba92..f43303eb8 100644 --- a/plugins/notifications/slack/Makefile +++ b/plugins/notifications/slack/Makefile @@ -16,8 +16,3 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) - -.PHONY: vendor -vendor: - @echo "vendoring $(PLUGIN) plugin..." - @$(GOCMD) mod vendor diff --git a/plugins/notifications/slack/go.mod b/plugins/notifications/slack/go.mod index 92640aad1..f8ef3c801 100644 --- a/plugins/notifications/slack/go.mod +++ b/plugins/notifications/slack/go.mod @@ -13,18 +13,18 @@ require ( ) require ( - github.com/fatih/color v1.13.0 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/gorilla/websocket v1.4.2 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect - github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/text v0.7.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/slack/go.sum b/plugins/notifications/slack/go.sum index d5f30d5e7..d28c26c3f 100644 --- a/plugins/notifications/slack/go.sum +++ b/plugins/notifications/slack/go.sum @@ -21,8 +21,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/go-test/deep v1.0.4 h1:u2CU3YKy9I2pmu9pX0eq50wCgjfGIt539SqR7FbHiho= github.com/go-test/deep v1.0.4/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= @@ -66,14 +66,13 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -93,7 +92,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -124,18 +123,15 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/plugins/notifications/splunk/Makefile b/plugins/notifications/splunk/Makefile index a49c87bd6..a7f04f4d0 100644 --- a/plugins/notifications/splunk/Makefile +++ b/plugins/notifications/splunk/Makefile @@ -16,8 +16,3 @@ build: clean .PHONY: clean clean: @$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR) - -.PHONY: vendor -vendor: - @echo "vendoring $(PLUGIN) plugin..." - @$(GOCMD) mod vendor diff --git a/plugins/notifications/splunk/go.mod b/plugins/notifications/splunk/go.mod index 04e4c3ccc..e2d45c365 100644 --- a/plugins/notifications/splunk/go.mod +++ b/plugins/notifications/splunk/go.mod @@ -12,16 +12,16 @@ require ( ) require ( - github.com/fatih/color v1.13.0 // indirect + github.com/fatih/color v1.15.0 // indirect github.com/golang/protobuf v1.5.2 // indirect github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb // indirect - github.com/mattn/go-colorable v0.1.12 // indirect - github.com/mattn/go-isatty v0.0.14 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect github.com/oklog/run v1.0.0 // indirect - golang.org/x/net v0.9.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.9.0 // indirect + golang.org/x/net v0.7.0 // indirect + golang.org/x/sys v0.9.0 // indirect + golang.org/x/text v0.7.0 // indirect google.golang.org/genproto v0.0.0-20220502173005-c8bf987b8c21 // indirect google.golang.org/grpc v1.47.0 // indirect google.golang.org/protobuf v1.28.1 // indirect diff --git a/plugins/notifications/splunk/go.sum b/plugins/notifications/splunk/go.sum index e160060b1..c7ba83755 100644 --- a/plugins/notifications/splunk/go.sum +++ b/plugins/notifications/splunk/go.sum @@ -21,8 +21,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/go-control-plane v0.10.2-0.20220325020618-49ff273808a1/go.mod h1:KJwIaB5Mv44NWtYuAOFCVOjcI94vtpEz2JU/D2v6IjE= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= -github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -62,14 +62,13 @@ github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKe github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= github.com/jhump/protoreflect v1.6.0/go.mod h1:eaTn3RZAmMBcV0fifFvlm6VHNz3wSkYyXYWUh7ymB74= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= -github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12 h1:jF+Du6AlPIjs2BiUiQlKOX0rt3SujHxPnksPKZbaA40= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= -github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= -github.com/mattn/go-isatty v0.0.14 h1:yVuAays6BHfxijgZPzw+3Zlu5yQgKGP2/hcQbHb7S9Y= -github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mitchellh/go-testing-interface v0.0.0-20171004221916-a61a99592b77/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= @@ -84,7 +83,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -115,18 +114,15 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.9.0 h1:KS/R3tvhPqvJvwcKfnBHJwwthS11LRhmM5D59eEXa0s= +golang.org/x/sys v0.9.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= diff --git a/rpm/SPECS/crowdsec.spec b/rpm/SPECS/crowdsec.spec index 246c7b181..a57492eea 100644 --- a/rpm/SPECS/crowdsec.spec +++ b/rpm/SPECS/crowdsec.spec @@ -12,8 +12,6 @@ Patch0: crowdsec.unit.patch Patch1: user.patch BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) -BuildRequires: git -BuildRequires: make BuildRequires: systemd Requires: crontabs %{?fc33:BuildRequires: systemd-rpm-macros} @@ -27,7 +25,6 @@ Requires: crontabs %define version_number %(echo $VERSION) %define releasever %(echo $RELEASEVER) -%global local_version v%{version_number}-%{releasever}-rpm %global name crowdsec %global __mangle_shebangs_exclude_from /usr/bin/env @@ -38,7 +35,6 @@ Requires: crontabs %patch1 %build -BUILD_VERSION=%{local_version} make build sed -i "s#/usr/local/lib/crowdsec/plugins/#%{_libdir}/%{name}/plugins/#g" config/config.yaml %install diff --git a/test/README.md b/test/README.md index bbc40a388..7f34bd3db 100644 --- a/test/README.md +++ b/test/README.md @@ -56,9 +56,6 @@ architectures. ## pre-requisites - `git submodule init; git submodule update` - - `go install github.com/cloudflare/cfssl/cmd/cfssl@latest` - - `go install github.com/cloudflare/cfssl/cmd/cfssljson@latest` - - `go install github.com/mikefarah/yq/v4@latest` - `base64` - `bash>=4.4` - `curl` @@ -89,7 +86,7 @@ In BATS, you write tests in the form of Bash functions that have unique descriptions (the name of the test). You can do most things that you can normally do in a shell function. If there is any error condition, the test fails. A set of functions is provided to implement assertions, and a mechanism -of `setup`/`teardown` is provided a the level of individual tests (functions) +of `setup`/`teardown` is provided at the level of individual tests (functions) or group of tests (files). The stdout/stderr of the commands within the test function are captured by @@ -129,11 +126,6 @@ included in a larger test suite. The TAP specification is pretty minimalist and some glue may be needed. -Other tools that you can find useful: - - - [mikefarah/yq](https://github.com/mikefarah/yq) - to parse and update YAML files on the fly - - [aliou/bats.vim](https://github.com/aliou/bats.vim) - for syntax highlighting (use bash otherwise) - # setup and teardown If you have read the bats-core tutorial linked above, you are aware of the diff --git a/test/ansible/debug_tools.yml b/test/ansible/debug_tools.yml index 15baa7cab..769a973fe 100644 --- a/test/ansible/debug_tools.yml +++ b/test/ansible/debug_tools.yml @@ -14,3 +14,5 @@ - zsh-autosuggestions - zsh-syntax-highlighting - zsh-theme-powerlevel9k + when: + - ansible_facts.os_family == "Debian" diff --git a/test/ansible/provision_dependencies.yml b/test/ansible/provision_dependencies.yml index 891bcc16e..bcfe8fcca 100644 --- a/test/ansible/provision_dependencies.yml +++ b/test/ansible/provision_dependencies.yml @@ -13,6 +13,8 @@ - crowdsecurity.testing.git - crowdsecurity.testing.gcc - crowdsecurity.testing.gnu_make + - crowdsecurity.testing.pkg_config + - crowdsecurity.testing.re2 - crowdsecurity.testing.bats_requirements - name: "Install Postgres" diff --git a/test/ansible/requirements.yml b/test/ansible/requirements.yml index b1a28b70a..70f8ca394 100644 --- a/test/ansible/requirements.yml +++ b/test/ansible/requirements.yml @@ -11,11 +11,11 @@ roles: - src: gantsign.golang collections: + - name: ansible.posix - name: https://github.com/crowdsecurity/ansible-collection-crowdsecurity.testing.git type: git - version: main + version: v0.0.4 # - name: crowdsecurity.testing # source: ../../../crowdsecurity.testing # type: dir - diff --git a/test/ansible/roles/make_fixture/tasks/main.yml b/test/ansible/roles/make_fixture/tasks/main.yml index 39f3e1785..305cec3a6 100644 --- a/test/ansible/roles/make_fixture/tasks/main.yml +++ b/test/ansible/roles/make_fixture/tasks/main.yml @@ -1,5 +1,6 @@ # vim: set ft=yaml.ansible: --- + - name: "Set make_cmd = make (!bsd)" ansible.builtin.set_fact: make_cmd: make @@ -17,7 +18,8 @@ block: - name: "Make bats-build bats-fixture" ansible.builtin.command: - cmd: "{{ make_cmd }} bats-build bats-fixture" + # static build and we don't have to mess with LD_LIBRARY_PATH + cmd: "{{ make_cmd }} bats-build bats-fixture BUILD_STATIC=1" chdir: "{{ ansible_env.HOME }}/crowdsec" creates: "{{ ansible_env.HOME }}/crowdsec/test/local-init/init-config-data.tar" environment: diff --git a/test/ansible/vagrant/alma-8/Vagrantfile b/test/ansible/vagrant/alma-8/Vagrantfile index 4b42adb3a..e246c77ff 100644 --- a/test/ansible/vagrant/alma-8/Vagrantfile +++ b/test/ansible/vagrant/alma-8/Vagrantfile @@ -2,6 +2,10 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/alma8' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf -y install dnf-plugins-core kitty-terminfo + sudo dnf config-manager --set-enabled powertools + SHELL end common = '../common' diff --git a/test/ansible/vagrant/alma-9/Vagrantfile b/test/ansible/vagrant/alma-9/Vagrantfile index 0ac3e5fe3..9c3d1b67c 100644 --- a/test/ansible/vagrant/alma-9/Vagrantfile +++ b/test/ansible/vagrant/alma-9/Vagrantfile @@ -2,6 +2,10 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/alma9' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf -y install kitty-terminfo + sudo dnf config-manager --set-enabled crb + SHELL end common = '../common' diff --git a/test/ansible/vagrant/centos-7/Vagrantfile b/test/ansible/vagrant/centos-7/Vagrantfile index d7ac021d2..fd207359f 100644 --- a/test/ansible/vagrant/centos-7/Vagrantfile +++ b/test/ansible/vagrant/centos-7/Vagrantfile @@ -2,6 +2,8 @@ Vagrant.configure('2') do |config| config.vm.box = 'centos/7' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/centos-8/Vagrantfile b/test/ansible/vagrant/centos-8/Vagrantfile index 24c37ada9..a88d621c4 100644 --- a/test/ansible/vagrant/centos-8/Vagrantfile +++ b/test/ansible/vagrant/centos-8/Vagrantfile @@ -1,7 +1,11 @@ # frozen_string_literal: true Vagrant.configure('2') do |config| - config.vm.box = 'centos/stream8' + config.vm.box = 'generic/centos8s' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf -y install dnf-plugins-core kitty-terminfo + sudo dnf config-manager --set-enabled powertools + SHELL end common = '../common' diff --git a/test/ansible/vagrant/centos-9/Vagrantfile b/test/ansible/vagrant/centos-9/Vagrantfile index 412354f3d..1087225be 100644 --- a/test/ansible/vagrant/centos-9/Vagrantfile +++ b/test/ansible/vagrant/centos-9/Vagrantfile @@ -2,6 +2,10 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/centos9s' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf -y install dnf-plugins-core + sudo dnf config-manager --set-enabled crb + SHELL end common = '../common' diff --git a/test/ansible/vagrant/common b/test/ansible/vagrant/common index adafa08c5..4bc237a7e 100644 --- a/test/ansible/vagrant/common +++ b/test/ansible/vagrant/common @@ -1,46 +1,75 @@ # vim: set ft=ruby: # frozen_string_literal: true +def find_ansible_cfg + path = Pathname.new(Dir.pwd) + until path.root? + ansible_cfg = path + 'ansible.cfg' + return path if ansible_cfg.exist? + path = path.parent + end + nil # return nil if not found +end + Vagrant.configure('2') do |config| config.vm.define 'crowdsec' + if ARGV.any? { |arg| arg == 'up' || arg == 'provision' } + unless ENV['DB_BACKEND'] + $stderr.puts "\e[31mThe DB_BACKEND environment variable is not defined. Please set up the environment and try again.\e[0m" + exit 1 + end + end + + config.vm.provision 'shell', path: 'bootstrap' if File.exists?('bootstrap') + config.vm.synced_folder '.', '/vagrant', disabled: true + config.vm.provider :libvirt do |libvirt| libvirt.cpus = 1 libvirt.memory = 1536 end - config.vm.synced_folder '.', '/vagrant', disabled: true + path = find_ansible_cfg + if !path + puts "ansible.cfg not found" + end config.vm.provision 'ansible' do |ansible| # ansible.verbose = 'vvvv' - ansible.config_file = '../../ansible.cfg' - ansible.playbook = '../../run_all.yml' + ansible.config_file = (path + 'ansible.cfg').to_s + ansible.playbook = (path + 'run_all.yml').to_s + ansible.compatibility_mode = "2.0" end # same as above, to run the steps separately - # config.vm.provision 'ansible' do |provdep| - # provdep.config_file = '../../ansible.cfg' - # provdep.playbook = '../../provision_dependencies.yml' + # config.vm.provision 'ansible' do |ansible| + # ansible.config_file = (path + 'ansible.cfg').to_s + # ansible.playbook = (path + 'provision_dependencies.yml').to_s + # ansible.compatibility_mode = "2.0" # end - # config.vm.provision 'ansible' do |provtest| - # provtest.config_file = '../../ansible.cfg' - # provtest.playbook = '../../provision_test_suite.yml' + # config.vm.provision 'ansible' do |ansible| + # ansible.config_file = (path + 'ansible.cfg').to_s + # ansible.playbook = (path + 'provision_test_suite.yml').to_s + # ansible.compatibility_mode = "2.0" # end - # config.vm.provision 'ansible' do |preptest| - # preptest.config_file = '../../ansible.cfg' - # preptest.playbook = '../../install_binary_package.yml' + # config.vm.provision 'ansible' do |ansible| + # ansible.config_file = (path + 'ansible.cfg').to_s + # ansible.playbook = (path + 'install_binary_package.yml').to_s + # ansible.compatibility_mode = "2.0" # end - # config.vm.provision 'ansible' do |preptest| - # preptest.config_file = '../../ansible.cfg' - # preptest.playbook = '../../prepare_tests.yml' + # config.vm.provision 'ansible' do |ansible| + # ansible.config_file = (path + 'ansible.cfg').to_s + # ansible.playbook = (path + 'prepare_tests.yml').to_s + # ansible.compatibility_mode = "2.0" # end - # config.vm.provision 'ansible' do |runtests| - # runtests.config_file = '../../ansible.cfg' - # runtests.playbook = '../../run_tests.yml' + # config.vm.provision 'ansible' do |ansible| + # ansible.config_file = (path + 'ansible.cfg').to_s + # ansible.playbook = (path + 'run_tests.yml').to_s + # ansible.compatibility_mode = "2.0" # end end diff --git a/test/ansible/vagrant/debian-10-buster/Vagrantfile b/test/ansible/vagrant/debian-10-buster/Vagrantfile index 2b1a4e2da..8d0a04eaf 100644 --- a/test/ansible/vagrant/debian-10-buster/Vagrantfile +++ b/test/ansible/vagrant/debian-10-buster/Vagrantfile @@ -2,6 +2,8 @@ Vagrant.configure('2') do |config| config.vm.box = 'debian/buster64' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/debian-11-bullseye/Vagrantfile b/test/ansible/vagrant/debian-11-bullseye/Vagrantfile index 9166427cb..fbd2ca41e 100644 --- a/test/ansible/vagrant/debian-11-bullseye/Vagrantfile +++ b/test/ansible/vagrant/debian-11-bullseye/Vagrantfile @@ -2,6 +2,8 @@ Vagrant.configure('2') do |config| config.vm.box = 'debian/bullseye64' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/debian-12-bookworm/Vagrantfile b/test/ansible/vagrant/debian-12-bookworm/Vagrantfile new file mode 100644 index 000000000..68a33a947 --- /dev/null +++ b/test/ansible/vagrant/debian-12-bookworm/Vagrantfile @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'debian/bookworm64' + config.vm.provision "shell", inline: <<-SHELL + # sudo apt install -y kitty-terminfo + SHELL +end + +common = '../common' +load common if File.exist?(common) diff --git a/test/ansible/vagrant/debian-9-stretch/Vagrantfile b/test/ansible/vagrant/debian-9-stretch/Vagrantfile index 4c4e39cf7..d40b1640c 100644 --- a/test/ansible/vagrant/debian-9-stretch/Vagrantfile +++ b/test/ansible/vagrant/debian-9-stretch/Vagrantfile @@ -2,6 +2,9 @@ Vagrant.configure('2') do |config| config.vm.box = 'debian/stretch64' + config.vm.provision "shell", inline: <<-SHELL + sudo sed -i s/httpredir.debian.org/archive.debian.org/g /etc/apt/sources.list + SHELL end common = '../common' diff --git a/test/ansible/vagrant/debian-testing/Vagrantfile b/test/ansible/vagrant/debian-testing/Vagrantfile index 5e3b68e54..6cd2667a9 100644 --- a/test/ansible/vagrant/debian-testing/Vagrantfile +++ b/test/ansible/vagrant/debian-testing/Vagrantfile @@ -2,6 +2,8 @@ Vagrant.configure('2') do |config| config.vm.box = 'debian/testing64' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/experimental/hardenedbsd-13/Vagrantfile b/test/ansible/vagrant/experimental/hardenedbsd-13/Vagrantfile index 0d34ea126..975d39ea0 100644 --- a/test/ansible/vagrant/experimental/hardenedbsd-13/Vagrantfile +++ b/test/ansible/vagrant/experimental/hardenedbsd-13/Vagrantfile @@ -2,19 +2,10 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/hardenedbsd13' - config.vm.define 'crowdsec' - - config.vm.provision 'shell', path: 'bootstrap' - - config.vm.provider :libvirt do |libvirt| - libvirt.cpus = 1 - libvirt.memory = 1536 - end - - config.vm.synced_folder '.', '/vagrant', disabled: true - - config.vm.provision 'ansible' do |ansible| - ansible.config_file = '../../../ansible.cfg' - ansible.playbook = '../../../run_all.yml' - end + config.vm.provision "shell", inline: <<-SHELL + sudo pkg install python3 + SHELL end + +common = '../../common' +load common if File.exist?(common) diff --git a/test/ansible/vagrant/experimental/hardenedbsd-13/bootstrap b/test/ansible/vagrant/experimental/hardenedbsd-13/bootstrap deleted file mode 100755 index 370b1b68e..000000000 --- a/test/ansible/vagrant/experimental/hardenedbsd-13/bootstrap +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -unset IFS -set -euf - -sudo pkg install python3 diff --git a/test/ansible/vagrant/experimental/openbsd-6/Vagrantfile b/test/ansible/vagrant/experimental/openbsd-6/Vagrantfile new file mode 100644 index 000000000..1a1fc3421 --- /dev/null +++ b/test/ansible/vagrant/experimental/openbsd-6/Vagrantfile @@ -0,0 +1,14 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'generic/openbsd6' + # config.vm.box_version = '4.2.16' + config.vm.provision "shell", inline: <<-SHELL + sudo pkg_add python py3-pip gcc openssl-1.0.2up3 gtar-1.34 + # sudo pkg_add -u + # sudo pkg_add kitty + SHELL +end + +common = '../../common' +load common if File.exist?(common) diff --git a/test/ansible/vagrant/experimental/openbsd-6/skip b/test/ansible/vagrant/experimental/openbsd-6/skip new file mode 100755 index 000000000..18d4c60f4 --- /dev/null +++ b/test/ansible/vagrant/experimental/openbsd-6/skip @@ -0,0 +1,9 @@ +#!/bin/sh + +die() { + echo "$@" >&2 + exit 1 +} + +[ "${PACKAGE_TESTING}" = "true" ] && die "no package available for this distribution" +exit 0 diff --git a/test/ansible/vagrant/experimental/openbsd-7/Vagrantfile b/test/ansible/vagrant/experimental/openbsd-7/Vagrantfile index d7f9801e1..1c11a94dc 100644 --- a/test/ansible/vagrant/experimental/openbsd-7/Vagrantfile +++ b/test/ansible/vagrant/experimental/openbsd-7/Vagrantfile @@ -2,19 +2,13 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/openbsd7' - config.vm.define 'crowdsec' - - config.vm.provision 'shell', path: 'bootstrap' - - config.vm.provider :libvirt do |libvirt| - libvirt.cpus = 1 - libvirt.memory = 1536 - end - - config.vm.synced_folder '.', '/vagrant', disabled: true - - config.vm.provision 'ansible' do |ansible| - ansible.config_file = '../../../ansible.cfg' - ansible.playbook = '../../../run_all.yml' - end + # config.vm.box_version = '4.2.16' + config.vm.provision "shell", inline: <<-SHELL + sudo pkg_add python-3.9.16 py3-pip gcc-11.2.0p3 openssl-3.0.8 gtar-1.34 + # sudo pkg_add -u + # sudo pkg_add kitty + SHELL end + +common = '../../common' +load common if File.exist?(common) diff --git a/test/ansible/vagrant/experimental/openbsd-7/bootstrap b/test/ansible/vagrant/experimental/openbsd-7/bootstrap deleted file mode 100755 index 3b2480d3b..000000000 --- a/test/ansible/vagrant/experimental/openbsd-7/bootstrap +++ /dev/null @@ -1,6 +0,0 @@ -#!/bin/sh -unset IFS -set -euf - -sudo pkg_add -u -sudo pkg_add python-3.9.13 py3-pip gcc-11.2.0p2 openssl-3.0.3p0 gtar-1.34 truncate-5.2.1 diff --git a/test/ansible/vagrant/fedora-33/Vagrantfile b/test/ansible/vagrant/fedora-33/Vagrantfile index 49b5ee990..df6f06944 100644 --- a/test/ansible/vagrant/fedora-33/Vagrantfile +++ b/test/ansible/vagrant/fedora-33/Vagrantfile @@ -3,6 +3,8 @@ Vagrant.configure('2') do |config| # config.vm.box = "fedora/33-cloud-base" config.vm.box = 'generic/fedora33' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/fedora-34/Vagrantfile b/test/ansible/vagrant/fedora-34/Vagrantfile index 1d172c9c7..db2db8d08 100644 --- a/test/ansible/vagrant/fedora-34/Vagrantfile +++ b/test/ansible/vagrant/fedora-34/Vagrantfile @@ -3,6 +3,8 @@ Vagrant.configure('2') do |config| # config.vm.box = "fedora/34-cloud-base" config.vm.box = 'generic/fedora34' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/fedora-35/Vagrantfile b/test/ansible/vagrant/fedora-35/Vagrantfile index f11730764..c52604340 100644 --- a/test/ansible/vagrant/fedora-35/Vagrantfile +++ b/test/ansible/vagrant/fedora-35/Vagrantfile @@ -3,6 +3,8 @@ Vagrant.configure('2') do |config| # config.vm.box = 'fedora/35-cloud-base' config.vm.box = 'generic/fedora35' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/fedora-36/Vagrantfile b/test/ansible/vagrant/fedora-36/Vagrantfile index ef80f514c..0cd4fafd0 100644 --- a/test/ansible/vagrant/fedora-36/Vagrantfile +++ b/test/ansible/vagrant/fedora-36/Vagrantfile @@ -3,6 +3,8 @@ Vagrant.configure('2') do |config| # config.vm.box = "fedora/36-cloud-base" config.vm.box = 'generic/fedora36' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/fedora-37/Vagrantfile b/test/ansible/vagrant/fedora-37/Vagrantfile new file mode 100644 index 000000000..0eb8978e3 --- /dev/null +++ b/test/ansible/vagrant/fedora-37/Vagrantfile @@ -0,0 +1,11 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'generic/fedora37' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf -y install kitty-terminfo + SHELL +end + +common = '../common' +load common if File.exist?(common) diff --git a/test/ansible/vagrant/fedora-38/Vagrantfile b/test/ansible/vagrant/fedora-38/Vagrantfile new file mode 100644 index 000000000..0e5bf79ed --- /dev/null +++ b/test/ansible/vagrant/fedora-38/Vagrantfile @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = "fedora/38-cloud-base" + config.vm.provision "shell", inline: <<-SHELL + SHELL +end + +common = '../common' +load common if File.exist?(common) diff --git a/test/ansible/vagrant/freebsd-12/Vagrantfile b/test/ansible/vagrant/freebsd-12/Vagrantfile index 33e6b473f..b500a64d9 100644 --- a/test/ansible/vagrant/freebsd-12/Vagrantfile +++ b/test/ansible/vagrant/freebsd-12/Vagrantfile @@ -2,6 +2,9 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/freebsd12' + config.vm.provision "shell", inline: <<-SHELL + pkg install -y gtar + SHELL end common = '../common' diff --git a/test/ansible/vagrant/freebsd-13/Vagrantfile b/test/ansible/vagrant/freebsd-13/Vagrantfile index 851c04254..f416ad677 100644 --- a/test/ansible/vagrant/freebsd-13/Vagrantfile +++ b/test/ansible/vagrant/freebsd-13/Vagrantfile @@ -2,6 +2,9 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/freebsd13' + config.vm.provision "shell", inline: <<-SHELL + pkg install -y gtar + SHELL end common = '../common' diff --git a/test/ansible/vagrant/oracle-7/Vagrantfile b/test/ansible/vagrant/oracle-7/Vagrantfile index 638a6123b..bd435ca54 100644 --- a/test/ansible/vagrant/oracle-7/Vagrantfile +++ b/test/ansible/vagrant/oracle-7/Vagrantfile @@ -2,6 +2,9 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/oracle7' + config.vm.provision "shell", inline: <<-SHELL + sudo yum-config-manager --enable ol7_optional_latest + SHELL end common = '../common' diff --git a/test/ansible/vagrant/oracle-8/Vagrantfile b/test/ansible/vagrant/oracle-8/Vagrantfile index 425ad5e16..6744ea87d 100644 --- a/test/ansible/vagrant/oracle-8/Vagrantfile +++ b/test/ansible/vagrant/oracle-8/Vagrantfile @@ -2,6 +2,9 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/oracle8' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf config-manager --set-enabled ol8_codeready_builder + SHELL end common = '../common' diff --git a/test/ansible/vagrant/oracle-9/Vagrantfile b/test/ansible/vagrant/oracle-9/Vagrantfile index d4e3f618f..8dcb4a13b 100644 --- a/test/ansible/vagrant/oracle-9/Vagrantfile +++ b/test/ansible/vagrant/oracle-9/Vagrantfile @@ -2,6 +2,9 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/oracle9' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf config-manager --set-enabled ol9_codeready_builder + SHELL end common = '../common' diff --git a/test/ansible/vagrant/rocky-8/Vagrantfile b/test/ansible/vagrant/rocky-8/Vagrantfile index c7315cc67..27caf80c7 100644 --- a/test/ansible/vagrant/rocky-8/Vagrantfile +++ b/test/ansible/vagrant/rocky-8/Vagrantfile @@ -2,6 +2,10 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/rocky8' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf config-manager --set-enabled powertools + sudo dnf -y install kitty-terminfo + SHELL end common = '../common' diff --git a/test/ansible/vagrant/rocky-9/Vagrantfile b/test/ansible/vagrant/rocky-9/Vagrantfile index 0adb3ab51..a4d06e4a9 100644 --- a/test/ansible/vagrant/rocky-9/Vagrantfile +++ b/test/ansible/vagrant/rocky-9/Vagrantfile @@ -2,6 +2,10 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/rocky9' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf config-manager --set-enabled crb + sudo dnf -y install kitty-terminfo + SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-16.04-xenial/Vagrantfile b/test/ansible/vagrant/ubuntu-16.04-xenial/Vagrantfile index 86646ee7a..3af880994 100644 --- a/test/ansible/vagrant/ubuntu-16.04-xenial/Vagrantfile +++ b/test/ansible/vagrant/ubuntu-16.04-xenial/Vagrantfile @@ -2,6 +2,8 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu1604' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-18.04-bionic/Vagrantfile b/test/ansible/vagrant/ubuntu-18.04-bionic/Vagrantfile index 70a77806b..5e25a2ee8 100644 --- a/test/ansible/vagrant/ubuntu-18.04-bionic/Vagrantfile +++ b/test/ansible/vagrant/ubuntu-18.04-bionic/Vagrantfile @@ -3,6 +3,8 @@ Vagrant.configure('2') do |config| # the official boxes only supports virtualbox config.vm.box = 'generic/ubuntu1804' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-20.04-focal/Vagrantfile b/test/ansible/vagrant/ubuntu-20.04-focal/Vagrantfile index 0006ae926..ea5b33907 100644 --- a/test/ansible/vagrant/ubuntu-20.04-focal/Vagrantfile +++ b/test/ansible/vagrant/ubuntu-20.04-focal/Vagrantfile @@ -2,6 +2,9 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu2004' + config.vm.provision "shell", inline: <<-SHELL + sudo apt install -y kitty-terminfo + SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-22.04-jammy/Vagrantfile b/test/ansible/vagrant/ubuntu-22.04-jammy/Vagrantfile index c0ccee54b..9e17f71fb 100644 --- a/test/ansible/vagrant/ubuntu-22.04-jammy/Vagrantfile +++ b/test/ansible/vagrant/ubuntu-22.04-jammy/Vagrantfile @@ -2,6 +2,8 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu2204' + config.vm.provision "shell", inline: <<-SHELL + SHELL end common = '../common' diff --git a/test/ansible/vagrant/ubuntu-22.10-kinetic/Vagrantfile b/test/ansible/vagrant/ubuntu-22.10-kinetic/Vagrantfile new file mode 100644 index 000000000..6c15b0a1e --- /dev/null +++ b/test/ansible/vagrant/ubuntu-22.10-kinetic/Vagrantfile @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'generic/ubuntu2210' + config.vm.provision "shell", inline: <<-SHELL + SHELL +end + +common = '../common' +load common if File.exist?(common) diff --git a/test/ansible/vagrant/ubuntu-23.04-lunar/Vagrantfile b/test/ansible/vagrant/ubuntu-23.04-lunar/Vagrantfile new file mode 100644 index 000000000..f40fb7bd5 --- /dev/null +++ b/test/ansible/vagrant/ubuntu-23.04-lunar/Vagrantfile @@ -0,0 +1,10 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'bento/ubuntu-23.04' + config.vm.provision "shell", inline: <<-SHELL + SHELL +end + +common = '../common' +load common if File.exist?(common) diff --git a/test/ansible/vagrant/wizard/centos-8/Vagrantfile b/test/ansible/vagrant/wizard/centos-8/Vagrantfile index 2df31a392..9db09a4ce 100644 --- a/test/ansible/vagrant/wizard/centos-8/Vagrantfile +++ b/test/ansible/vagrant/wizard/centos-8/Vagrantfile @@ -1,38 +1,13 @@ # frozen_string_literal: true Vagrant.configure('2') do |config| - config.vm.box = 'centos/stream8' - config.vm.define 'wizard' - - config.vm.provision 'shell', path: 'bootstrap' - - config.vm.provider :libvirt do |libvirt| - libvirt.cpus = 4 - libvirt.memory = 4096 - end - - config.vm.synced_folder '.', '/vagrant', disabled: true - - # install the dependencies for functional tests - - config.vm.provision 'ansible' do |provdep| - provdep.config_file = '../../../ansible.cfg' - provdep.playbook = '../../../provision_dependencies.yml' - end - - config.vm.provision 'ansible' do |provtest| - provtest.config_file = '../../../ansible.cfg' - provtest.playbook = '../../../provision_test_suite.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../prepare_tests.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../run_wizard_tests.yml' - end - + config.vm.box = 'generic/centos8s' + config.vm.provision "shell", inline: <<-SHELL + sudo dnf -y install dnf-plugins-core kitty-terminfo + dnf config-manager --set-enabled powertools + # sudo dnf -y update + SHELL end + +common = '../common' +load common if File.exists?(common) diff --git a/test/ansible/vagrant/wizard/centos-8/bootstrap b/test/ansible/vagrant/wizard/centos-8/bootstrap deleted file mode 100755 index b33ad9c88..000000000 --- a/test/ansible/vagrant/wizard/centos-8/bootstrap +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -unset IFS -set -euf - -sudo dnf -y update diff --git a/test/ansible/vagrant/wizard/common b/test/ansible/vagrant/wizard/common new file mode 100644 index 000000000..be1820914 --- /dev/null +++ b/test/ansible/vagrant/wizard/common @@ -0,0 +1,67 @@ +# vim: set ft=ruby: +# frozen_string_literal: true + +def find_ansible_cfg + path = Pathname.new(Dir.pwd) + until path.root? + ansible_cfg = path + 'ansible.cfg' + return path if ansible_cfg.exist? + path = path.parent + end + nil # return nil if not found +end + +Vagrant.configure('2') do |config| + config.vm.define 'wizard' + + if ARGV.any? { |arg| arg == 'up' || arg == 'provision' } + unless ENV['DB_BACKEND'] + $stderr.puts "\e[31mThe DB_BACKEND environment variable is not defined. Please set up the environment and try again.\e[0m" + exit 1 + end + end + + config.vm.provision 'shell', path: 'bootstrap' if File.exists?('bootstrap') + config.vm.synced_folder '.', '/vagrant', disabled: true + + config.vm.provider :libvirt do |libvirt| + libvirt.cpus = 4 + libvirt.memory = 4096 + end + + path = find_ansible_cfg + if !path + puts "ansible.cfg not found" + end + + config.vm.provision 'ansible' do |ansible| + ansible.config_file = (path + 'ansible.cfg').to_s + ansible.playbook = (path + 'provision_dependencies.yml').to_s + ansible.compatibility_mode = "2.0" + end + + config.vm.provision 'ansible' do |ansible| + ansible.config_file = (path + 'ansible.cfg').to_s + ansible.playbook = (path + 'provision_test_suite.yml').to_s + ansible.compatibility_mode = "2.0" + end + + config.vm.provision 'ansible' do |ansible| + ansible.config_file = (path + 'ansible.cfg').to_s + ansible.playbook = (path + 'prepare_tests.yml').to_s + ansible.compatibility_mode = "2.0" + end + + config.vm.provision 'ansible' do |ansible| + ansible.config_file = (path + 'ansible.cfg').to_s + ansible.playbook = (path + 'debug_tools.yml').to_s + ansible.compatibility_mode = "2.0" + end + + config.vm.provision 'ansible' do |ansible| + ansible.config_file = (path + 'ansible.cfg').to_s + ansible.playbook = (path + 'run_wizard_tests.yml').to_s + ansible.compatibility_mode = "2.0" + end + +end diff --git a/test/ansible/vagrant/wizard/debian-10-buster/Vagrantfile b/test/ansible/vagrant/wizard/debian-10-buster/Vagrantfile new file mode 100644 index 000000000..3b10b312d --- /dev/null +++ b/test/ansible/vagrant/wizard/debian-10-buster/Vagrantfile @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'debian/buster64' + config.vm.provision "shell", inline: <<-SHELL + sudo apt update + sudo apt install -y aptitude kitty-terminfo + SHELL +end + +common = '../common' +load common if File.exists?(common) diff --git a/test/ansible/vagrant/wizard/debian-11-bullseye/Vagrantfile b/test/ansible/vagrant/wizard/debian-11-bullseye/Vagrantfile new file mode 100644 index 000000000..6dd7bb2fc --- /dev/null +++ b/test/ansible/vagrant/wizard/debian-11-bullseye/Vagrantfile @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'debian/bullseye64' + config.vm.provision "shell", inline: <<-SHELL + sudo apt update + sudo apt install -y aptitude kitty-terminfo + SHELL +end + +common = '../common' +load common if File.exists?(common) diff --git a/test/ansible/vagrant/wizard/debian-12-bookworm/Vagrantfile b/test/ansible/vagrant/wizard/debian-12-bookworm/Vagrantfile new file mode 100644 index 000000000..5ccf234eb --- /dev/null +++ b/test/ansible/vagrant/wizard/debian-12-bookworm/Vagrantfile @@ -0,0 +1,12 @@ +# frozen_string_literal: true + +Vagrant.configure('2') do |config| + config.vm.box = 'debian/bookworm64' + config.vm.provision "shell", inline: <<-SHELL + sudo apt update + sudo apt install -y aptitude kitty-terminfo + SHELL +end + +common = '../common' +load common if File.exists?(common) diff --git a/test/ansible/vagrant/wizard/debian-bullseye/Vagrantfile b/test/ansible/vagrant/wizard/debian-bullseye/Vagrantfile deleted file mode 100644 index 1d6993a4e..000000000 --- a/test/ansible/vagrant/wizard/debian-bullseye/Vagrantfile +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'debian/bullseye64' - config.vm.define 'wizard' - - config.vm.provision 'shell', path: 'bootstrap' - - config.vm.provider :libvirt do |libvirt| - libvirt.cpus = 4 - libvirt.memory = 4096 - end - - config.vm.synced_folder '.', '/vagrant', disabled: true - - # install the dependencies for functional tests - - config.vm.provision 'ansible' do |provdep| - provdep.config_file = '../../../ansible.cfg' - provdep.playbook = '../../../provision_dependencies.yml' - end - - config.vm.provision 'ansible' do |provtest| - provtest.config_file = '../../../ansible.cfg' - provtest.playbook = '../../../provision_test_suite.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../prepare_tests.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../debug_tools.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../run_wizard_tests.yml' - end - -end diff --git a/test/ansible/vagrant/wizard/debian-bullseye/bootstrap b/test/ansible/vagrant/wizard/debian-bullseye/bootstrap deleted file mode 100755 index 6a5df521a..000000000 --- a/test/ansible/vagrant/wizard/debian-bullseye/bootstrap +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -unset IFS -set -euf - -sudo apt install -y aptitude diff --git a/test/ansible/vagrant/wizard/debian-buster/Vagrantfile b/test/ansible/vagrant/wizard/debian-buster/Vagrantfile deleted file mode 100644 index d4380a945..000000000 --- a/test/ansible/vagrant/wizard/debian-buster/Vagrantfile +++ /dev/null @@ -1,43 +0,0 @@ -# frozen_string_literal: true - -Vagrant.configure('2') do |config| - config.vm.box = 'debian/buster64' - config.vm.define 'wizard' - - config.vm.provision 'shell', path: 'bootstrap' - - config.vm.provider :libvirt do |libvirt| - libvirt.cpus = 4 - libvirt.memory = 4096 - end - - config.vm.synced_folder '.', '/vagrant', disabled: true - - # install the dependencies for functional tests - - config.vm.provision 'ansible' do |provdep| - provdep.config_file = '../../../ansible.cfg' - provdep.playbook = '../../../provision_dependencies.yml' - end - - config.vm.provision 'ansible' do |provtest| - provtest.config_file = '../../../ansible.cfg' - provtest.playbook = '../../../provision_test_suite.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../prepare_tests.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../debug_tools.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../run_wizard_tests.yml' - end - -end diff --git a/test/ansible/vagrant/wizard/debian-buster/bootstrap b/test/ansible/vagrant/wizard/debian-buster/bootstrap deleted file mode 100755 index 6a5df521a..000000000 --- a/test/ansible/vagrant/wizard/debian-buster/bootstrap +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -unset IFS -set -euf - -sudo apt install -y aptitude diff --git a/test/ansible/vagrant/wizard/fedora-36/Vagrantfile b/test/ansible/vagrant/wizard/fedora-36/Vagrantfile index 09ee8a39d..969a8e70c 100644 --- a/test/ansible/vagrant/wizard/fedora-36/Vagrantfile +++ b/test/ansible/vagrant/wizard/fedora-36/Vagrantfile @@ -2,37 +2,10 @@ Vagrant.configure('2') do |config| config.vm.box = 'fedora/36-cloud-base' - config.vm.define 'wizard' - - config.vm.provision 'shell', path: 'bootstrap' - - config.vm.provider :libvirt do |libvirt| - libvirt.cpus = 4 - libvirt.memory = 4096 - end - - config.vm.synced_folder '.', '/vagrant', disabled: true - - # install the dependencies for functional tests - - config.vm.provision 'ansible' do |provdep| - provdep.config_file = '../../../ansible.cfg' - provdep.playbook = '../../../provision_dependencies.yml' - end - - config.vm.provision 'ansible' do |provtest| - provtest.config_file = '../../../ansible.cfg' - provtest.playbook = '../../../provision_test_suite.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../prepare_tests.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../run_wizard_tests.yml' - end - + config.vm.provision "shell", inline: <<-SHELL + # sudo dnf -y update + SHELL end + +common = '../common' +load common if File.exists?(common) diff --git a/test/ansible/vagrant/wizard/fedora-36/bootstrap b/test/ansible/vagrant/wizard/fedora-36/bootstrap deleted file mode 100755 index b33ad9c88..000000000 --- a/test/ansible/vagrant/wizard/fedora-36/bootstrap +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -unset IFS -set -euf - -sudo dnf -y update diff --git a/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/Vagrantfile b/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/Vagrantfile index 933dabab5..c13d2f946 100644 --- a/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/Vagrantfile +++ b/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/Vagrantfile @@ -2,42 +2,10 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu2204' - config.vm.define 'wizard' - - config.vm.provision 'shell', path: 'bootstrap' - - config.vm.provider :libvirt do |libvirt| - libvirt.cpus = 4 - libvirt.memory = 4096 - end - - config.vm.synced_folder '.', '/vagrant', disabled: true - - # install the dependencies for functional tests - - config.vm.provision 'ansible' do |provdep| - provdep.config_file = '../../../ansible.cfg' - provdep.playbook = '../../../provision_dependencies.yml' - end - - config.vm.provision 'ansible' do |provtest| - provtest.config_file = '../../../ansible.cfg' - provtest.playbook = '../../../provision_test_suite.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../prepare_tests.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../debug_tools.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../run_wizard_tests.yml' - end - + config.vm.provision "shell", inline: <<-SHELL + sudo apt install -y aptitude kitty-terminfo + SHELL end + +common = '../common' +load common if File.exists?(common) diff --git a/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/bootstrap b/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/bootstrap deleted file mode 100755 index 6a5df521a..000000000 --- a/test/ansible/vagrant/wizard/ubuntu-22.04-jammy/bootstrap +++ /dev/null @@ -1,5 +0,0 @@ -#!/bin/sh -unset IFS -set -euf - -sudo apt install -y aptitude diff --git a/test/ansible/vagrant/wizard/ubuntu-22.10-kinetic/Vagrantfile b/test/ansible/vagrant/wizard/ubuntu-22.10-kinetic/Vagrantfile index c3e8759aa..d0e2e3cda 100644 --- a/test/ansible/vagrant/wizard/ubuntu-22.10-kinetic/Vagrantfile +++ b/test/ansible/vagrant/wizard/ubuntu-22.10-kinetic/Vagrantfile @@ -2,42 +2,10 @@ Vagrant.configure('2') do |config| config.vm.box = 'generic/ubuntu2210' - config.vm.define 'wizard' - - config.vm.provision 'shell', path: 'bootstrap' - - config.vm.provider :libvirt do |libvirt| - libvirt.cpus = 4 - libvirt.memory = 4096 - end - - config.vm.synced_folder '.', '/vagrant', disabled: true - - # install the dependencies for functional tests - - config.vm.provision 'ansible' do |provdep| - provdep.config_file = '../../../ansible.cfg' - provdep.playbook = '../../../provision_dependencies.yml' - end - - config.vm.provision 'ansible' do |provtest| - provtest.config_file = '../../../ansible.cfg' - provtest.playbook = '../../../provision_test_suite.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../prepare_tests.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../debug_tools.yml' - end - - config.vm.provision 'ansible' do |preptest| - preptest.config_file = '../../../ansible.cfg' - preptest.playbook = '../../../run_wizard_tests.yml' - end - + config.vm.provision "shell", inline: <<-SHELL + sudo apt install -y aptitude kitty-terminfo + SHELL end + +common = '../common' +load common if File.exists?(common) diff --git a/test/ansible/vars/go.yml b/test/ansible/vars/go.yml index 683b4fbbe..929e412cd 100644 --- a/test/ansible/vars/go.yml +++ b/test/ansible/vars/go.yml @@ -1,5 +1,5 @@ # vim: set ft=yaml.ansible: --- -golang_version: "1.20.4" +golang_version: "1.20.5" golang_install_dir: "/opt/go/{{ golang_version }}" diff --git a/test/bats-detect/openresty-deb.bats b/test/bats-detect/openresty-deb.bats index c1e91949d..0c8bc3c9a 100644 --- a/test/bats-detect/openresty-deb.bats +++ b/test/bats-detect/openresty-deb.bats @@ -37,8 +37,13 @@ setup() { run -0 sudo gpg --yes --dearmor -o /usr/share/keyrings/openresty.gpg < <(output) run -0 sudo tee <<< "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/openresty.gpg] http://openresty.org/package/ubuntu $(lsb_release -sc) main" /etc/apt/sources.list.d/openresty.list else + release="$(lsb_release -sc)" + # Debian 12 package is not available as of 2023-07-3 + if [[ "$release" == "bookworm" ]]; then + release="bullseye" + fi run -0 sudo apt-key add - < <(output) - run -0 sudo tee <<< "deb http://openresty.org/package/debian $(lsb_release -sc) openresty" /etc/apt/sources.list.d/openresty.list + run -0 sudo tee <<< "deb http://openresty.org/package/debian $release openresty" /etc/apt/sources.list.d/openresty.list fi run -0 deb-update run -0 deb-install openresty diff --git a/test/bats-detect/openresty-rpm.bats b/test/bats-detect/openresty-rpm.bats index 6fc0a8a09..d4c3661bc 100644 --- a/test/bats-detect/openresty-rpm.bats +++ b/test/bats-detect/openresty-rpm.bats @@ -34,10 +34,14 @@ setup() { run -0 rpm-install redhat-lsb-core if [[ "$(lsb_release -is)" == "Fedora" ]]; then run -0 sudo curl -1sSLf "https://openresty.org/package/fedora/openresty.repo" -o "/etc/yum.repos.d/openresty.repo" - elif [[ "$(lsb_release -is)" == "CentOS" ]]; then + elif [[ "$(lsb_release -is)" == CentOS* ]]; then # must match CentOSStream run -0 sudo curl -1sSLf "https://openresty.org/package/centos/openresty.repo" -o "/etc/yum.repos.d/openresty.repo" fi - run -0 sudo dnf check-update + run sudo dnf check-update + # 0 = up to date, 100 = updates available + if [[ "$status" -ne 0 ]] && [[ "$status" -ne 100 ]]; then + fail "dnf check-update failed with status $status" + fi run -0 rpm-install openresty run -0 sudo systemctl enable openresty.service } diff --git a/test/bats.mk b/test/bats.mk index 65bb4a286..4eb7abcbf 100644 --- a/test/bats.mk +++ b/test/bats.mk @@ -10,7 +10,7 @@ ifdef PACKAGE_TESTING INIT_BACKEND = systemd CONFIG_BACKEND = global else - # LOCAL_DIR will contain contains a local instance of crowdsec, complete with + # LOCAL_DIR contains a local instance of crowdsec, complete with # configuration and data LOCAL_DIR = $(TEST_DIR)/local BIN_DIR = $(LOCAL_DIR)/bin @@ -51,6 +51,7 @@ export CONFIG_BACKEND="$(CONFIG_BACKEND)" export PACKAGE_TESTING="$(PACKAGE_TESTING)" export TEST_COVERAGE="$(TEST_COVERAGE)" export GOCOVERDIR="$(TEST_DIR)/coverage" +export PATH="$(TEST_DIR)/tools:$(PATH)" endef bats-all: bats-clean bats-build bats-fixture bats-test bats-test-hub diff --git a/test/bats/01_crowdsec.bats b/test/bats/01_crowdsec.bats index 75b29033e..a1a2861f6 100644 --- a/test/bats/01_crowdsec.bats +++ b/test/bats/01_crowdsec.bats @@ -151,9 +151,10 @@ teardown() { rm -f "$ACQUIS_DIR" config_set '.common.log_media="stdout"' - rune -124 timeout 2s "${CROWDSEC}" + rune -1 timeout 2s "${CROWDSEC}" # check warning - assert_stderr_line --partial "no acquisition file found" + assert_stderr --partial "no acquisition file found" + assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" } @test "crowdsec (error if acquisition_path and acquisition_dir are not defined)" { @@ -166,19 +167,56 @@ teardown() { config_set '.crowdsec_service.acquisition_dir=""' config_set '.common.log_media="stdout"' - rune -124 timeout 2s "${CROWDSEC}" + rune -1 timeout 2s "${CROWDSEC}" # check warning - assert_stderr_line --partial "no acquisition_path or acquisition_dir specified" + assert_stderr --partial "no acquisition_path or acquisition_dir specified" + assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" } @test "crowdsec (no error if acquisition_path is empty string but acquisition_dir is not empty)" { ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') - rm -f "$ACQUIS_YAML" config_set '.crowdsec_service.acquisition_path=""' ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir') mkdir -p "$ACQUIS_DIR" - touch "$ACQUIS_DIR"/foo.yaml + mv "$ACQUIS_YAML" "$ACQUIS_DIR"/foo.yaml rune -124 timeout 2s "${CROWDSEC}" + + # now, if foo.yaml is empty instead, there won't be valid datasources. + + cat /dev/null >"$ACQUIS_DIR"/foo.yaml + + rune -1 timeout 2s "${CROWDSEC}" + assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" } + +@test "crowdsec (disabled datasources)" { + config_set '.common.log_media="stdout"' + + # a datasource cannot run - missing journalctl command + + ACQUIS_DIR=$(config_get '.crowdsec_service.acquisition_dir') + mkdir -p "$ACQUIS_DIR" + cat >"$ACQUIS_DIR"/foo.yaml <<-EOT + source: journalctl + journalctl_filter: + - "_SYSTEMD_UNIT=ssh.service" + labels: + type: syslog + EOT + + rune -124 timeout 2s env PATH='' "${CROWDSEC}" + #shellcheck disable=SC2016 + assert_stderr --partial 'datasource '\''journalctl'\'' is not available: exec: "journalctl": executable file not found in $PATH' + + # if all datasources are disabled, crowdsec should exit + + ACQUIS_YAML=$(config_get '.crowdsec_service.acquisition_path') + rm -f "$ACQUIS_YAML" + config_set '.crowdsec_service.acquisition_path=""' + + rune -1 timeout 2s env PATH='' "${CROWDSEC}" + assert_stderr --partial "crowdsec init: while loading acquisition config: no datasource enabled" +} + diff --git a/test/bats/01_cscli.bats b/test/bats/01_cscli.bats index a01d936b7..ef825d2a8 100644 --- a/test/bats/01_cscli.bats +++ b/test/bats/01_cscli.bats @@ -101,9 +101,9 @@ teardown() { # check that LAPI configuration is loaded (human and json, not shows in raw) rune -0 cscli config show -o human - assert_line --regexp ".*- URL\s+: http://127.0.0.1:8080/" - assert_line --regexp ".*- Login\s+: githubciXXXXXXXXXXXXXXXXXXXXXXXX" - assert_line --regexp ".*- Credentials File\s+: .*/local_api_credentials.yaml" + assert_line --regexp ".*- URL +: http://127.0.0.1:8080/" + assert_line --regexp ".*- Login +: githubciXXXXXXXXXXXXXXXXXXXXXXXX" + assert_line --regexp ".*- Credentials File +: .*/local_api_credentials.yaml" rune -0 cscli config show -o json rune -0 jq -c '.API.Client.Credentials | [.url,.login]' <(output) @@ -182,7 +182,7 @@ teardown() { @test "cscli - empty LAPI credentials file" { LOCAL_API_CREDENTIALS=$(config_get '.api.client.credentials_path') - truncate -s 0 "${LOCAL_API_CREDENTIALS}" + : > "${LOCAL_API_CREDENTIALS}" rune -1 cscli lapi status assert_stderr --partial "no credentials or URL found in api client configuration '${LOCAL_API_CREDENTIALS}'" diff --git a/test/bats/04_capi.bats b/test/bats/04_capi.bats index 7015f2c5d..f4c9f49e0 100644 --- a/test/bats/04_capi.bats +++ b/test/bats/04_capi.bats @@ -60,5 +60,5 @@ setup() { ONLINE_API_CREDENTIALS_YAML="$(config_get '.api.server.online_client.credentials_path')" rm "${ONLINE_API_CREDENTIALS_YAML}" rune -1 cscli capi status - assert_stderr --partial "Local API is disabled, please run this command on the local API machine: loading online client credentials: failed to read api server credentials configuration file '${ONLINE_API_CREDENTIALS_YAML}': open ${ONLINE_API_CREDENTIALS_YAML}: no such file or directory" + assert_stderr --partial "local API is disabled, please run this command on the local API machine: loading online client credentials: failed to read api server credentials configuration file '${ONLINE_API_CREDENTIALS_YAML}': open ${ONLINE_API_CREDENTIALS_YAML}: no such file or directory" } diff --git a/test/bats/72_plugin_badconfig.bats b/test/bats/72_plugin_badconfig.bats index c216067f9..9640e3330 100644 --- a/test/bats/72_plugin_badconfig.bats +++ b/test/bats/72_plugin_badconfig.bats @@ -36,35 +36,35 @@ teardown() { config_set '.plugin_config.user="" | .plugin_config.group="nogroup"' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: both plugin user and group must be set" + assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set" } @test "misconfigured plugin, only group is empty" { config_set '(.plugin_config.user="nobody") | (.plugin_config.group="")' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: both plugin user and group must be set" + assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: both plugin user and group must be set" } @test "misconfigured plugin, user does not exist" { config_set '(.plugin_config.user="userdoesnotexist") | (.plugin_config.group="groupdoesnotexist")' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: user: unknown user userdoesnotexist" + assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: user: unknown user userdoesnotexist" } @test "misconfigured plugin, group does not exist" { config_set '(.plugin_config.user=strenv(USER)) | (.plugin_config.group="groupdoesnotexist")' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run local API: while loading plugin: while getting process attributes: group: unknown group groupdoesnotexist" + assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: while getting process attributes: group: unknown group groupdoesnotexist" } @test "bad plugin name" { config_set "${PROFILES_PATH}" '.notifications=["http_default"]' cp "${PLUGIN_DIR}"/notification-http "${PLUGIN_DIR}"/badname rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run local API: while loading plugin: plugin name ${PLUGIN_DIR}/badname is invalid. Name should be like {type-name}" + assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin name ${PLUGIN_DIR}/badname is invalid. Name should be like {type-name}" } @test "duplicate notification config" { @@ -85,14 +85,14 @@ teardown() { config_set "${PROFILES_PATH}" '.notifications=["http_default"]' chmod g+w "${PLUGIN_DIR}"/notification-http rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run local API: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is group writable, group writable plugins are invalid" + assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is group writable, group writable plugins are invalid" } @test "bad plugin permission (world writable)" { config_set "${PROFILES_PATH}" '.notifications=["http_default"]' chmod o+w "${PLUGIN_DIR}"/notification-http rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run local API: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is world writable, world writable plugins are invalid" + assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin: plugin at ${PLUGIN_DIR}/notification-http is world writable, world writable plugins are invalid" } @test "config.yaml: missing .plugin_config section" { @@ -116,9 +116,9 @@ teardown() { assert_stderr --partial "api server init: plugins are enabled, but config_paths.plugin_dir is not defined" } -@test "unable to run local API: while reading plugin config" { +@test "unable to run plugin broker: while reading plugin config" { config_set '.config_paths.notification_dir="/this/path/does/not/exist"' config_set "${PROFILES_PATH}" '.notifications=["http_default"]' rune -1 timeout 2s "${CROWDSEC}" - assert_stderr --partial "api server init: unable to run local API: while loading plugin config: open /this/path/does/not/exist: no such file or directory" + assert_stderr --partial "api server init: unable to run plugin broker: while loading plugin config: open /this/path/does/not/exist: no such file or directory" } diff --git a/test/bats/90_decisions.bats b/test/bats/90_decisions.bats index 3499f3e0e..bcb410de9 100644 --- a/test/bats/90_decisions.bats +++ b/test/bats/90_decisions.bats @@ -5,6 +5,9 @@ set -u setup_file() { load "../lib/setup_file.sh" + + TESTDATA="${BATS_TEST_DIRNAME}/testdata/90_decisions" + export TESTDATA } teardown_file() { @@ -56,8 +59,122 @@ teardown() { @test "cscli decisions list, incorrect parameters" { rune -1 cscli decisions list --until toto - assert_stderr --partial 'Unable to list decisions : performing request: API error: while parsing duration: time: invalid duration \"toto\"' + assert_stderr --partial 'unable to retrieve decisions: performing request: API error: while parsing duration: time: invalid duration \"toto\"' rune -1 cscli decisions list --until toto -o json rune -0 jq -c '[.level, .msg]' <(stderr | grep "^{") - assert_output '["fatal","Unable to list decisions : performing request: API error: while parsing duration: time: invalid duration \"toto\""]' + assert_output '["fatal","unable to retrieve decisions: performing request: API error: while parsing duration: time: invalid duration \"toto\""]' +} + +@test "cscli decisions import" { + # required input + rune -1 cscli decisions import + assert_stderr --partial 'required flag(s) \"input\" not set"' + + # unsupported format + rune -1 cscli decisions import -i - <<<'value\n5.6.7.8' --format xml + assert_stderr --partial "invalid format 'xml', expected one of 'json', 'csv', 'values'" + + # invalid defaults + rune -1 cscli decisions import --duration "" -i - <<<'value\n5.6.7.8' --format csv + assert_stderr --partial "--duration cannot be empty" + rune -1 cscli decisions import --scope "" -i - <<<'value\n5.6.7.8' --format csv + assert_stderr --partial "--scope cannot be empty" + rune -1 cscli decisions import --reason "" -i - <<<'value\n5.6.7.8' --format csv + assert_stderr --partial "--reason cannot be empty" + rune -1 cscli decisions import --type "" -i - <<<'value\n5.6.7.8' --format csv + assert_stderr --partial "--type cannot be empty" + + #---------- + # JSON + #---------- + + # import from file + rune -1 cscli decisions import -i "${TESTDATA}/json_decisions" + assert_stderr --partial "unable to guess format from file extension, please provide a format with --format flag" + + rune -0 cscli decisions import -i "${TESTDATA}/decisions.json" + assert_stderr --partial "Parsing json" + assert_stderr --partial "Imported 5 decisions" + + # import from stdin + rune -1 cscli decisions import -i /dev/stdin < <(cat "${TESTDATA}/decisions.json") + assert_stderr --partial "unable to guess format from file extension, please provide a format with --format flag" + rune -0 cscli decisions import -i /dev/stdin < <(cat "${TESTDATA}/decisions.json") --format json + assert_stderr --partial "Parsing json" + assert_stderr --partial "Imported 5 decisions" + + # invalid json + rune -1 cscli decisions import -i - <<<'{"blah":"blah"}' --format json + assert_stderr --partial 'Parsing json' + assert_stderr --partial 'json: cannot unmarshal object into Go value of type []main.decisionRaw' + + # json with extra data + rune -1 cscli decisions import -i - <<<'{"values":"1.2.3.4","blah":"blah"}' --format json + assert_stderr --partial 'Parsing json' + assert_stderr --partial 'json: cannot unmarshal object into Go value of type []main.decisionRaw' + + #---------- + # CSV + #---------- + + # import from file + rune -1 cscli decisions import -i "${TESTDATA}/csv_decisions" + assert_stderr --partial "unable to guess format from file extension, please provide a format with --format flag" + + rune -0 cscli decisions import -i "${TESTDATA}/decisions.csv" + assert_stderr --partial 'Parsing csv' + assert_stderr --partial 'Imported 5 decisions' + + # import from stdin + rune -1 cscli decisions import -i /dev/stdin < <(cat "${TESTDATA}/decisions.csv") + assert_stderr --partial "unable to guess format from file extension, please provide a format with --format flag" + rune -0 cscli decisions import -i /dev/stdin < <(cat "${TESTDATA}/decisions.csv") --format csv + assert_stderr --partial "Parsing csv" + assert_stderr --partial "Imported 5 decisions" + + # invalid csv + # XXX: improve validation + rune -0 cscli decisions import -i - <<<'value\n1.2.3.4,5.6.7.8' --format csv + assert_stderr --partial 'Parsing csv' + assert_stderr --partial "Imported 0 decisions" + + #---------- + # VALUES + #---------- + + # can use '-' as stdin + rune -0 cscli decisions import -i - --format values <<-EOT + 1.2.3.4 + 1.2.3.5 + 1.2.3.6 + EOT + assert_stderr --partial 'Parsing values' + assert_stderr --partial 'Imported 3 decisions' + + rune -0 cscli decisions import -i - --format values <<-EOT + 10.2.3.4 + 10.2.3.5 + 10.2.3.6 + EOT + assert_stderr --partial 'Parsing values' + assert_stderr --partial 'Imported 3 decisions' + + rune -1 cscli decisions import -i - --format values <<-EOT + whatever + EOT + assert_stderr --partial 'Parsing values' + assert_stderr --partial 'API error: unable to create alerts: whatever: invalid ip address / range' + + #---------- + # Batch + #---------- + + rune -0 cscli decisions import -i - --format values --batch 2 --debug <<-EOT + 1.2.3.4 + 1.2.3.5 + 1.2.3.6 + EOT + assert_stderr --partial 'Processing chunk of 2 decisions' + assert_stderr --partial 'Processing chunk of 1 decisions' + assert_stderr --partial 'Imported 3 decisions' } diff --git a/test/bats/testdata/90_decisions/csv_decisions b/test/bats/testdata/90_decisions/csv_decisions new file mode 100644 index 000000000..858654b63 --- /dev/null +++ b/test/bats/testdata/90_decisions/csv_decisions @@ -0,0 +1,6 @@ +origin,scope,value,reason,type,duration +cscli,ip,1.6.11.16,manual import from csv,ban,1h +cscli,ip,2.7.12.17,manual import from csv,ban,1h +cscli,ip,3.8.13.18,manual import from csv,ban,1h +cscli,ip,4.9.14.19,manual import from csv,ban,1h +cscli,ip,5.10.15.20,manual import from csv,ban,1h diff --git a/test/bats/testdata/90_decisions/decisions.csv b/test/bats/testdata/90_decisions/decisions.csv new file mode 100644 index 000000000..858654b63 --- /dev/null +++ b/test/bats/testdata/90_decisions/decisions.csv @@ -0,0 +1,6 @@ +origin,scope,value,reason,type,duration +cscli,ip,1.6.11.16,manual import from csv,ban,1h +cscli,ip,2.7.12.17,manual import from csv,ban,1h +cscli,ip,3.8.13.18,manual import from csv,ban,1h +cscli,ip,4.9.14.19,manual import from csv,ban,1h +cscli,ip,5.10.15.20,manual import from csv,ban,1h diff --git a/test/bats/testdata/90_decisions/decisions.json b/test/bats/testdata/90_decisions/decisions.json new file mode 100644 index 000000000..395458c97 --- /dev/null +++ b/test/bats/testdata/90_decisions/decisions.json @@ -0,0 +1,42 @@ +[ + { + "origin": "cscli", + "scope": "ip", + "value": "1.6.11.16", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + }, + { + "origin": "cscli", + "scope": "ip", + "value": "2.7.12.17", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + }, + { + "origin": "cscli", + "scope": "ip", + "value": "3.8.13.18", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + }, + { + "origin": "cscli", + "scope": "ip", + "value": "4.9.14.19", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + }, + { + "origin": "cscli", + "scope": "ip", + "value": "5.10.15.20", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + } +] diff --git a/test/bats/testdata/90_decisions/json_decisions b/test/bats/testdata/90_decisions/json_decisions new file mode 100644 index 000000000..395458c97 --- /dev/null +++ b/test/bats/testdata/90_decisions/json_decisions @@ -0,0 +1,42 @@ +[ + { + "origin": "cscli", + "scope": "ip", + "value": "1.6.11.16", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + }, + { + "origin": "cscli", + "scope": "ip", + "value": "2.7.12.17", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + }, + { + "origin": "cscli", + "scope": "ip", + "value": "3.8.13.18", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + }, + { + "origin": "cscli", + "scope": "ip", + "value": "4.9.14.19", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + }, + { + "origin": "cscli", + "scope": "ip", + "value": "5.10.15.20", + "reason": "manual import from csv", + "type": "ban", + "duration": "1h" + } +] diff --git a/test/bin/assert-crowdsec-not-running b/test/bin/assert-crowdsec-not-running index b545ebf0a..3171287d0 100755 --- a/test/bin/assert-crowdsec-not-running +++ b/test/bin/assert-crowdsec-not-running @@ -1,8 +1,15 @@ #!/usr/bin/env bash is_crowdsec_running() { - # ignore processes in containers - PIDS=$(pgrep --ns $$ -x 'crowdsec') + case $(uname) in + "Linux") + # ignore processes in containers + PIDS=$(pgrep --ns $$ -x 'crowdsec') + ;; + *) + PIDS=$(pgrep -x 'crowdsec') + ;; + esac } # The process can be slow, especially on CI and during test coverage. diff --git a/test/bin/check-requirements b/test/bin/check-requirements index 351c0a01b..0563eec01 100755 --- a/test/bin/check-requirements +++ b/test/bin/check-requirements @@ -54,47 +54,25 @@ check_pkill() { fi } -check_yq() { - # shellcheck disable=SC2016 - howto_install='You can install it with your favorite package manager (including snap) or with "go install github.com/mikefarah/yq/v4@latest" and add ~/go/bin to $PATH.' - if ! command -v yq >/dev/null; then - die "Missing required program 'yq'. ${howto_install}" - fi - if ! (yq --version | grep mikefarah >/dev/null); then - die "yq exists but it's not the one we need (mikefarah/yq). ${howto_install}" - fi -} - check_daemonizer() { if ! command -v daemonize >/dev/null; then die "missing required program 'daemonize' (package 'daemonize' or 'https://github.com/bmc/daemonize')" fi } -check_cfssl() { - # shellcheck disable=SC2016 - howto_install='You can install it with "go install github.com/cloudflare/cfssl/cmd/cfssl@latest" and add ~/go/bin to $PATH.' - if ! command -v cfssl >/dev/null; then - die "Missing required program 'cfssl'. ${howto_install}" - fi -} +echo "Checking requirements..." -check_cfssljson() { - # shellcheck disable=SC2016 - howto_install='You can install it with "go install github.com/cloudflare/cfssl/cmd/cfssljson@latest" and add ~/go/bin to $PATH.' - if ! command -v cfssljson >/dev/null; then - die "Missing required program 'cfssljson'. ${howto_install}" - fi -} +GOBIN=${TEST_DIR}/tools +export GOBIN +go install github.com/mikefarah/yq/v4@latest +go install github.com/cloudflare/cfssl/cmd/cfssl@master +go install github.com/cloudflare/cfssl/cmd/cfssljson@master check_bats_core check_curl check_daemonizer -check_cfssl -check_cfssljson check_jq check_nc check_base64 check_python3 -check_yq check_pkill diff --git a/test/lib/setup_file.sh b/test/lib/setup_file.sh index 22a9bc303..a4231c98e 100755 --- a/test/lib/setup_file.sh +++ b/test/lib/setup_file.sh @@ -127,6 +127,24 @@ is_db_sqlite() { } export -f is_db_sqlite +crowdsec_log() { + echo "$(config_get .common.log_dir)"/crowdsec.log +} +export -f crowdsec_log + +truncate_log() { + true > "$(crowdsec_log)" +} +export -f truncate_log + +assert_log() { + local oldout="${output:-}" + output="$(cat "$(crowdsec_log)")" + assert_output "$@" + output="${oldout}" +} +export -f assert_log + # Compare ignoring the key order, and allow "expected" without quoted identifiers. # Preserve the output variable in case the following commands require it. assert_json() { diff --git a/test/tools/.do-not-remove b/test/tools/.do-not-remove new file mode 100644 index 000000000..311928170 --- /dev/null +++ b/test/tools/.do-not-remove @@ -0,0 +1 @@ +this directory is populated by test dependencies, and is not checked in git