0ct0pu5/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'master' into postoverflow_reinject_meta

Browse source
This commit is contained in:
Thibault "bui" Koechlin 2024-02-14 13:39:28 +01:00 • committed by GitHub
commit 11d0855ca9
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
365 changed files with 22275 additions and 15016 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.github/governance.yml vendored
View file

@ -81,7 +81,7 @@ pull_request:
failure: Missing kind label to generate release automatically.
- prefix: area
list: [ "agent", "local-api", "cscli", "security", "configuration"]
list: [ "agent", "local-api", "cscli", "security", "configuration", "appsec"]
multiple: true
needs:
comment: |
@ -89,6 +89,7 @@ pull_request:
* `/area agent`
* `/area local-api`
* `/area cscli`
* `/area appsec`
* `/area security`
* `/area configuration`
@ -98,4 +99,4 @@ pull_request:
author_association:
collaborator: true
member: true
owner: true
owner: true

23
.github/workflows/bats-hub.yml vendored
View file

@ -1,4 +1,4 @@
name: Hub tests
name: (sub) Bats / Hub
on:
workflow_call:
@ -8,16 +8,13 @@ on:
GIST_BADGES_ID:
required: true
env:
PREFIX_TEST_NAMES_WITH_FILE: true
jobs:
build:
strategy:
matrix:
go-version: ["1.21.4"]
test-file: ["hub-1.bats", "hub-2.bats", "hub-3.bats"]
name: "Build + tests"
name: "Functional tests"
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
@ -28,27 +25,29 @@ jobs:
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Check out CrowdSec repository"
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: true
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
go-version: "1.21.7"
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
- name: "Build crowdsec and fixture"
run: make bats-clean bats-build bats-fixture BUILD_STATIC=1
- name: "Run hub tests"
run: make bats-test-hub
run: |
./test/bin/generate-hub-tests
./test/run-tests ./test/dyn-bats/${{ matrix.test-file }} --formatter $(pwd)/test/lib/color-formatter
- name: "Collect hub coverage"
run: ./test/bin/collect-hub-coverage >> $GITHUB_ENV

23
.github/workflows/bats-mysql.yml vendored
View file

@ -1,4 +1,4 @@
name: Functional tests (MySQL)
name: (sub) Bats / MySQL
on:
workflow_call:
@ -7,16 +7,9 @@ on:
required: true
type: string
env:
PREFIX_TEST_NAMES_WITH_FILE: true
jobs:
build:
strategy:
matrix:
go-version: ["1.21.4"]
name: "Build + tests"
name: "Functional tests"
runs-on: ubuntu-latest
timeout-minutes: 30
services:
@ -35,21 +28,21 @@ jobs:
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Check out CrowdSec repository"
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: true
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
go-version: "1.21.7"
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
- name: "Build crowdsec and fixture"
run: |
@ -62,7 +55,7 @@ jobs:
MYSQL_USER: root
- name: "Run tests"
run: make bats-test
run: ./test/run-tests ./test/bats --formatter $(pwd)/test/lib/color-formatter
env:
DB_BACKEND: mysql
MYSQL_HOST: 127.0.0.1

23
.github/workflows/bats-postgres.yml vendored
View file

@ -1,18 +1,11 @@
name: Functional tests (Postgres)
name: (sub) Bats / Postgres
on:
workflow_call:
env:
PREFIX_TEST_NAMES_WITH_FILE: true
jobs:
build:
strategy:
matrix:
go-version: ["1.21.4"]
name: "Build + tests"
name: "Functional tests"
runs-on: ubuntu-latest
timeout-minutes: 30
services:
@ -44,21 +37,21 @@ jobs:
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Check out CrowdSec repository"
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: true
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
go-version: "1.21.7"
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
- name: "Build crowdsec and fixture (DB_BACKEND: pgx)"
run: |
@ -71,7 +64,7 @@ jobs:
PGUSER: postgres
- name: "Run tests (DB_BACKEND: pgx)"
run: make bats-test
run: ./test/run-tests ./test/bats --formatter $(pwd)/test/lib/color-formatter
env:
DB_BACKEND: pgx
PGHOST: 127.0.0.1

21
.github/workflows/bats-sqlite-coverage.yml vendored
View file

@ -1,19 +1,14 @@
name: Functional tests (sqlite)
name: (sub) Bats / sqlite + coverage
on:
workflow_call:
env:
PREFIX_TEST_NAMES_WITH_FILE: true
TEST_COVERAGE: true
jobs:
build:
strategy:
matrix:
go-version: ["1.21.4"]
name: "Build + tests"
name: "Functional tests"
runs-on: ubuntu-latest
timeout-minutes: 20
@ -25,28 +20,28 @@ jobs:
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Check out CrowdSec repository"
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: true
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
go-version: "1.21.7"
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
- name: "Build crowdsec and fixture"
run: |
make clean bats-build bats-fixture BUILD_STATIC=1
- name: "Run tests"
run: make bats-test
run: ./test/run-tests ./test/bats --formatter $(pwd)/test/lib/color-formatter
- name: "Collect coverage data"
run: |

2
.github/workflows/cache-cleanup.yaml vendored
View file

@ -11,7 +11,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Cleanup
run: |

12
.github/workflows/ci-windows-build-msi.yml vendored
View file

@ -21,25 +21,21 @@ on:
jobs:
build:
strategy:
matrix:
go-version: ["1.21.4"]
name: Build
runs-on: windows-2019
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: false
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
go-version: "1.21.7"
- name: Build
run: make windows_installer BUILD_RE2_WASM=1

18
.github/workflows/codeql-analysis.yml vendored
View file

@ -44,14 +44,20 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
# required to pick up tags for BUILD_VERSION
fetch-depth: 0
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: "1.21.7"
cache-dependency-path: "**/go.sum"
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
uses: github/codeql-action/init@v3
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@ -71,14 +77,8 @@ jobs:
# and modify them (or add more) to build your code if your project
# uses a compiled language
- name: "Set up Go"
uses: actions/setup-go@v4
with:
go-version: "1.21.0"
cache-dependency-path: "**/go.sum"
- run: |
make clean build BUILD_RE2_WASM=1
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2
uses: github/codeql-action/analyze@v3

56
.github/workflows/docker-tests.yml vendored
View file

@ -15,59 +15,42 @@ on:
- 'README.md'
jobs:
test_docker_image:
test_flavor:
strategy:
# we could test all the flavors in a single pytest job,
# but let's split them (and the image build) in multiple runners for performance
matrix:
# can be slim, full or debian (no debian slim).
flavor: ["slim", "debian"]
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Check out the repo
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3
with:
config: .github/buildkit.toml
- name: "Build flavor: slim"
uses: docker/build-push-action@v4
- name: "Build image"
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile
tags: crowdsecurity/crowdsec:test-slim
target: slim
platforms: linux/amd64
load: true
cache-from: type=gha
cache-to: type=gha,mode=min
- name: "Build flavor: full"
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile
tags: crowdsecurity/crowdsec:test
target: full
platforms: linux/amd64
load: true
cache-from: type=gha
cache-to: type=gha,mode=min
- name: "Build flavor: full (debian)"
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile.debian
tags: crowdsecurity/crowdsec:test-debian
target: full
file: ./Dockerfile${{ matrix.flavor == 'debian' && '.debian' || '' }}
tags: crowdsecurity/crowdsec:test${{ matrix.flavor == 'full' && '' || '-' }}${{ matrix.flavor == 'full' && '' || matrix.flavor }}
target: ${{ matrix.flavor == 'debian' && 'full' || matrix.flavor }}
platforms: linux/amd64
load: true
cache-from: type=gha
cache-to: type=gha,mode=min
- name: "Setup Python"
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: "3.x"
@ -78,7 +61,7 @@ jobs:
- name: "Cache virtualenvs"
id: cache-pipenv
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: ~/.local/share/virtualenvs
key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
@ -95,9 +78,10 @@ jobs:
- name: "Run tests"
env:
CROWDSEC_TEST_VERSION: test
CROWDSEC_TEST_FLAVORS: slim,debian
CROWDSEC_TEST_FLAVORS: ${{ matrix.flavor }}
CROWDSEC_TEST_NETWORK: net-test
CROWDSEC_TEST_TIMEOUT: 90
# running serially to reduce test flakiness
run: |
cd docker/test
pipenv run pytest -n 2 --durations=0 --color=yes
pipenv run pytest -n 1 --durations=0 --color=yes

14
.github/workflows/go-tests-windows.yml vendored
View file

@ -20,25 +20,21 @@ env:
jobs:
build:
strategy:
matrix:
go-version: ["1.21.4"]
name: "Build + tests"
runs-on: windows-2022
steps:
- name: Check out CrowdSec repository
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: false
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
go-version: "1.21.7"
- name: Build
run: |
@ -60,7 +56,7 @@ jobs:
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
version: v1.54
version: v1.55
args: --issues-exit-code=1 --timeout 10m
only-new-issues: false
# the cache is already managed above, enabling it here

34
.github/workflows/go-tests.yml vendored
View file

@ -24,23 +24,18 @@ env:
RICHGO_FORCE_COLOR: 1
AWS_HOST: localstack
# these are to mimic aws config
AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_REGION: us-east-1
KINESIS_INITIALIZE_STREAMS: "stream-1-shard:1,stream-2-shards:2"
CROWDSEC_FEATURE_DISABLE_HTTP_RETRY_BACKOFF: true
jobs:
build:
strategy:
matrix:
go-version: ["1.21.4"]
name: "Build + tests"
runs-on: ubuntu-latest
services:
localstack:
image: localstack/localstack:1.3.0
image: localstack/localstack:3.0
ports:
- 4566:4566 # Localstack exposes all services on the same port
env:
@ -49,7 +44,7 @@ jobs:
KINESIS_ERROR_PROBABILITY: ""
DOCKER_HOST: unix:///var/run/docker.sock
KINESIS_INITIALIZE_STREAMS: ${{ env.KINESIS_INITIALIZE_STREAMS }}
HOSTNAME_EXTERNAL: ${{ env.AWS_HOST }} # Required so that resource urls are provided properly
LOCALSTACK_HOST: ${{ env.AWS_HOST }} # Required so that resource urls are provided properly
# e.g sqs url will get localhost if we don't set this env to map our service
options: >-
--name=localstack
@ -58,7 +53,7 @@ jobs:
--health-timeout=5s
--health-retries=3
zoo1:
image: confluentinc/cp-zookeeper:7.3.0
image: confluentinc/cp-zookeeper:7.4.3
ports:
- "2181:2181"
env:
@ -109,12 +104,12 @@ jobs:
--health-retries 5
loki:
image: grafana/loki:2.8.0
image: grafana/loki:2.9.1
ports:
- "3100:3100"
options: >-
--name=loki1
--health-cmd "wget -q -O - http://localhost:3100/ready | grep 'ready'"
--health-cmd "wget -q -O - http://localhost:3100/ready | grep 'ready'"
--health-interval 30s
--health-timeout 10s
--health-retries 5
@ -123,15 +118,20 @@ jobs:
steps:
- name: Check out CrowdSec repository
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: false
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
go-version: "1.21.7"
- name: Create localstack streams
run: |
aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-1-shard --shard-count 1
aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-2-shards --shard-count 2
- name: Build and run tests, static
run: |
@ -157,7 +157,7 @@ jobs:
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
version: v1.54
version: v1.55
args: --issues-exit-code=1 --timeout 10m
only-new-issues: false
# the cache is already managed above, enabling it here

47
.github/workflows/publish-docker-master.yml vendored Normal file
View file

@ -0,0 +1,47 @@
name: (push-master) Publish latest Docker images
on:
push:
branches: [ master ]
paths:
- 'pkg/**'
- 'cmd/**'
- 'mk/**'
- 'docker/docker_start.sh'
- 'docker/config.yaml'
- '.github/workflows/publish-docker-master.yml'
- '.github/workflows/publish-docker.yml'
- 'Dockerfile'
- 'Dockerfile.debian'
- 'go.mod'
- 'go.sum'
- 'Makefile'
jobs:
dev-alpine:
uses: ./.github/workflows/publish-docker.yml
with:
platform: linux/amd64
crowdsec_version: ""
image_version: dev
latest: false
push: true
slim: false
debian: false
secrets:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
dev-debian:
uses: ./.github/workflows/publish-docker.yml
with:
platform: linux/amd64
crowdsec_version: ""
image_version: dev
latest: false
push: true
slim: false
debian: true
secrets:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}

48
.github/workflows/publish-docker-release.yml vendored Normal file
View file

@ -0,0 +1,48 @@
name: (manual) Publish Docker images
on:
workflow_dispatch:
inputs:
image_version:
description: Docker Image version (base tag, i.e. v1.6.0-2)
required: true
crowdsec_version:
description: Crowdsec version (BUILD_VERSION)
required: true
latest:
description: Overwrite latest (and slim) tags?
default: false
required: true
push:
description: Really push?
default: false
required: true
jobs:
alpine:
uses: ./.github/workflows/publish-docker.yml
secrets:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
with:
image_version: ${{ github.event.inputs.image_version }}
crowdsec_version: ${{ github.event.inputs.crowdsec_version }}
latest: ${{ github.event.inputs.latest == 'true' }}
push: ${{ github.event.inputs.push == 'true' }}
slim: true
debian: false
platform: "linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6"
debian:
uses: ./.github/workflows/publish-docker.yml
secrets:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
with:
image_version: ${{ github.event.inputs.image_version }}
crowdsec_version: ${{ github.event.inputs.crowdsec_version }}
latest: ${{ github.event.inputs.latest == 'true' }}
push: ${{ github.event.inputs.push == 'true' }}
slim: false
debian: true
platform: "linux/amd64,linux/386,linux/arm64"

125
.github/workflows/publish-docker.yml vendored Normal file
View file

@ -0,0 +1,125 @@
name: (sub) Publish Docker images
on:
workflow_call:
secrets:
DOCKER_USERNAME:
required: true
DOCKER_PASSWORD:
required: true
inputs:
platform:
required: true
type: string
image_version:
required: true
type: string
crowdsec_version:
required: true
type: string
latest:
required: true
type: boolean
push:
required: true
type: boolean
slim:
required: true
type: boolean
debian:
required: true
type: boolean
jobs:
push_to_registry:
name: Push Docker image to registries
runs-on: ubuntu-latest
steps:
- name: Check out the repo
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
config: .github/buildkit.toml
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Prepare (slim)
if: ${{ inputs.slim }}
id: slim
run: |
DOCKERHUB_IMAGE=${{ secrets.DOCKER_USERNAME }}/crowdsec
GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
VERSION=${{ inputs.image_version }}
DEBIAN=${{ inputs.debian && '-debian' || '' }}
TAGS="${DOCKERHUB_IMAGE}:${VERSION}-slim${DEBIAN},${GHCR_IMAGE}:${VERSION}-slim${DEBIAN}"
if [[ ${{ inputs.latest }} == true ]]; then
TAGS=$TAGS,${DOCKERHUB_IMAGE}:slim${DEBIAN},${GHCR_IMAGE}:slim${DEBIAN}
fi
echo "tags=${TAGS}" >> $GITHUB_OUTPUT
echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
- name: Prepare (full)
id: full
run: |
DOCKERHUB_IMAGE=${{ secrets.DOCKER_USERNAME }}/crowdsec
GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
VERSION=${{ inputs.image_version }}
DEBIAN=${{ inputs.debian && '-debian' || '' }}
TAGS="${DOCKERHUB_IMAGE}:${VERSION}${DEBIAN},${GHCR_IMAGE}:${VERSION}${DEBIAN}"
if [[ ${{ inputs.latest }} == true ]]; then
TAGS=$TAGS,${DOCKERHUB_IMAGE}:latest${DEBIAN},${GHCR_IMAGE}:latest${DEBIAN}
fi
echo "tags=${TAGS}" >> $GITHUB_OUTPUT
echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
- name: Build and push image (slim)
if: ${{ inputs.slim }}
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile${{ inputs.debian && '.debian' || '' }}
push: ${{ inputs.push }}
tags: ${{ steps.slim.outputs.tags }}
target: slim
platforms: ${{ inputs.platform }}
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.slim.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}
build-args: |
BUILD_VERSION=${{ inputs.crowdsec_version }}
- name: Build and push image (full)
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile${{ inputs.debian && '.debian' || '' }}
push: ${{ inputs.push }}
tags: ${{ steps.full.outputs.tags }}
target: full
platforms: ${{ inputs.platform }}
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.full.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}
build-args: |
BUILD_VERSION=${{ inputs.crowdsec_version }}

14
.github/workflows/release_publish-package.yml → .github/workflows/publish-tarball-release.yml vendored
View file

@ -1,5 +1,5 @@
# .github/workflows/build-docker-image.yml
name: build
name: Release
on:
release:
@ -12,24 +12,20 @@ permissions:
jobs:
build:
strategy:
matrix:
go-version: ["1.21.4"]
name: Build and upload binary package
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: false
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
go-version: "1.21.7"
- name: Build the binaries
run: |

70
.github/workflows/publish_docker-image_on_master-debian.yml vendored
View file

@ -1,70 +0,0 @@
name: Publish Debian Docker image on Push to Master
on:
push:
branches: [ master ]
paths:
- 'pkg/**'
- 'cmd/**'
- 'plugins/**'
- 'docker/docker_start.sh'
- 'docker/config.yaml'
- '.github/workflows/publish_docker-image_on_master-debian.yml'
- 'Dockerfile.debian'
- 'go.mod'
- 'go.sum'
- 'Makefile'
jobs:
push_to_registry:
name: Push Debian Docker image to Docker Hub
runs-on: ubuntu-latest
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Prepare
id: prep
run: |
DOCKER_IMAGE=crowdsecurity/crowdsec
GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
VERSION=dev-debian
TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}"
echo "tags=${TAGS}" >> $GITHUB_OUTPUT
echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
config: .github/buildkit.toml
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push full image
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile.debian
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.prep.outputs.tags }}
platforms: linux/amd64
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.prep.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}
cache-from: type=gha
cache-to: type=gha,mode=min

70
.github/workflows/publish_docker-image_on_master.yml vendored
View file

@ -1,70 +0,0 @@
name: Publish Docker image on Push to Master
on:
push:
branches: [ master ]
paths:
- 'pkg/**'
- 'cmd/**'
- 'plugins/**'
- 'docker/docker_start.sh'
- 'docker/config.yaml'
- '.github/workflows/publish_docker-image_on_master.yml'
- 'Dockerfile'
- 'go.mod'
- 'go.sum'
- 'Makefile'
jobs:
push_to_registry:
name: Push Docker image to Docker Hub
runs-on: ubuntu-latest
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Prepare
id: prep
run: |
DOCKER_IMAGE=crowdsecurity/crowdsec
GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
VERSION=dev
TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}"
echo "tags=${TAGS}" >> $GITHUB_OUTPUT
echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
config: .github/buildkit.toml
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push full image
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.prep.outputs.tags }}
platforms: linux/amd64
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.prep.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}
cache-from: type=gha
cache-to: type=gha,mode=min

61
.github/workflows/release_publish_docker-image-debian.yml vendored
View file

@ -1,61 +0,0 @@
name: Publish Docker Debian image
on:
release:
types:
- released
- prereleased
workflow_dispatch:
jobs:
push_to_registry:
name: Push Docker debian image to Docker Hub
runs-on: ubuntu-latest
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Prepare
id: prep
run: |
DOCKER_IMAGE=crowdsecurity/crowdsec
VERSION=bullseye
if [[ $GITHUB_REF == refs/tags/* ]]; then
VERSION=${GITHUB_REF#refs/tags/}
elif [[ $GITHUB_REF == refs/heads/* ]]; then
VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -E 's#/+#-#g')
elif [[ $GITHUB_REF == refs/pull/* ]]; then
VERSION=pr-${{ github.event.number }}
fi
TAGS="${DOCKER_IMAGE}:${VERSION}-debian"
if [[ "${{ github.event.action }}" == "released" ]]; then
TAGS=$TAGS,${DOCKER_IMAGE}:latest-debian
fi
echo "version=${VERSION}" >> $GITHUB_OUTPUT
echo "tags=${TAGS}" >> $GITHUB_OUTPUT
echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
config: .github/buildkit.toml
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile.debian
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.prep.outputs.tags }}
platforms: linux/amd64,linux/arm64,linux/386
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.prep.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}

86
.github/workflows/release_publish_docker-image.yml vendored
View file

@ -1,86 +0,0 @@
name: Publish Docker image
on:
release:
types:
- released
- prereleased
jobs:
push_to_registry:
name: Push Docker image to Docker Hub
runs-on: ubuntu-latest
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Prepare
id: prep
run: |
DOCKER_IMAGE=crowdsecurity/crowdsec
GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
VERSION=edge
if [[ $GITHUB_REF == refs/tags/* ]]; then
VERSION=${GITHUB_REF#refs/tags/}
elif [[ $GITHUB_REF == refs/heads/* ]]; then
VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -E 's#/+#-#g')
elif [[ $GITHUB_REF == refs/pull/* ]]; then
VERSION=pr-${{ github.event.number }}
fi
TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}"
TAGS_SLIM="${DOCKER_IMAGE}:${VERSION}-slim"
if [[ ${{ github.event.action }} == released ]]; then
TAGS=$TAGS,${DOCKER_IMAGE}:latest,${GHCR_IMAGE}:latest
TAGS_SLIM=$TAGS_SLIM,${DOCKER_IMAGE}:slim
fi
echo "version=${VERSION}" >> $GITHUB_OUTPUT
echo "tags=${TAGS}" >> $GITHUB_OUTPUT
echo "tags_slim=${TAGS_SLIM}" >> $GITHUB_OUTPUT
echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
config: .github/buildkit.toml
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push slim image
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.prep.outputs.tags_slim }}
target: slim
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6,linux/386
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.prep.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}
- name: Build and push full image
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.prep.outputs.tags }}
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6,linux/386
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.prep.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}

4
.github/workflows/update_docker_hub_doc.yml vendored
View file

@ -1,4 +1,4 @@
name: Update Docker Hub README
name: (push-master) Update Docker Hub README
on:
push:
@ -13,7 +13,7 @@ jobs:
steps:
-
name: Check out the repo
uses: actions/checkout@v3
uses: actions/checkout@v4
if: ${{ github.repository_owner == 'crowdsecurity' }}
-
name: Update docker hub README

153
.golangci.yml
View file

@ -9,8 +9,24 @@ run:
- pkg/yamlpatch/merge_test.go
linters-settings:
cyclop:
# lower this after refactoring
max-complexity: 53
gci:
sections:
- standard
- default
- prefix(github.com/crowdsecurity)
- prefix(github.com/crowdsecurity/crowdsec)
gocognit:
# lower this after refactoring
min-complexity: 145
gocyclo:
min-complexity: 30
# lower this after refactoring
min-complexity: 49
funlen:
# Checks the number of lines in a function.
@ -28,11 +44,21 @@ linters-settings:
lll:
line-length: 140
maintidx:
# raise this after refactoring
under: 11
misspell:
locale: US
nestif:
# lower this after refactoring
min-complexity: 28
nlreturn:
block-size: 4
nolintlint:
allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
allow-unused: false # report any unused nolint directives
require-explanation: false # don't require an explanation for nolint directives
require-specific: false # don't require nolint directives to be specific about which linter is being skipped
@ -40,6 +66,13 @@ linters-settings:
interfacebloat:
max: 12
depguard:
rules:
main:
deny:
- pkg: "github.com/pkg/errors"
desc: "errors.Wrap() is deprecated in favor of fmt.Errorf()"
linters:
enable-all: true
disable:
@ -64,15 +97,21 @@ linters:
# - asasalint # check for pass []any as any in variadic func(...any)
# - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers
# - bidichk # Checks for dangerous unicode character sequences
# - bodyclose # checks whether HTTP response body is closed successfully
# - cyclop # checks function and package cyclomatic complexity
# - decorder # check declaration order and count of types, constants, variables and functions
# - depguard # Go linter that checks if package imports are in a list of acceptable packages
# - dupword # checks for duplicate words in the source code
# - durationcheck # check for two durations multiplied together
# - errcheck # Errcheck is a program for checking for unchecked errors in go programs. These unchecked errors can be critical bugs in some cases
# - errorlint # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13.
# - exportloopref # checks for pointers to enclosing loop variables
# - funlen # Tool for detection of long functions
# - ginkgolinter # enforces standards of using ginkgo and gomega
# - gochecknoinits # Checks that no init functions are present in Go code
# - gocognit # Computes and checks the cognitive complexity of functions
# - gocritic # Provides diagnostics that check for bugs, performance and style issues.
# - gocyclo # Computes and checks the cyclomatic complexity of functions
# - goheader # Checks is file header matches to pattern
# - gomoddirectives # Manage the use of 'replace', 'retract', and 'excludes' directives in go.mod.
# - gomodguard # Allow and block list linter for direct Go module dependencies. This is different from depguard where there are different block types for example version constraints and module recommendations.
@ -84,10 +123,15 @@ linters:
# - ineffassign # Detects when assignments to existing variables are not used
# - interfacebloat # A linter that checks the number of methods inside an interface.
# - logrlint # Check logr arguments.
# - maintidx # maintidx measures the maintainability index of each function.
# - makezero # Finds slice declarations with non-zero initial length
# - misspell # Finds commonly misspelled English words in comments
# - nakedret # Finds naked returns in functions greater than a specified function length
# - nestif # Reports deeply nested if statements
# - nilerr # Finds the code that returns nil even if it checks that the error is not nil.
# - nolintlint # Reports ill-formed or insufficient nolint directives
# - nonamedreturns # Reports all named returns
# - nosprintfhostport # Checks for misuse of Sprintf to construct a host with port in a URL.
# - predeclared # find code that shadows one of Go's predeclared identifiers
# - reassign # Checks that package variables are not reassigned
# - rowserrcheck # checks whether Err of rows is checked successfully
@ -100,38 +144,34 @@ linters:
# - unconvert # Remove unnecessary type conversions
# - unused # (megacheck): Checks Go code for unused constants, variables, functions and types
# - usestdlibvars # A linter that detect the possibility to use variables/constants from the Go standard library.
# - wastedassign # wastedassign finds wasted assignment statements.
#
# Recommended? (easy)
#
- depguard # Go linter that checks if package imports are in a list of acceptable packages
- dogsled # Checks assignments with too many blank identifiers (e.g. x, _, _, _, := f())
- errchkjson # Checks types passed to the json encoding functions. Reports unsupported types and optionally reports occations, where the check for the returned error can be omitted.
- errorlint # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13.
- exhaustive # check exhaustiveness of enum switch statements
- gci # Gci control golang package import order and make it always deterministic.
- godot # Check if comments end in a period
- gofmt # Gofmt checks whether code was gofmt-ed. By default this tool runs with -s option to check for code simplification
- goimports # In addition to fixing imports, goimports also formats your code in the same style as gofmt.
- gosec # (gas): Inspects source code for security problems
- inamedparam # reports interfaces with unnamed method parameters
- lll # Reports long lines
- musttag # enforce field tags in (un)marshaled structs
- nakedret # Finds naked returns in functions greater than a specified function length
- nonamedreturns # Reports all named returns
- nosprintfhostport # Checks for misuse of Sprintf to construct a host with port in a URL.
- promlinter # Check Prometheus metrics naming via promlint
- protogetter # Reports direct reads from proto message fields when getters should be used
- revive # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint.
- tagalign # check that struct tags are well aligned [fast: true, auto-fix: true]
- tagalign # check that struct tags are well aligned
- thelper # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers
- wastedassign # wastedassign finds wasted assignment statements.
- wrapcheck # Checks that errors returned from external packages are wrapped
#
# Recommended? (requires some work)
#
- bodyclose # checks whether HTTP response body is closed successfully
- containedctx # containedctx is a linter that detects struct contained context.Context field
- contextcheck # check the function whether use a non-inherited context
- errname # Checks that sentinel errors are prefixed with the `Err` and error types are suffixed with the `Error`.
@ -153,15 +193,10 @@ linters:
#
# Well intended, but not ready for this
#
- cyclop # checks function and package cyclomatic complexity
- dupl # Tool for code clone detection
- forcetypeassert # finds forced type assertions
- gocognit # Computes and checks the cognitive complexity of functions
- gocyclo # Computes and checks the cyclomatic complexity of functions
- godox # Tool for detection of FIXME, TODO and other comment keywords
- goerr113 # Golang linter to check the errors handling expressions
- maintidx # maintidx measures the maintainability index of each function.
- nestif # Reports deeply nested if statements
- paralleltest # paralleltest detects missing usage of t.Parallel() method in your Go test
- testpackage # linter that makes you use a separate _test package
@ -189,8 +224,11 @@ issues:
# break ‘em.” ― Terry Pratchett
max-issues-per-linter: 0
max-same-issues: 10
max-same-issues: 0
exclude-rules:
# Won't fix:
- path: go.mod
text: "replacement are not allowed: golang.org/x/time/rate"
@ -199,30 +237,10 @@ issues:
- govet
text: "shadow: declaration of \"err\" shadows declaration"
#
# typecheck
#
- linters:
- typecheck
text: "undefined: min"
- linters:
- typecheck
text: "undefined: max"
#
# errcheck
#
- linters:
- errcheck
text: "Error return value of `.*` is not checked"
#
# gocritic
#
- linters:
- gocritic
text: "ifElseChain: rewrite if-else to switch statement"
@ -239,6 +257,65 @@ issues:
- gocritic
text: "commentFormatting: put a space between `//` and comment text"
# Will fix, trivial - just beware of merge conflicts
- linters:
- staticcheck
text: "x509.ParseCRL has been deprecated since Go 1.19: Use ParseRevocationList instead"
- perfsprint
text: "fmt.Sprintf can be replaced .*"
- linters:
- perfsprint
text: "fmt.Errorf can be replaced with errors.New"
#
# Will fix, easy but some neurons required
#
- linters:
- errorlint
text: "non-wrapping format verb for fmt.Errorf. Use `%w` to format errors"
- linters:
- errorlint
text: "type assertion on error will fail on wrapped errors. Use errors.As to check for specific errors"
- linters:
- errorlint
text: "type switch on error will fail on wrapped errors. Use errors.As to check for specific errors"
- linters:
- errorlint
text: "type assertion on error will fail on wrapped errors. Use errors.Is to check for specific errors"
- linters:
- errorlint
text: "comparing with .* will fail on wrapped errors. Use errors.Is to check for a specific error"
- linters:
- errorlint
text: "switch on an error will fail on wrapped errors. Use errors.Is to check for specific errors"
- linters:
- nosprintfhostport
text: "host:port in url should be constructed with net.JoinHostPort and not directly with fmt.Sprintf"
# https://github.com/timakin/bodyclose
- linters:
- bodyclose
text: "response body must be closed"
# named/naked returns are evil, with a single exception
# https://go.dev/wiki/CodeReviewComments#named-result-parameters
- linters:
- nonamedreturns
text: "named return .* with type .* found"
#
# Will fix, might be trickier
#
# https://github.com/pkg/errors/issues/245
- linters:
- depguard
text: "import 'github.com/pkg/errors' is not allowed .*"

26
Dockerfile
View file

@ -1,12 +1,13 @@
# vim: set ft=dockerfile:
ARG GOVERSION=1.21.4
FROM golang:1.21.7-alpine3.18 AS build
FROM golang:${GOVERSION}-alpine AS build
ARG BUILD_VERSION
WORKDIR /go/src/crowdsec
# We like to choose the release of re2 to use, and Alpine does not ship a static version anyway.
ENV RE2_VERSION=2023-03-01
ENV BUILD_VERSION=${BUILD_VERSION}
# wizard.sh requires GNU coreutils
RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils pkgconfig && \
@ -15,7 +16,7 @@ RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold core
cd re2-${RE2_VERSION} && \
make install && \
echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
go install github.com/mikefarah/yq/v4@v4.34.1
go install github.com/mikefarah/yq/v4@v4.40.4
COPY . .
@ -38,25 +39,26 @@ RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/co
mkdir -p /staging/var/lib/crowdsec && \
mkdir -p /var/lib/crowdsec/data
COPY --from=build /go/bin/yq /usr/local/bin/yq
COPY --from=build /go/bin/yq /usr/local/bin/crowdsec /usr/local/bin/cscli /usr/local/bin/
COPY --from=build /etc/crowdsec /staging/etc/crowdsec
COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml
ENTRYPOINT /bin/bash docker_start.sh
ENTRYPOINT /bin/bash /docker_start.sh
FROM slim as plugins
# Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
# The files are here for reference, as users will need to mount a new version to be actually able to use notifications
COPY --from=build /go/src/crowdsec/cmd/notification-email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml /staging/etc/crowdsec/notifications/sentinel.yaml
COPY --from=build \
/go/src/crowdsec/cmd/notification-email/email.yaml \
/go/src/crowdsec/cmd/notification-http/http.yaml \
/go/src/crowdsec/cmd/notification-slack/slack.yaml \
/go/src/crowdsec/cmd/notification-splunk/splunk.yaml \
/go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml \
/staging/etc/crowdsec/notifications/
COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
FROM slim as geoip

24
Dockerfile.debian
View file

@ -1,7 +1,7 @@
# vim: set ft=dockerfile:
ARG GOVERSION=1.21.4
FROM golang:1.21.7-bookworm AS build
FROM golang:${GOVERSION}-bookworm AS build
ARG BUILD_VERSION
WORKDIR /go/src/crowdsec
@ -10,6 +10,7 @@ ENV DEBCONF_NOWARNINGS="yes"
# We like to choose the release of re2 to use, the debian version is usually older.
ENV RE2_VERSION=2023-03-01
ENV BUILD_VERSION=${BUILD_VERSION}
# wizard.sh requires GNU coreutils
RUN apt-get update && \
@ -20,7 +21,7 @@ RUN apt-get update && \
make && \
make install && \
echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
go install github.com/mikefarah/yq/v4@v4.34.1
go install github.com/mikefarah/yq/v4@v4.40.4
COPY . .
@ -54,10 +55,8 @@ RUN apt-get update && \
mkdir -p /staging/var/lib/crowdsec && \
mkdir -p /var/lib/crowdsec/data
COPY --from=build /go/bin/yq /usr/local/bin/yq
COPY --from=build /go/bin/yq /usr/local/bin/crowdsec /usr/local/bin/cscli /usr/local/bin/
COPY --from=build /etc/crowdsec /staging/etc/crowdsec
COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml && \
@ -69,11 +68,14 @@ FROM slim as plugins
# Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
# The files are here for reference, as users will need to mount a new version to be actually able to use notifications
COPY --from=build /go/src/crowdsec/cmd/notification-email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml /staging/etc/crowdsec/notifications/sentinel.yaml
COPY --from=build \
/go/src/crowdsec/cmd/notification-email/email.yaml \
/go/src/crowdsec/cmd/notification-http/http.yaml \
/go/src/crowdsec/cmd/notification-slack/slack.yaml \
/go/src/crowdsec/cmd/notification-splunk/splunk.yaml \
/go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml \
/staging/etc/crowdsec/notifications/
COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
FROM slim as geoip

52
Makefile
View file

@ -128,11 +128,10 @@ endif
#--------------------------------------
.PHONY: build
build: pre-build goversion crowdsec cscli plugins
build: pre-build goversion crowdsec cscli plugins ## Build crowdsec, cscli and plugins
# Sanity checks and build information
.PHONY: pre-build
pre-build:
pre-build: ## Sanity checks and build information
$(info Building $(BUILD_VERSION) ($(BUILD_TAG)) $(BUILD_TYPE) for $(GOOS)/$(GOARCH))
ifneq (,$(RE2_FAIL))
@ -153,14 +152,14 @@ ifeq ($(call bool,$(TEST_COVERAGE)),1)
$(info Test coverage collection enabled)
endif
# intentional, empty line
$(info )
.PHONY: all
all: clean test build
all: clean test build ## Clean, test and build (requires localstack)
.PHONY: plugins
plugins:
plugins: ## Build notification plugins
@$(foreach plugin,$(PLUGINS), \
$(MAKE) -C $(PLUGINS_DIR_PREFIX)$(plugin) build $(MAKE_FLAGS); \
)
@ -184,7 +183,7 @@ clean-rpm:
@$(RM) -r rpm/SRPMS
.PHONY: clean
clean: clean-debian clean-rpm testclean
clean: clean-debian clean-rpm testclean ## Remove build artifacts
@$(MAKE) -C $(CROWDSEC_FOLDER) clean $(MAKE_FLAGS)
@$(MAKE) -C $(CSCLI_FOLDER) clean $(MAKE_FLAGS)
@$(RM) $(CROWDSEC_BIN) $(WIN_IGNORE_ERR)
@ -196,21 +195,16 @@ clean: clean-debian clean-rpm testclean
)
.PHONY: cscli
cscli: goversion
cscli: goversion ## Build cscli
@$(MAKE) -C $(CSCLI_FOLDER) build $(MAKE_FLAGS)
.PHONY: crowdsec
crowdsec: goversion
crowdsec: goversion ## Build crowdsec
@$(MAKE) -C $(CROWDSEC_FOLDER) build $(MAKE_FLAGS)
.PHONY: notification-email
notification-email: goversion
@$(MAKE) -C cmd/notification-email build $(MAKE_FLAGS)
.PHONY: testclean
testclean: bats-clean
testclean: bats-clean ## Remove test artifacts
@$(RM) pkg/apiserver/ent $(WIN_IGNORE_ERR)
@$(RM) pkg/cwhub/hubdir $(WIN_IGNORE_ERR)
@$(RM) pkg/cwhub/install $(WIN_IGNORE_ERR)
@ -218,41 +212,39 @@ testclean: bats-clean
# for the tests with localstack
export AWS_ENDPOINT_FORCE=http://localhost:4566
export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
export AWS_ACCESS_KEY_ID=test
export AWS_SECRET_ACCESS_KEY=test
testenv:
@echo 'NOTE: You need Docker, docker-compose and run "make localstack" in a separate shell ("make localstack-stop" to terminate it)'
# run the tests with localstack
.PHONY: test
test: testenv goversion
test: testenv goversion ## Run unit tests with localstack
$(GOTEST) $(LD_OPTS) ./...
# run the tests with localstack and coverage
.PHONY: go-acc
go-acc: testenv goversion
go-acc: testenv goversion ## Run unit tests with localstack + coverage
go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS)
# mock AWS services
.PHONY: localstack
localstack:
localstack: ## Run localstack containers (required for unit testing)
docker-compose -f test/localstack/docker-compose.yml up
.PHONY: localstack-stop
localstack-stop:
localstack-stop: ## Stop localstack containers
docker-compose -f test/localstack/docker-compose.yml down
# build vendor.tgz to be distributed with the release
.PHONY: vendor
vendor: vendor-remove
vendor: vendor-remove ## CI only - vendor dependencies and archive them for packaging
$(GO) mod vendor
tar czf vendor.tgz vendor
tar --create --auto-compress --file=$(RELDIR)-vendor.tar.xz vendor
# remove vendor directories and vendor.tgz
.PHONY: vendor-remove
vendor-remove:
vendor-remove: ## Remove vendor dependencies and archives
$(RM) vendor vendor.tgz *-vendor.tar.xz
.PHONY: package
@ -284,18 +276,15 @@ else
@if (Test-Path -Path $(RELDIR)) { echo "$(RELDIR) already exists, abort" ; exit 1 ; }
endif
# build a release tarball
.PHONY: release
release: check_release build package
release: check_release build package ## Build a release tarball
# build the windows installer
.PHONY: windows_installer
windows_installer: build
windows_installer: build ## Windows - build the installer
@.\make_installer.ps1 -version $(BUILD_VERSION)
# build the chocolatey package
.PHONY: chocolatey
chocolatey: windows_installer
chocolatey: windows_installer ## Windows - build the chocolatey package
@.\make_chocolatey.ps1 -version $(BUILD_VERSION)
# Include test/bats.mk only if it exists
@ -308,3 +297,4 @@ include test/bats.mk
endif
include mk/goversion.mk
include mk/help.mk

4
azure-pipelines.yml
View file

@ -25,9 +25,9 @@ stages:
custom: 'tool'
arguments: 'install --global SignClient --version 1.3.155'
- task: GoTool@0
displayName: "Install Go 1.20"
displayName: "Install Go"
inputs:
version: '1.21.4'
version: '1.21.7'
- pwsh: |
choco install -y make

367
cmd/crowdsec-cli/alerts.go
View file

@ -11,7 +11,6 @@ import (
"strconv"
"strings"
"text/template"
"time"
"github.com/fatih/color"
"github.com/go-openapi/strfmt"
@ -21,93 +20,55 @@ import (
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
func DecisionsFromAlert(alert *models.Alert) string {
ret := ""
var decMap = make(map[string]int)
decMap := make(map[string]int)
for _, decision := range alert.Decisions {
k := *decision.Type
if *decision.Simulated {
k = fmt.Sprintf("(simul)%s", k)
}
v := decMap[k]
decMap[k] = v + 1
}
for k, v := range decMap {
if len(ret) > 0 {
ret += " "
}
ret += fmt.Sprintf("%s:%d", k, v)
}
return ret
}
func DateFromAlert(alert *models.Alert) string {
ts, err := time.Parse(time.RFC3339, alert.CreatedAt)
if err != nil {
log.Infof("while parsing %s with %s : %s", alert.CreatedAt, time.RFC3339, err)
return alert.CreatedAt
}
return ts.Format(time.RFC822)
}
func SourceFromAlert(alert *models.Alert) string {
//more than one item, just number and scope
if len(alert.Decisions) > 1 {
return fmt.Sprintf("%d %ss (%s)", len(alert.Decisions), *alert.Decisions[0].Scope, *alert.Decisions[0].Origin)
}
//fallback on single decision information
if len(alert.Decisions) == 1 {
return fmt.Sprintf("%s:%s", *alert.Decisions[0].Scope, *alert.Decisions[0].Value)
}
//try to compose a human friendly version
if *alert.Source.Value != "" && *alert.Source.Scope != "" {
scope := ""
scope = fmt.Sprintf("%s:%s", *alert.Source.Scope, *alert.Source.Value)
extra := ""
if alert.Source.Cn != "" {
extra = alert.Source.Cn
}
if alert.Source.AsNumber != "" {
extra += fmt.Sprintf("/%s", alert.Source.AsNumber)
}
if alert.Source.AsName != "" {
extra += fmt.Sprintf("/%s", alert.Source.AsName)
}
if extra != "" {
scope += " (" + extra + ")"
}
return scope
}
return ""
}
func AlertsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
if csConfig.Cscli.Output == "raw" {
func (cli *cliAlerts) alertsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
switch cli.cfg().Cscli.Output {
case "raw":
csvwriter := csv.NewWriter(os.Stdout)
header := []string{"id", "scope", "value", "reason", "country", "as", "decisions", "created_at"}
if printMachine {
header = append(header, "machine")
}
err := csvwriter.Write(header)
if err != nil {
if err := csvwriter.Write(header); err != nil {
return err
}
for _, alertItem := range *alerts {
row := []string{
fmt.Sprintf("%d", alertItem.ID),
strconv.FormatInt(alertItem.ID, 10),
*alertItem.Source.Scope,
*alertItem.Source.Value,
*alertItem.Scenario,
@ -119,28 +80,32 @@ func AlertsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
if printMachine {
row = append(row, alertItem.MachineID)
}
err := csvwriter.Write(row)
if err != nil {
if err := csvwriter.Write(row); err != nil {
return err
}
}
csvwriter.Flush()
} else if csConfig.Cscli.Output == "json" {
case "json":
if *alerts == nil {
// avoid returning "null" in json
// could be cleaner if we used slice of alerts directly
fmt.Println("[]")
return nil
}
x, _ := json.MarshalIndent(alerts, "", " ")
fmt.Printf("%s", string(x))
} else if csConfig.Cscli.Output == "human" {
fmt.Print(string(x))
case "human":
if len(*alerts) == 0 {
fmt.Println("No active alerts")
return nil
}
alertsTable(color.Output, alerts, printMachine)
}
return nil
}
@ -162,71 +127,86 @@ var alertTemplate = `
`
func DisplayOneAlert(alert *models.Alert, withDetail bool) error {
if csConfig.Cscli.Output == "human" {
tmpl, err := template.New("alert").Parse(alertTemplate)
if err != nil {
return err
}
err = tmpl.Execute(os.Stdout, alert)
if err != nil {
return err
}
func (cli *cliAlerts) displayOneAlert(alert *models.Alert, withDetail bool) error {
tmpl, err := template.New("alert").Parse(alertTemplate)
if err != nil {
return err
}
alertDecisionsTable(color.Output, alert)
if err = tmpl.Execute(os.Stdout, alert); err != nil {
return err
}
if len(alert.Meta) > 0 {
fmt.Printf("\n - Context :\n")
sort.Slice(alert.Meta, func(i, j int) bool {
return alert.Meta[i].Key < alert.Meta[j].Key
})
table := newTable(color.Output)
table.SetRowLines(false)
table.SetHeaders("Key", "Value")
for _, meta := range alert.Meta {
var valSlice []string
if err := json.Unmarshal([]byte(meta.Value), &valSlice); err != nil {
return fmt.Errorf("unknown context value type '%s' : %s", meta.Value, err)
}
for _, value := range valSlice {
table.AddRow(
meta.Key,
value,
)
}
alertDecisionsTable(color.Output, alert)
if len(alert.Meta) > 0 {
fmt.Printf("\n - Context :\n")
sort.Slice(alert.Meta, func(i, j int) bool {
return alert.Meta[i].Key < alert.Meta[j].Key
})
table := newTable(color.Output)
table.SetRowLines(false)
table.SetHeaders("Key", "Value")
for _, meta := range alert.Meta {
var valSlice []string
if err := json.Unmarshal([]byte(meta.Value), &valSlice); err != nil {
return fmt.Errorf("unknown context value type '%s': %w", meta.Value, err)
}
for _, value := range valSlice {
table.AddRow(
meta.Key,
value,
)
}
table.Render()
}
if withDetail {
fmt.Printf("\n - Events :\n")
for _, event := range alert.Events {
alertEventTable(color.Output, event)
}
table.Render()
}
if withDetail {
fmt.Printf("\n - Events :\n")
for _, event := range alert.Events {
alertEventTable(color.Output, event)
}
}
return nil
}
func NewAlertsCmd() *cobra.Command {
var cmdAlerts = &cobra.Command{
type cliAlerts struct{
client *apiclient.ApiClient
cfg configGetter
}
func NewCLIAlerts(getconfig configGetter) *cliAlerts {
return &cliAlerts{
cfg: getconfig,
}
}
func (cli *cliAlerts) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "alerts [action]",
Short: "Manage alerts",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
Aliases: []string{"alert"},
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
var err error
if err := csConfig.LoadAPIClient(); err != nil {
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := cfg.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
}
apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
apiURL, err := url.Parse(cfg.API.Client.Credentials.URL)
if err != nil {
return fmt.Errorf("parsing api url %s: %w", apiURL, err)
}
Client, err = apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
Password: strfmt.Password(csConfig.API.Client.Credentials.Password),
cli.client, err = apiclient.NewClient(&apiclient.Config{
MachineID: cfg.API.Client.Credentials.Login,
Password: strfmt.Password(cfg.API.Client.Credentials.Password),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
@ -235,19 +215,20 @@ func NewAlertsCmd() *cobra.Command {
if err != nil {
return fmt.Errorf("new api client: %w", err)
}
return nil
},
}
cmdAlerts.AddCommand(NewAlertsListCmd())
cmdAlerts.AddCommand(NewAlertsInspectCmd())
cmdAlerts.AddCommand(NewAlertsFlushCmd())
cmdAlerts.AddCommand(NewAlertsDeleteCmd())
cmd.AddCommand(cli.NewListCmd())
cmd.AddCommand(cli.NewInspectCmd())
cmd.AddCommand(cli.NewFlushCmd())
cmd.AddCommand(cli.NewDeleteCmd())
return cmdAlerts
return cmd
}
func NewAlertsListCmd() *cobra.Command {
func (cli *cliAlerts) NewListCmd() *cobra.Command {
var alertListFilter = apiclient.AlertsListOpts{
ScopeEquals: new(string),
ValueEquals: new(string),
@ -260,10 +241,13 @@ func NewAlertsListCmd() *cobra.Command {
IncludeCAPI: new(bool),
OriginEquals: new(string),
}
var limit = new(int)
limit := new(int)
contained := new(bool)
var printMachine bool
var cmdAlertsList = &cobra.Command{
cmd := &cobra.Command{
Use: "list [filters]",
Short: "List alerts",
Example: `cscli alerts list
@ -272,9 +256,7 @@ cscli alerts list --range 1.2.3.0/24
cscli alerts list -s crowdsecurity/ssh-bf
cscli alerts list --type ban`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
var err error
RunE: func(cmd *cobra.Command, _ []string) error {
if err := manageCliDecisionAlerts(alertListFilter.IPEquals, alertListFilter.RangeEquals,
alertListFilter.ScopeEquals, alertListFilter.ValueEquals); err != nil {
printHelp(cmd)
@ -340,42 +322,45 @@ cscli alerts list --type ban`,
alertListFilter.Contains = new(bool)
}
alerts, _, err := Client.Alerts.List(context.Background(), alertListFilter)
alerts, _, err := cli.client.Alerts.List(context.Background(), alertListFilter)
if err != nil {
return fmt.Errorf("unable to list alerts: %v", err)
return fmt.Errorf("unable to list alerts: %w", err)
}
err = AlertsToTable(alerts, printMachine)
if err != nil {
return fmt.Errorf("unable to list alerts: %v", err)
if err = cli.alertsToTable(alerts, printMachine); err != nil {
return fmt.Errorf("unable to list alerts: %w", err)
}
return nil
},
}
cmdAlertsList.Flags().SortFlags = false
cmdAlertsList.Flags().BoolVarP(alertListFilter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
cmdAlertsList.Flags().StringVar(alertListFilter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
cmdAlertsList.Flags().StringVar(alertListFilter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
cmdAlertsList.Flags().StringVarP(alertListFilter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
cmdAlertsList.Flags().StringVarP(alertListFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
cmdAlertsList.Flags().StringVarP(alertListFilter.RangeEquals, "range", "r", "", "restrict to alerts from this range (shorthand for --scope range --value <RANGE/X>)")
cmdAlertsList.Flags().StringVar(alertListFilter.TypeEquals, "type", "", "restrict to alerts with given decision type (ie. ban, captcha)")
cmdAlertsList.Flags().StringVar(alertListFilter.ScopeEquals, "scope", "", "restrict to alerts of this scope (ie. ip,range)")
cmdAlertsList.Flags().StringVarP(alertListFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmdAlertsList.Flags().StringVar(alertListFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmdAlertsList.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
cmdAlertsList.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that sent alerts")
cmdAlertsList.Flags().IntVarP(limit, "limit", "l", 50, "limit size of alerts list table (0 to view all alerts)")
return cmdAlertsList
flags := cmd.Flags()
flags.SortFlags = false
flags.BoolVarP(alertListFilter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
flags.StringVar(alertListFilter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
flags.StringVar(alertListFilter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
flags.StringVarP(alertListFilter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
flags.StringVarP(alertListFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
flags.StringVarP(alertListFilter.RangeEquals, "range", "r", "", "restrict to alerts from this range (shorthand for --scope range --value <RANGE/X>)")
flags.StringVar(alertListFilter.TypeEquals, "type", "", "restrict to alerts with given decision type (ie. ban, captcha)")
flags.StringVar(alertListFilter.ScopeEquals, "scope", "", "restrict to alerts of this scope (ie. ip,range)")
flags.StringVarP(alertListFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
flags.StringVar(alertListFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
flags.BoolVar(contained, "contained", false, "query decisions contained by range")
flags.BoolVarP(&printMachine, "machine", "m", false, "print machines that sent alerts")
flags.IntVarP(limit, "limit", "l", 50, "limit size of alerts list table (0 to view all alerts)")
return cmd
}
func NewAlertsDeleteCmd() *cobra.Command {
var ActiveDecision *bool
var AlertDeleteAll bool
var delAlertByID string
contained := new(bool)
func (cli *cliAlerts) NewDeleteCmd() *cobra.Command {
var (
ActiveDecision *bool
AlertDeleteAll bool
delAlertByID string
)
var alertDeleteFilter = apiclient.AlertsDeleteOpts{
ScopeEquals: new(string),
ValueEquals: new(string),
@ -383,7 +368,10 @@ func NewAlertsDeleteCmd() *cobra.Command {
IPEquals: new(string),
RangeEquals: new(string),
}
var cmdAlertsDelete = &cobra.Command{
contained := new(bool)
cmd := &cobra.Command{
Use: "delete [filters] [--all]",
Short: `Delete alerts
/!\ This command can be use only on the same machine than the local API.`,
@ -393,7 +381,7 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`,
DisableAutoGenTag: true,
Aliases: []string{"remove"},
Args: cobra.ExactArgs(0),
PreRunE: func(cmd *cobra.Command, args []string) error {
PreRunE: func(cmd *cobra.Command, _ []string) error {
if AlertDeleteAll {
return nil
}
@ -406,11 +394,11 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`,
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(cmd *cobra.Command, _ []string) error {
var err error
if !AlertDeleteAll {
if err := manageCliDecisionAlerts(alertDeleteFilter.IPEquals, alertDeleteFilter.RangeEquals,
if err = manageCliDecisionAlerts(alertDeleteFilter.IPEquals, alertDeleteFilter.RangeEquals,
alertDeleteFilter.ScopeEquals, alertDeleteFilter.ValueEquals); err != nil {
printHelp(cmd)
return err
@ -446,14 +434,14 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`,
var alerts *models.DeleteAlertsResponse
if delAlertByID == "" {
alerts, _, err = Client.Alerts.Delete(context.Background(), alertDeleteFilter)
alerts, _, err = cli.client.Alerts.Delete(context.Background(), alertDeleteFilter)
if err != nil {
return fmt.Errorf("unable to delete alerts : %v", err)
return fmt.Errorf("unable to delete alerts: %w", err)
}
} else {
alerts, _, err = Client.Alerts.DeleteOne(context.Background(), delAlertByID)
alerts, _, err = cli.client.Alerts.DeleteOne(context.Background(), delAlertByID)
if err != nil {
return fmt.Errorf("unable to delete alert: %v", err)
return fmt.Errorf("unable to delete alert: %w", err)
}
}
log.Infof("%s alert(s) deleted", alerts.NbDeleted)
@ -461,26 +449,31 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`,
return nil
},
}
cmdAlertsDelete.Flags().SortFlags = false
cmdAlertsDelete.Flags().StringVar(alertDeleteFilter.ScopeEquals, "scope", "", "the scope (ie. ip,range)")
cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmdAlertsDelete.Flags().StringVar(&delAlertByID, "id", "", "alert ID")
cmdAlertsDelete.Flags().BoolVarP(&AlertDeleteAll, "all", "a", false, "delete all alerts")
cmdAlertsDelete.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
return cmdAlertsDelete
flags := cmd.Flags()
flags.SortFlags = false
flags.StringVar(alertDeleteFilter.ScopeEquals, "scope", "", "the scope (ie. ip,range)")
flags.StringVarP(alertDeleteFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
flags.StringVarP(alertDeleteFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
flags.StringVarP(alertDeleteFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
flags.StringVarP(alertDeleteFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
flags.StringVar(&delAlertByID, "id", "", "alert ID")
flags.BoolVarP(&AlertDeleteAll, "all", "a", false, "delete all alerts")
flags.BoolVar(contained, "contained", false, "query decisions contained by range")
return cmd
}
func NewAlertsInspectCmd() *cobra.Command {
func (cli *cliAlerts) NewInspectCmd() *cobra.Command {
var details bool
var cmdAlertsInspect = &cobra.Command{
cmd := &cobra.Command{
Use: `inspect "alert_id"`,
Short: `Show info about an alert`,
Example: `cscli alerts inspect 123`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
cfg := cli.cfg()
if len(args) == 0 {
printHelp(cmd)
return fmt.Errorf("missing alert_id")
@ -490,61 +483,65 @@ func NewAlertsInspectCmd() *cobra.Command {
if err != nil {
return fmt.Errorf("bad alert id %s", alertID)
}
alert, _, err := Client.Alerts.GetByID(context.Background(), id)
alert, _, err := cli.client.Alerts.GetByID(context.Background(), id)
if err != nil {
return fmt.Errorf("can't find alert with id %s: %s", alertID, err)
return fmt.Errorf("can't find alert with id %s: %w", alertID, err)
}
switch csConfig.Cscli.Output {
switch cfg.Cscli.Output {
case "human":
if err := DisplayOneAlert(alert, details); err != nil {
if err := cli.displayOneAlert(alert, details); err != nil {
continue
}
case "json":
data, err := json.MarshalIndent(alert, "", " ")
if err != nil {
return fmt.Errorf("unable to marshal alert with id %s: %s", alertID, err)
return fmt.Errorf("unable to marshal alert with id %s: %w", alertID, err)
}
fmt.Printf("%s\n", string(data))
case "raw":
data, err := yaml.Marshal(alert)
if err != nil {
return fmt.Errorf("unable to marshal alert with id %s: %s", alertID, err)
return fmt.Errorf("unable to marshal alert with id %s: %w", alertID, err)
}
fmt.Printf("%s\n", string(data))
fmt.Println(string(data))
}
}
return nil
},
}
cmdAlertsInspect.Flags().SortFlags = false
cmdAlertsInspect.Flags().BoolVarP(&details, "details", "d", false, "show alerts with events")
return cmdAlertsInspect
cmd.Flags().SortFlags = false
cmd.Flags().BoolVarP(&details, "details", "d", false, "show alerts with events")
return cmd
}
func NewAlertsFlushCmd() *cobra.Command {
var maxItems int
var maxAge string
var cmdAlertsFlush = &cobra.Command{
func (cli *cliAlerts) NewFlushCmd() *cobra.Command {
var (
maxItems int
maxAge string
)
cmd := &cobra.Command{
Use: `flush`,
Short: `Flush alerts
/!\ This command can be used only on the same machine than the local API`,
Example: `cscli alerts flush --max-items 1000 --max-age 7d`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
var err error
if err := require.LAPI(csConfig); err != nil {
RunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := require.LAPI(cfg); err != nil {
return err
}
dbClient, err = database.NewClient(csConfig.DbConfig)
db, err := database.NewClient(cfg.DbConfig)
if err != nil {
return fmt.Errorf("unable to create new database client: %s", err)
return fmt.Errorf("unable to create new database client: %w", err)
}
log.Info("Flushing alerts. !! This may take a long time !!")
err = dbClient.FlushAlerts(maxAge, maxItems)
err = db.FlushAlerts(maxAge, maxItems)
if err != nil {
return fmt.Errorf("unable to flush alerts: %s", err)
return fmt.Errorf("unable to flush alerts: %w", err)
}
log.Info("Alerts flushed")
@ -552,9 +549,9 @@ func NewAlertsFlushCmd() *cobra.Command {
},
}
cmdAlertsFlush.Flags().SortFlags = false
cmdAlertsFlush.Flags().IntVar(&maxItems, "max-items", 5000, "Maximum number of alert items to keep in the database")
cmdAlertsFlush.Flags().StringVar(&maxAge, "max-age", "7d", "Maximum age of alert items to keep in the database")
cmd.Flags().SortFlags = false
cmd.Flags().IntVar(&maxItems, "max-items", 5000, "Maximum number of alert items to keep in the database")
cmd.Flags().StringVar(&maxAge, "max-age", "7d", "Maximum age of alert items to keep in the database")
return cmdAlertsFlush
return cmd
}

505
cmd/crowdsec-cli/bouncers.go
View file

@ -4,7 +4,8 @@ import (
"encoding/csv"
"encoding/json"
"fmt"
"io"
"os"
"slices"
"strings"
"time"
@ -12,7 +13,6 @@ import (
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"slices"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1"
@ -20,246 +20,34 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/types"
)
func getBouncers(out io.Writer, dbClient *database.Client) error {
bouncers, err := dbClient.ListBouncers()
if err != nil {
return fmt.Errorf("unable to list bouncers: %s", err)
func askYesNo(message string, defaultAnswer bool) (bool, error) {
var answer bool
prompt := &survey.Confirm{
Message: message,
Default: defaultAnswer,
}
switch csConfig.Cscli.Output {
case "human":
getBouncersTable(out, bouncers)
case "json":
enc := json.NewEncoder(out)
enc.SetIndent("", " ")
if err := enc.Encode(bouncers); err != nil {
return fmt.Errorf("failed to unmarshal: %w", err)
}
return nil
case "raw":
csvwriter := csv.NewWriter(out)
err := csvwriter.Write([]string{"name", "ip", "revoked", "last_pull", "type", "version", "auth_type"})
if err != nil {
return fmt.Errorf("failed to write raw header: %w", err)
}
for _, b := range bouncers {
var revoked string
if !b.Revoked {
revoked = "validated"
} else {
revoked = "pending"
}
err := csvwriter.Write([]string{b.Name, b.IPAddress, revoked, b.LastPull.Format(time.RFC3339), b.Type, b.Version, b.AuthType})
if err != nil {
return fmt.Errorf("failed to write raw: %w", err)
}
}
csvwriter.Flush()
if err := survey.AskOne(prompt, &answer); err != nil {
return defaultAnswer, err
}
return nil
return answer, nil
}
func NewBouncersListCmd() *cobra.Command {
cmdBouncersList := &cobra.Command{
Use: "list",
Short: "list all bouncers within the database",
Example: `cscli bouncers list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, arg []string) error {
err := getBouncers(color.Output, dbClient)
if err != nil {
return fmt.Errorf("unable to list bouncers: %s", err)
}
return nil
},
}
return cmdBouncersList
type cliBouncers struct {
db *database.Client
cfg configGetter
}
func runBouncersAdd(cmd *cobra.Command, args []string) error {
keyLength := 32
flags := cmd.Flags()
key, err := flags.GetString("key")
if err != nil {
return err
func NewCLIBouncers(cfg configGetter) *cliBouncers {
return &cliBouncers{
cfg: cfg,
}
keyName := args[0]
var apiKey string
if keyName == "" {
return fmt.Errorf("please provide a name for the api key")
}
apiKey = key
if key == "" {
apiKey, err = middlewares.GenerateAPIKey(keyLength)
}
if err != nil {
return fmt.Errorf("unable to generate api key: %s", err)
}
_, err = dbClient.CreateBouncer(keyName, "", middlewares.HashSHA512(apiKey), types.ApiKeyAuthType)
if err != nil {
return fmt.Errorf("unable to create bouncer: %s", err)
}
switch csConfig.Cscli.Output {
case "human":
fmt.Printf("API key for '%s':\n\n", keyName)
fmt.Printf(" %s\n\n", apiKey)
fmt.Print("Please keep this key since you will not be able to retrieve it!\n")
case "raw":
fmt.Printf("%s", apiKey)
case "json":
j, err := json.Marshal(apiKey)
if err != nil {
return fmt.Errorf("unable to marshal api key")
}
fmt.Printf("%s", string(j))
}
return nil
}
func NewBouncersAddCmd() *cobra.Command {
cmdBouncersAdd := &cobra.Command{
Use: "add MyBouncerName",
Short: "add a single bouncer to the database",
Example: `cscli bouncers add MyBouncerName
cscli bouncers add MyBouncerName --key <random-key>`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: runBouncersAdd,
}
flags := cmdBouncersAdd.Flags()
flags.StringP("length", "l", "", "length of the api key")
flags.MarkDeprecated("length", "use --key instead")
flags.StringP("key", "k", "", "api key for the bouncer")
return cmdBouncersAdd
}
func runBouncersDelete(cmd *cobra.Command, args []string) error {
for _, bouncerID := range args {
err := dbClient.DeleteBouncer(bouncerID)
if err != nil {
return fmt.Errorf("unable to delete bouncer '%s': %s", bouncerID, err)
}
log.Infof("bouncer '%s' deleted successfully", bouncerID)
}
return nil
}
func NewBouncersDeleteCmd() *cobra.Command {
cmdBouncersDelete := &cobra.Command{
Use: "delete MyBouncerName",
Short: "delete bouncer(s) from the database",
Args: cobra.MinimumNArgs(1),
Aliases: []string{"remove"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
var err error
dbClient, err = getDBClient()
if err != nil {
cobra.CompError("unable to create new database client: " + err.Error())
return nil, cobra.ShellCompDirectiveNoFileComp
}
bouncers, err := dbClient.ListBouncers()
if err != nil {
cobra.CompError("unable to list bouncers " + err.Error())
}
ret := make([]string, 0)
for _, bouncer := range bouncers {
if strings.Contains(bouncer.Name, toComplete) && !slices.Contains(args, bouncer.Name) {
ret = append(ret, bouncer.Name)
}
}
return ret, cobra.ShellCompDirectiveNoFileComp
},
RunE: runBouncersDelete,
}
return cmdBouncersDelete
}
func NewBouncersPruneCmd() *cobra.Command {
var parsedDuration time.Duration
cmdBouncersPrune := &cobra.Command{
Use: "prune",
Short: "prune multiple bouncers from the database",
Args: cobra.NoArgs,
DisableAutoGenTag: true,
Example: `cscli bouncers prune -d 60m
cscli bouncers prune -d 60m --force`,
PreRunE: func(cmd *cobra.Command, args []string) error {
dur, _ := cmd.Flags().GetString("duration")
var err error
parsedDuration, err = time.ParseDuration(fmt.Sprintf("-%s", dur))
if err != nil {
return fmt.Errorf("unable to parse duration '%s': %s", dur, err)
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
force, _ := cmd.Flags().GetBool("force")
if parsedDuration >= 0-2*time.Minute {
var answer bool
prompt := &survey.Confirm{
Message: "The duration you provided is less than or equal 2 minutes this may remove active bouncers continue ?",
Default: false,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask about prune check: %s", err)
}
if !answer {
fmt.Println("user aborted prune no changes were made")
return nil
}
}
bouncers, err := dbClient.QueryBouncersLastPulltimeLT(time.Now().UTC().Add(parsedDuration))
if err != nil {
return fmt.Errorf("unable to query bouncers: %s", err)
}
if len(bouncers) == 0 {
fmt.Println("no bouncers to prune")
return nil
}
getBouncersTable(color.Output, bouncers)
if !force {
var answer bool
prompt := &survey.Confirm{
Message: "You are about to PERMANENTLY remove the above bouncers from the database these will NOT be recoverable, continue ?",
Default: false,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask about prune check: %s", err)
}
if !answer {
fmt.Println("user aborted prune no changes were made")
return nil
}
}
nbDeleted, err := dbClient.BulkDeleteBouncers(bouncers)
if err != nil {
return fmt.Errorf("unable to prune bouncers: %s", err)
}
fmt.Printf("successfully delete %d bouncers\n", nbDeleted)
return nil
},
}
cmdBouncersPrune.Flags().StringP("duration", "d", "60m", "duration of time since last pull")
cmdBouncersPrune.Flags().Bool("force", false, "force prune without asking for confirmation")
return cmdBouncersPrune
}
func NewBouncersCmd() *cobra.Command {
var cmdBouncers = &cobra.Command{
func (cli *cliBouncers) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "bouncers [action]",
Short: "Manage bouncers [requires local API]",
Long: `To list/add/delete/prune bouncers.
@ -268,24 +56,261 @@ Note: This command requires database direct access, so is intended to be run on
Args: cobra.MinimumNArgs(1),
Aliases: []string{"bouncer"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
var err error
if err = require.LAPI(csConfig); err != nil {
if err = require.LAPI(cli.cfg()); err != nil {
return err
}
dbClient, err = database.NewClient(csConfig.DbConfig)
cli.db, err = database.NewClient(cli.cfg().DbConfig)
if err != nil {
return fmt.Errorf("unable to create new database client: %s", err)
return fmt.Errorf("can't connect to the database: %s", err)
}
return nil
},
}
cmdBouncers.AddCommand(NewBouncersListCmd())
cmdBouncers.AddCommand(NewBouncersAddCmd())
cmdBouncers.AddCommand(NewBouncersDeleteCmd())
cmdBouncers.AddCommand(NewBouncersPruneCmd())
cmd.AddCommand(cli.newListCmd())
cmd.AddCommand(cli.newAddCmd())
cmd.AddCommand(cli.newDeleteCmd())
cmd.AddCommand(cli.newPruneCmd())
return cmdBouncers
return cmd
}
func (cli *cliBouncers) list() error {
out := color.Output
bouncers, err := cli.db.ListBouncers()
if err != nil {
return fmt.Errorf("unable to list bouncers: %s", err)
}
switch cli.cfg().Cscli.Output {
case "human":
getBouncersTable(out, bouncers)
case "json":
enc := json.NewEncoder(out)
enc.SetIndent("", " ")
if err := enc.Encode(bouncers); err != nil {
return fmt.Errorf("failed to marshal: %w", err)
}
return nil
case "raw":
csvwriter := csv.NewWriter(out)
if err := csvwriter.Write([]string{"name", "ip", "revoked", "last_pull", "type", "version", "auth_type"}); err != nil {
return fmt.Errorf("failed to write raw header: %w", err)
}
for _, b := range bouncers {
valid := "validated"
if b.Revoked {
valid = "pending"
}
if err := csvwriter.Write([]string{b.Name, b.IPAddress, valid, b.LastPull.Format(time.RFC3339), b.Type, b.Version, b.AuthType}); err != nil {
return fmt.Errorf("failed to write raw: %w", err)
}
}
csvwriter.Flush()
}
return nil
}
func (cli *cliBouncers) newListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "list",
Short: "list all bouncers within the database",
Example: `cscli bouncers list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.list()
},
}
return cmd
}
func (cli *cliBouncers) add(bouncerName string, key string) error {
var err error
keyLength := 32
if key == "" {
key, err = middlewares.GenerateAPIKey(keyLength)
if err != nil {
return fmt.Errorf("unable to generate api key: %s", err)
}
}
_, err = cli.db.CreateBouncer(bouncerName, "", middlewares.HashSHA512(key), types.ApiKeyAuthType)
if err != nil {
return fmt.Errorf("unable to create bouncer: %s", err)
}
switch cli.cfg().Cscli.Output {
case "human":
fmt.Printf("API key for '%s':\n\n", bouncerName)
fmt.Printf(" %s\n\n", key)
fmt.Print("Please keep this key since you will not be able to retrieve it!\n")
case "raw":
fmt.Print(key)
case "json":
j, err := json.Marshal(key)
if err != nil {
return fmt.Errorf("unable to marshal api key")
}
fmt.Print(string(j))
}
return nil
}
func (cli *cliBouncers) newAddCmd() *cobra.Command {
var key string
cmd := &cobra.Command{
Use: "add MyBouncerName",
Short: "add a single bouncer to the database",
Example: `cscli bouncers add MyBouncerName
cscli bouncers add MyBouncerName --key <random-key>`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
return cli.add(args[0], key)
},
}
flags := cmd.Flags()
flags.StringP("length", "l", "", "length of the api key")
flags.MarkDeprecated("length", "use --key instead")
flags.StringVarP(&key, "key", "k", "", "api key for the bouncer")
return cmd
}
func (cli *cliBouncers) deleteValid(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
bouncers, err := cli.db.ListBouncers()
if err != nil {
cobra.CompError("unable to list bouncers " + err.Error())
}
ret := []string{}
for _, bouncer := range bouncers {
if strings.Contains(bouncer.Name, toComplete) && !slices.Contains(args, bouncer.Name) {
ret = append(ret, bouncer.Name)
}
}
return ret, cobra.ShellCompDirectiveNoFileComp
}
func (cli *cliBouncers) delete(bouncers []string) error {
for _, bouncerID := range bouncers {
err := cli.db.DeleteBouncer(bouncerID)
if err != nil {
return fmt.Errorf("unable to delete bouncer '%s': %s", bouncerID, err)
}
log.Infof("bouncer '%s' deleted successfully", bouncerID)
}
return nil
}
func (cli *cliBouncers) newDeleteCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "delete MyBouncerName",
Short: "delete bouncer(s) from the database",
Args: cobra.MinimumNArgs(1),
Aliases: []string{"remove"},
DisableAutoGenTag: true,
ValidArgsFunction: cli.deleteValid,
RunE: func(_ *cobra.Command, args []string) error {
return cli.delete(args)
},
}
return cmd
}
func (cli *cliBouncers) prune(duration time.Duration, force bool) error {
if duration < 2*time.Minute {
if yes, err := askYesNo(
"The duration you provided is less than 2 minutes. " +
"This may remove active bouncers. Continue?", false); err != nil {
return err
} else if !yes {
fmt.Println("User aborted prune. No changes were made.")
return nil
}
}
bouncers, err := cli.db.QueryBouncersLastPulltimeLT(time.Now().UTC().Add(duration))
if err != nil {
return fmt.Errorf("unable to query bouncers: %w", err)
}
if len(bouncers) == 0 {
fmt.Println("No bouncers to prune.")
return nil
}
getBouncersTable(color.Output, bouncers)
if !force {
if yes, err := askYesNo(
"You are about to PERMANENTLY remove the above bouncers from the database. " +
"These will NOT be recoverable. Continue?", false); err != nil {
return err
} else if !yes {
fmt.Println("User aborted prune. No changes were made.")
return nil
}
}
deleted, err := cli.db.BulkDeleteBouncers(bouncers)
if err != nil {
return fmt.Errorf("unable to prune bouncers: %s", err)
}
fmt.Fprintf(os.Stderr, "Successfully deleted %d bouncers\n", deleted)
return nil
}
func (cli *cliBouncers) newPruneCmd() *cobra.Command {
var (
duration time.Duration
force bool
)
const defaultDuration = 60 * time.Minute
cmd := &cobra.Command{
Use: "prune",
Short: "prune multiple bouncers from the database",
Args: cobra.NoArgs,
DisableAutoGenTag: true,
Example: `cscli bouncers prune -d 45m
cscli bouncers prune -d 45m --force`,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.prune(duration, force)
},
}
flags := cmd.Flags()
flags.DurationVarP(&duration, "duration", "d", defaultDuration, "duration of time since last pull")
flags.BoolVar(&force, "force", false, "force prune without asking for confirmation")
return cmd
}

75
cmd/crowdsec-cli/capi.go
View file

@ -13,26 +13,32 @@ import (
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
const CAPIBaseURL string = "https://api.crowdsec.net/"
const CAPIURLPrefix = "v3"
const (
CAPIBaseURL = "https://api.crowdsec.net/"
CAPIURLPrefix = "v3"
)
func NewCapiCmd() *cobra.Command {
var cmdCapi = &cobra.Command{
type cliCapi struct{}
func NewCLICapi() *cliCapi {
return &cliCapi{}
}
func (cli cliCapi) NewCommand() *cobra.Command {
var cmd = &cobra.Command{
Use: "capi [action]",
Short: "Manage interaction with Central API (CAPI)",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
if err := require.LAPI(csConfig); err != nil {
return err
}
@ -45,22 +51,24 @@ func NewCapiCmd() *cobra.Command {
},
}
cmdCapi.AddCommand(NewCapiRegisterCmd())
cmdCapi.AddCommand(NewCapiStatusCmd())
cmd.AddCommand(cli.NewRegisterCmd())
cmd.AddCommand(cli.NewStatusCmd())
return cmdCapi
return cmd
}
func NewCapiRegisterCmd() *cobra.Command {
var capiUserPrefix string
var outputFile string
func (cli cliCapi) NewRegisterCmd() *cobra.Command {
var (
capiUserPrefix string
outputFile string
)
var cmdCapiRegister = &cobra.Command{
var cmd = &cobra.Command{
Use: "register",
Short: "Register to Central API (CAPI)",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
var err error
capiUser, err := generateID(capiUserPrefix)
if err != nil {
@ -98,21 +106,18 @@ func NewCapiRegisterCmd() *cobra.Command {
Password: password.String(),
URL: types.CAPIBaseURL,
}
if fflag.PapiClient.IsEnabled() {
apiCfg.PapiURL = types.PAPIBaseURL
}
apiConfigDump, err := yaml.Marshal(apiCfg)
if err != nil {
return fmt.Errorf("unable to marshal api credentials: %w", err)
}
if dumpFile != "" {
err = os.WriteFile(dumpFile, apiConfigDump, 0600)
err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
if err != nil {
return fmt.Errorf("write api credentials in '%s' failed: %w", dumpFile, err)
}
log.Printf("Central API credentials written to '%s'", dumpFile)
} else {
fmt.Printf("%s\n", string(apiConfigDump))
fmt.Println(string(apiConfigDump))
}
log.Warning(ReloadMessage())
@ -120,28 +125,26 @@ func NewCapiRegisterCmd() *cobra.Command {
return nil
},
}
cmdCapiRegister.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
cmdCapiRegister.Flags().StringVar(&capiUserPrefix, "schmilblick", "", "set a schmilblick (use in tests only)")
if err := cmdCapiRegister.Flags().MarkHidden("schmilblick"); err != nil {
cmd.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
cmd.Flags().StringVar(&capiUserPrefix, "schmilblick", "", "set a schmilblick (use in tests only)")
if err := cmd.Flags().MarkHidden("schmilblick"); err != nil {
log.Fatalf("failed to hide flag: %s", err)
}
return cmdCapiRegister
return cmd
}
func NewCapiStatusCmd() *cobra.Command {
var cmdCapiStatus = &cobra.Command{
func (cli cliCapi) NewStatusCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "status",
Short: "Check status with the Central API (CAPI)",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if csConfig.API.Server.OnlineClient == nil {
return fmt.Errorf("please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
}
if csConfig.API.Server.OnlineClient.Credentials == nil {
return fmt.Errorf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
RunE: func(_ *cobra.Command, _ []string) error {
if err := require.CAPIRegistered(csConfig); err != nil {
return err
}
password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
@ -151,7 +154,7 @@ func NewCapiStatusCmd() *cobra.Command {
return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err)
}
hub, err := require.Hub(csConfig, nil)
hub, err := require.Hub(csConfig, nil, nil)
if err != nil {
return err
}
@ -189,5 +192,5 @@ func NewCapiStatusCmd() *cobra.Command {
},
}
return cmdCapiStatus
return cmd
}

4
cmd/crowdsec-cli/completion.go
View file

@ -7,8 +7,7 @@ import (
)
func NewCompletionCmd() *cobra.Command {
var completionCmd = &cobra.Command{
completionCmd := &cobra.Command{
Use: "completion [bash|zsh|powershell|fish]",
Short: "Generate completion script",
Long: `To load completions:
@ -82,5 +81,6 @@ func NewCompletionCmd() *cobra.Command {
}
},
}
return completionCmd
}

25
cmd/crowdsec-cli/config_backup.go
View file

@ -9,15 +9,12 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func backupHub(dirPath string) error {
var itemDirectory string
var upstreamParsers []string
hub, err := require.Hub(csConfig, nil)
hub, err := require.Hub(csConfig, nil, nil)
if err != nil {
return err
}
@ -26,16 +23,20 @@ func backupHub(dirPath string) error {
clog := log.WithFields(log.Fields{
"type": itemType,
})
itemMap := hub.GetItemMap(itemType)
if itemMap == nil {
clog.Infof("No %s to backup.", itemType)
continue
}
itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itemType)
if err = os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
}
upstreamParsers = []string{}
upstreamParsers := []string{}
for k, v := range itemMap {
clog = clog.WithFields(log.Fields{
"file": v.Name,
@ -54,28 +55,36 @@ func backupHub(dirPath string) error {
return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
}
}
clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.State.IsLocal(), v.State.UpToDate)
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
if err = CopyFile(v.State.LocalPath, tfile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.State.LocalPath, tfile, err)
}
clog.Infof("local/tainted saved %s to %s", v.State.LocalPath, tfile)
continue
}
clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.State.UpToDate)
clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.State.UpToDate)
upstreamParsers = append(upstreamParsers, v.Name)
}
//write the upstream items
upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
if err != nil {
return fmt.Errorf("failed marshaling upstream parsers : %s", err)
}
err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0o644)
if err != nil {
return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
}
clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
}

3
cmd/crowdsec-cli/config_feature_flags.go
View file

@ -44,6 +44,7 @@ func runConfigFeatureFlags(cmd *cobra.Command, args []string) error {
if feat.State == fflag.RetiredState {
fmt.Printf("\n %s %s", magenta("RETIRED"), feat.DeprecationMsg)
}
fmt.Println()
}
@ -58,10 +59,12 @@ func runConfigFeatureFlags(cmd *cobra.Command, args []string) error {
retired = append(retired, feat)
continue
}
if feat.IsEnabled() {
enabled = append(enabled, feat)
continue
}
disabled = append(disabled, feat)
}

27
cmd/crowdsec-cli/config_restore.go
View file

@ -22,7 +22,7 @@ type OldAPICfg struct {
}
func restoreHub(dirPath string) error {
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), nil)
if err != nil {
return err
}
@ -35,21 +35,26 @@ func restoreHub(dirPath string) error {
}
/*restore the upstream items*/
upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
file, err := os.ReadFile(upstreamListFN)
if err != nil {
return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
}
var upstreamList []string
err = json.Unmarshal(file, &upstreamList)
if err != nil {
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
}
for _, toinstall := range upstreamList {
item := hub.GetItem(itype, toinstall)
if item == nil {
log.Errorf("Item %s/%s not found in hub", itype, toinstall)
continue
}
err := item.Install(false, false)
if err != nil {
log.Errorf("Error while installing %s : %s", toinstall, err)
@ -61,23 +66,28 @@ func restoreHub(dirPath string) error {
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
}
for _, file := range files {
//this was the upstream data
if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
continue
}
if itype == cwhub.PARSERS || itype == cwhub.POSTOVERFLOWS {
//we expect a stage here
if !file.IsDir() {
continue
}
stage := file.Name()
stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
}
/*find items*/
// find items
ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
@ -86,10 +96,12 @@ func restoreHub(dirPath string) error {
for _, tfile := range ifiles {
log.Infof("Going to restore local/tainted [%s]", tfile.Name())
sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
} else {
@ -101,9 +113,9 @@ func restoreHub(dirPath string) error {
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
}
}
return nil
}
@ -134,7 +146,12 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
// Now we have config.yaml, we should regenerate config struct to have rights paths etc
ConfigFilePath = fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)
initConfig()
log.Debug("Reloading configuration")
csConfig, _, err = loadConfigFor("config")
if err != nil {
return fmt.Errorf("failed to reload configuration: %s", err)
}
backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath)
if _, err = os.Stat(backupCAPICreds); err == nil {
@ -215,7 +232,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
}
}
// if there is files in the acquis backup dir, restore them
// if there are files in the acquis backup dir, restore them
acquisBackupDir := filepath.Join(dirPath, "acquis", "*.yaml")
if acquisFiles, err := filepath.Glob(acquisBackupDir); err == nil {
for _, acquisFile := range acquisFiles {

4
cmd/crowdsec-cli/config_show.go
View file

@ -24,6 +24,7 @@ func showConfigKey(key string) error {
opts := []expr.Option{}
opts = append(opts, exprhelpers.GetExprOptions(map[string]interface{}{})...)
opts = append(opts, expr.Env(Env{}))
program, err := expr.Compile(key, opts...)
if err != nil {
return err
@ -52,6 +53,7 @@ func showConfigKey(key string) error {
fmt.Printf("%s\n", string(data))
}
return nil
}
@ -211,6 +213,7 @@ func runConfigShow(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
err = tmp.Execute(os.Stdout, csConfig)
if err != nil {
return err
@ -230,6 +233,7 @@ func runConfigShow(cmd *cobra.Command, args []string) error {
fmt.Printf("%s\n", string(data))
}
return nil
}

288
cmd/crowdsec-cli/console.go
View file

@ -7,6 +7,7 @@ import (
"fmt"
"net/url"
"os"
"strings"
"github.com/fatih/color"
"github.com/go-openapi/strfmt"
@ -17,40 +18,60 @@ import (
"github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
func NewConsoleCmd() *cobra.Command {
var cmdConsole = &cobra.Command{
type cliConsole struct {
cfg configGetter
}
func NewCLIConsole(cfg configGetter) *cliConsole {
return &cliConsole{
cfg: cfg,
}
}
func (cli *cliConsole) NewCommand() *cobra.Command {
var cmd = &cobra.Command{
Use: "console [action]",
Short: "Manage interaction with Crowdsec console (https://app.crowdsec.net)",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.LAPI(csConfig); err != nil {
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := require.LAPI(cfg); err != nil {
return err
}
if err := require.CAPI(csConfig); err != nil {
if err := require.CAPI(cfg); err != nil {
return err
}
if err := require.CAPIRegistered(csConfig); err != nil {
if err := require.CAPIRegistered(cfg); err != nil {
return err
}
return nil
},
}
cmd.AddCommand(cli.newEnrollCmd())
cmd.AddCommand(cli.newEnableCmd())
cmd.AddCommand(cli.newDisableCmd())
cmd.AddCommand(cli.newStatusCmd())
return cmd
}
func (cli *cliConsole) newEnrollCmd() *cobra.Command {
name := ""
overwrite := false
tags := []string{}
opts := []string{}
cmdEnroll := &cobra.Command{
cmd := &cobra.Command{
Use: "enroll [enroll-key]",
Short: "Enroll this instance to https://app.crowdsec.net [requires local API]",
Long: `
@ -58,68 +79,115 @@ Enroll this instance to https://app.crowdsec.net
You can get your enrollment key by creating an account on https://app.crowdsec.net.
After running this command your will need to validate the enrollment in the webapp.`,
Example: `cscli console enroll YOUR-ENROLL-KEY
Example: fmt.Sprintf(`cscli console enroll YOUR-ENROLL-KEY
cscli console enroll --name [instance_name] YOUR-ENROLL-KEY
cscli console enroll --name [instance_name] --tags [tag_1] --tags [tag_2] YOUR-ENROLL-KEY
`,
cscli console enroll --enable context,manual YOUR-ENROLL-KEY
valid options are : %s,all (see 'cscli console status' for details)`, strings.Join(csconfig.CONSOLE_CONFIGS, ",")),
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
apiURL, err := url.Parse(csConfig.API.Server.OnlineClient.Credentials.URL)
RunE: func(_ *cobra.Command, args []string) error {
cfg := cli.cfg()
password := strfmt.Password(cfg.API.Server.OnlineClient.Credentials.Password)
apiURL, err := url.Parse(cfg.API.Server.OnlineClient.Credentials.URL)
if err != nil {
return fmt.Errorf("could not parse CAPI URL: %s", err)
return fmt.Errorf("could not parse CAPI URL: %w", err)
}
hub, err := require.Hub(csConfig, nil)
hub, err := require.Hub(cfg, nil, nil)
if err != nil {
return err
}
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("failed to get installed scenarios: %s", err)
return fmt.Errorf("failed to get installed scenarios: %w", err)
}
if len(scenarios) == 0 {
scenarios = make([]string, 0)
}
enableOpts := []string{csconfig.SEND_MANUAL_SCENARIOS, csconfig.SEND_TAINTED_SCENARIOS}
if len(opts) != 0 {
for _, opt := range opts {
valid := false
if opt == "all" {
enableOpts = csconfig.CONSOLE_CONFIGS
break
}
for _, availableOpt := range csconfig.CONSOLE_CONFIGS {
if opt == availableOpt {
valid = true
enable := true
for _, enabledOpt := range enableOpts {
if opt == enabledOpt {
enable = false
continue
}
}
if enable {
enableOpts = append(enableOpts, opt)
}
break
}
}
if !valid {
return fmt.Errorf("option %s doesn't exist", opt)
}
}
}
c, _ := apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Server.OnlineClient.Credentials.Login,
MachineID: cli.cfg().API.Server.OnlineClient.Credentials.Login,
Password: password,
Scenarios: scenarios,
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v3",
})
resp, err := c.Auth.EnrollWatcher(context.Background(), args[0], name, tags, overwrite)
if err != nil {
return fmt.Errorf("could not enroll instance: %s", err)
return fmt.Errorf("could not enroll instance: %w", err)
}
if resp.Response.StatusCode == 200 && !overwrite {
log.Warning("Instance already enrolled. You can use '--overwrite' to force enroll")
return nil
}
if err := SetConsoleOpts([]string{csconfig.SEND_MANUAL_SCENARIOS, csconfig.SEND_TAINTED_SCENARIOS}, true); err != nil {
if err := cli.setConsoleOpts(enableOpts, true); err != nil {
return err
}
log.Info("Enabled tainted&manual alerts sharing, see 'cscli console status'.")
for _, opt := range enableOpts {
log.Infof("Enabled %s : %s", opt, csconfig.CONSOLE_CONFIGS_HELP[opt])
}
log.Info("Watcher successfully enrolled. Visit https://app.crowdsec.net to accept it.")
log.Info("Please restart crowdsec after accepting the enrollment.")
return nil
},
}
cmdEnroll.Flags().StringVarP(&name, "name", "n", "", "Name to display in the console")
cmdEnroll.Flags().BoolVarP(&overwrite, "overwrite", "", false, "Force enroll the instance")
cmdEnroll.Flags().StringSliceVarP(&tags, "tags", "t", tags, "Tags to display in the console")
cmdConsole.AddCommand(cmdEnroll)
var enableAll, disableAll bool
flags := cmd.Flags()
flags.StringVarP(&name, "name", "n", "", "Name to display in the console")
flags.BoolVarP(&overwrite, "overwrite", "", false, "Force enroll the instance")
flags.StringSliceVarP(&tags, "tags", "t", tags, "Tags to display in the console")
flags.StringSliceVarP(&opts, "enable", "e", opts, "Enable console options")
cmdEnable := &cobra.Command{
return cmd
}
func (cli *cliConsole) newEnableCmd() *cobra.Command {
var enableAll bool
cmd := &cobra.Command{
Use: "enable [option]",
Short: "Enable a console option",
Example: "sudo cscli console enable tainted",
@ -127,9 +195,9 @@ After running this command your will need to validate the enrollment in the weba
Enable given information push to the central API. Allows to empower the console`,
ValidArgs: csconfig.CONSOLE_CONFIGS,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, args []string) error {
if enableAll {
if err := SetConsoleOpts(csconfig.CONSOLE_CONFIGS, true); err != nil {
if err := cli.setConsoleOpts(csconfig.CONSOLE_CONFIGS, true); err != nil {
return err
}
log.Infof("All features have been enabled successfully")
@ -137,19 +205,26 @@ Enable given information push to the central API. Allows to empower the console`
if len(args) == 0 {
return fmt.Errorf("you must specify at least one feature to enable")
}
if err := SetConsoleOpts(args, true); err != nil {
if err := cli.setConsoleOpts(args, true); err != nil {
return err
}
log.Infof("%v have been enabled", args)
}
log.Infof(ReloadMessage())
return nil
},
}
cmdEnable.Flags().BoolVarP(&enableAll, "all", "a", false, "Enable all console options")
cmdConsole.AddCommand(cmdEnable)
cmd.Flags().BoolVarP(&enableAll, "all", "a", false, "Enable all console options")
cmdDisable := &cobra.Command{
return cmd
}
func (cli *cliConsole) newDisableCmd() *cobra.Command {
var disableAll bool
cmd := &cobra.Command{
Use: "disable [option]",
Short: "Disable a console option",
Example: "sudo cscli console disable tainted",
@ -157,47 +232,52 @@ Enable given information push to the central API. Allows to empower the console`
Disable given information push to the central API.`,
ValidArgs: csconfig.CONSOLE_CONFIGS,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, args []string) error {
if disableAll {
if err := SetConsoleOpts(csconfig.CONSOLE_CONFIGS, false); err != nil {
if err := cli.setConsoleOpts(csconfig.CONSOLE_CONFIGS, false); err != nil {
return err
}
log.Infof("All features have been disabled")
} else {
if err := SetConsoleOpts(args, false); err != nil {
if err := cli.setConsoleOpts(args, false); err != nil {
return err
}
log.Infof("%v have been disabled", args)
}
log.Infof(ReloadMessage())
return nil
},
}
cmdDisable.Flags().BoolVarP(&disableAll, "all", "a", false, "Disable all console options")
cmdConsole.AddCommand(cmdDisable)
cmd.Flags().BoolVarP(&disableAll, "all", "a", false, "Disable all console options")
cmdConsoleStatus := &cobra.Command{
return cmd
}
func (cli *cliConsole) newStatusCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "status",
Short: "Shows status of the console options",
Example: `sudo cscli console status`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
switch csConfig.Cscli.Output {
RunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
consoleCfg := cfg.API.Server.ConsoleConfig
switch cfg.Cscli.Output {
case "human":
cmdConsoleStatusTable(color.Output, *csConfig)
cmdConsoleStatusTable(color.Output, *consoleCfg)
case "json":
c := csConfig.API.Server.ConsoleConfig
out := map[string](*bool){
csconfig.SEND_MANUAL_SCENARIOS: c.ShareManualDecisions,
csconfig.SEND_CUSTOM_SCENARIOS: c.ShareCustomScenarios,
csconfig.SEND_TAINTED_SCENARIOS: c.ShareTaintedScenarios,
csconfig.SEND_CONTEXT: c.ShareContext,
csconfig.CONSOLE_MANAGEMENT: c.ConsoleManagement,
csconfig.SEND_MANUAL_SCENARIOS: consoleCfg.ShareManualDecisions,
csconfig.SEND_CUSTOM_SCENARIOS: consoleCfg.ShareCustomScenarios,
csconfig.SEND_TAINTED_SCENARIOS: consoleCfg.ShareTaintedScenarios,
csconfig.SEND_CONTEXT: consoleCfg.ShareContext,
csconfig.CONSOLE_MANAGEMENT: consoleCfg.ConsoleManagement,
}
data, err := json.MarshalIndent(out, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal configuration: %s", err)
return fmt.Errorf("failed to marshal configuration: %w", err)
}
fmt.Println(string(data))
case "raw":
@ -208,11 +288,11 @@ Disable given information push to the central API.`,
}
rows := [][]string{
{csconfig.SEND_MANUAL_SCENARIOS, fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareManualDecisions)},
{csconfig.SEND_CUSTOM_SCENARIOS, fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareCustomScenarios)},
{csconfig.SEND_TAINTED_SCENARIOS, fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios)},
{csconfig.SEND_CONTEXT, fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareContext)},
{csconfig.CONSOLE_MANAGEMENT, fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ConsoleManagement)},
{csconfig.SEND_MANUAL_SCENARIOS, fmt.Sprintf("%t", *consoleCfg.ShareManualDecisions)},
{csconfig.SEND_CUSTOM_SCENARIOS, fmt.Sprintf("%t", *consoleCfg.ShareCustomScenarios)},
{csconfig.SEND_TAINTED_SCENARIOS, fmt.Sprintf("%t", *consoleCfg.ShareTaintedScenarios)},
{csconfig.SEND_CONTEXT, fmt.Sprintf("%t", *consoleCfg.ShareContext)},
{csconfig.CONSOLE_MANAGEMENT, fmt.Sprintf("%t", *consoleCfg.ConsoleManagement)},
}
for _, row := range rows {
err = csvwriter.Write(row)
@ -222,131 +302,137 @@ Disable given information push to the central API.`,
}
csvwriter.Flush()
}
return nil
},
}
cmdConsole.AddCommand(cmdConsoleStatus)
return cmdConsole
return cmd
}
func dumpConsoleConfig(c *csconfig.LocalApiServerCfg) error {
out, err := yaml.Marshal(c.ConsoleConfig)
func (cli *cliConsole) dumpConfig() error {
serverCfg := cli.cfg().API.Server
out, err := yaml.Marshal(serverCfg.ConsoleConfig)
if err != nil {
return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", c.ConsoleConfigPath, err)
return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", serverCfg.ConsoleConfigPath, err)
}
if c.ConsoleConfigPath == "" {
c.ConsoleConfigPath = csconfig.DefaultConsoleConfigFilePath
log.Debugf("Empty console_path, defaulting to %s", c.ConsoleConfigPath)
if serverCfg.ConsoleConfigPath == "" {
serverCfg.ConsoleConfigPath = csconfig.DefaultConsoleConfigFilePath
log.Debugf("Empty console_path, defaulting to %s", serverCfg.ConsoleConfigPath)
}
if err := os.WriteFile(c.ConsoleConfigPath, out, 0600); err != nil {
return fmt.Errorf("while dumping console config to %s: %w", c.ConsoleConfigPath, err)
if err := os.WriteFile(serverCfg.ConsoleConfigPath, out, 0o600); err != nil {
return fmt.Errorf("while dumping console config to %s: %w", serverCfg.ConsoleConfigPath, err)
}
return nil
}
func SetConsoleOpts(args []string, wanted bool) error {
func (cli *cliConsole) setConsoleOpts(args []string, wanted bool) error {
cfg := cli.cfg()
consoleCfg := cfg.API.Server.ConsoleConfig
for _, arg := range args {
switch arg {
case csconfig.CONSOLE_MANAGEMENT:
if !fflag.PapiClient.IsEnabled() {
continue
}
/*for each flag check if it's already set before setting it*/
if csConfig.API.Server.ConsoleConfig.ConsoleManagement != nil {
if *csConfig.API.Server.ConsoleConfig.ConsoleManagement == wanted {
if consoleCfg.ConsoleManagement != nil {
if *consoleCfg.ConsoleManagement == wanted {
log.Debugf("%s already set to %t", csconfig.CONSOLE_MANAGEMENT, wanted)
} else {
log.Infof("%s set to %t", csconfig.CONSOLE_MANAGEMENT, wanted)
*csConfig.API.Server.ConsoleConfig.ConsoleManagement = wanted
*consoleCfg.ConsoleManagement = wanted
}
} else {
log.Infof("%s set to %t", csconfig.CONSOLE_MANAGEMENT, wanted)
csConfig.API.Server.ConsoleConfig.ConsoleManagement = ptr.Of(wanted)
consoleCfg.ConsoleManagement = ptr.Of(wanted)
}
if csConfig.API.Server.OnlineClient.Credentials != nil {
if cfg.API.Server.OnlineClient.Credentials != nil {
changed := false
if wanted && csConfig.API.Server.OnlineClient.Credentials.PapiURL == "" {
if wanted && cfg.API.Server.OnlineClient.Credentials.PapiURL == "" {
changed = true
csConfig.API.Server.OnlineClient.Credentials.PapiURL = types.PAPIBaseURL
} else if !wanted && csConfig.API.Server.OnlineClient.Credentials.PapiURL != "" {
cfg.API.Server.OnlineClient.Credentials.PapiURL = types.PAPIBaseURL
} else if !wanted && cfg.API.Server.OnlineClient.Credentials.PapiURL != "" {
changed = true
csConfig.API.Server.OnlineClient.Credentials.PapiURL = ""
cfg.API.Server.OnlineClient.Credentials.PapiURL = ""
}
if changed {
fileContent, err := yaml.Marshal(csConfig.API.Server.OnlineClient.Credentials)
fileContent, err := yaml.Marshal(cfg.API.Server.OnlineClient.Credentials)
if err != nil {
return fmt.Errorf("cannot marshal credentials: %s", err)
return fmt.Errorf("cannot marshal credentials: %w", err)
}
log.Infof("Updating credentials file: %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
err = os.WriteFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, fileContent, 0600)
log.Infof("Updating credentials file: %s", cfg.API.Server.OnlineClient.CredentialsFilePath)
err = os.WriteFile(cfg.API.Server.OnlineClient.CredentialsFilePath, fileContent, 0o600)
if err != nil {
return fmt.Errorf("cannot write credentials file: %s", err)
return fmt.Errorf("cannot write credentials file: %w", err)
}
}
}
case csconfig.SEND_CUSTOM_SCENARIOS:
/*for each flag check if it's already set before setting it*/
if csConfig.API.Server.ConsoleConfig.ShareCustomScenarios != nil {
if *csConfig.API.Server.ConsoleConfig.ShareCustomScenarios == wanted {
if consoleCfg.ShareCustomScenarios != nil {
if *consoleCfg.ShareCustomScenarios == wanted {
log.Debugf("%s already set to %t", csconfig.SEND_CUSTOM_SCENARIOS, wanted)
} else {
log.Infof("%s set to %t", csconfig.SEND_CUSTOM_SCENARIOS, wanted)
*csConfig.API.Server.ConsoleConfig.ShareCustomScenarios = wanted
*consoleCfg.ShareCustomScenarios = wanted
}
} else {
log.Infof("%s set to %t", csconfig.SEND_CUSTOM_SCENARIOS, wanted)
csConfig.API.Server.ConsoleConfig.ShareCustomScenarios = ptr.Of(wanted)
consoleCfg.ShareCustomScenarios = ptr.Of(wanted)
}
case csconfig.SEND_TAINTED_SCENARIOS:
/*for each flag check if it's already set before setting it*/
if csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios != nil {
if *csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios == wanted {
if consoleCfg.ShareTaintedScenarios != nil {
if *consoleCfg.ShareTaintedScenarios == wanted {
log.Debugf("%s already set to %t", csconfig.SEND_TAINTED_SCENARIOS, wanted)
} else {
log.Infof("%s set to %t", csconfig.SEND_TAINTED_SCENARIOS, wanted)
*csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios = wanted
*consoleCfg.ShareTaintedScenarios = wanted
}
} else {
log.Infof("%s set to %t", csconfig.SEND_TAINTED_SCENARIOS, wanted)
csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios = ptr.Of(wanted)
consoleCfg.ShareTaintedScenarios = ptr.Of(wanted)
}
case csconfig.SEND_MANUAL_SCENARIOS:
/*for each flag check if it's already set before setting it*/
if csConfig.API.Server.ConsoleConfig.ShareManualDecisions != nil {
if *csConfig.API.Server.ConsoleConfig.ShareManualDecisions == wanted {
if consoleCfg.ShareManualDecisions != nil {
if *consoleCfg.ShareManualDecisions == wanted {
log.Debugf("%s already set to %t", csconfig.SEND_MANUAL_SCENARIOS, wanted)
} else {
log.Infof("%s set to %t", csconfig.SEND_MANUAL_SCENARIOS, wanted)
*csConfig.API.Server.ConsoleConfig.ShareManualDecisions = wanted
*consoleCfg.ShareManualDecisions = wanted
}
} else {
log.Infof("%s set to %t", csconfig.SEND_MANUAL_SCENARIOS, wanted)
csConfig.API.Server.ConsoleConfig.ShareManualDecisions = ptr.Of(wanted)
consoleCfg.ShareManualDecisions = ptr.Of(wanted)
}
case csconfig.SEND_CONTEXT:
/*for each flag check if it's already set before setting it*/
if csConfig.API.Server.ConsoleConfig.ShareContext != nil {
if *csConfig.API.Server.ConsoleConfig.ShareContext == wanted {
if consoleCfg.ShareContext != nil {
if *consoleCfg.ShareContext == wanted {
log.Debugf("%s already set to %t", csconfig.SEND_CONTEXT, wanted)
} else {
log.Infof("%s set to %t", csconfig.SEND_CONTEXT, wanted)
*csConfig.API.Server.ConsoleConfig.ShareContext = wanted
*consoleCfg.ShareContext = wanted
}
} else {
log.Infof("%s set to %t", csconfig.SEND_CONTEXT, wanted)
csConfig.API.Server.ConsoleConfig.ShareContext = ptr.Of(wanted)
consoleCfg.ShareContext = ptr.Of(wanted)
}
default:
return fmt.Errorf("unknown flag %s", arg)
}
}
if err := dumpConsoleConfig(csConfig.API.Server); err != nil {
return fmt.Errorf("failed writing console config: %s", err)
if err := cli.dumpConfig(); err != nil {
return fmt.Errorf("failed writing console config: %w", err)
}
return nil

31
cmd/crowdsec-cli/console_table.go
View file

@ -9,7 +9,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
func cmdConsoleStatusTable(out io.Writer, csConfig csconfig.Config) {
func cmdConsoleStatusTable(out io.Writer, consoleCfg csconfig.ConsoleConfig) {
t := newTable(out)
t.SetRowLines(false)
@ -17,43 +17,32 @@ func cmdConsoleStatusTable(out io.Writer, csConfig csconfig.Config) {
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
for _, option := range csconfig.CONSOLE_CONFIGS {
activated := string(emoji.CrossMark)
switch option {
case csconfig.SEND_CUSTOM_SCENARIOS:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareCustomScenarios {
if *consoleCfg.ShareCustomScenarios {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send alerts from custom scenarios to the console")
case csconfig.SEND_MANUAL_SCENARIOS:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareManualDecisions {
if *consoleCfg.ShareManualDecisions {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send manual decisions to the console")
case csconfig.SEND_TAINTED_SCENARIOS:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios {
if *consoleCfg.ShareTaintedScenarios {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send alerts from tainted scenarios to the console")
case csconfig.SEND_CONTEXT:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareContext {
if *consoleCfg.ShareContext {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send context with alerts to the console")
case csconfig.CONSOLE_MANAGEMENT:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ConsoleManagement {
if *consoleCfg.ConsoleManagement {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Receive decisions from console")
}
t.AddRow(option, activated, csconfig.CONSOLE_CONFIGS_HELP[option])
}
t.Render()

20
cmd/crowdsec-cli/copyfile.go
View file

@ -18,56 +18,66 @@ func copyFileContents(src, dst string) (err error) {
return
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
return
}
defer func() {
cerr := out.Close()
if err == nil {
err = cerr
}
}()
if _, err = io.Copy(out, in); err != nil {
return
}
err = out.Sync()
return
}
/*copy the file, ioutile doesn't offer the feature*/
func CopyFile(sourceSymLink, destinationFile string) (err error) {
func CopyFile(sourceSymLink, destinationFile string) error {
sourceFile, err := filepath.EvalSymlinks(sourceSymLink)
if err != nil {
log.Infof("Not a symlink : %s", err)
sourceFile = sourceSymLink
}
sourceFileStat, err := os.Stat(sourceFile)
if err != nil {
return
return err
}
if !sourceFileStat.Mode().IsRegular() {
// cannot copy non-regular files (e.g., directories,
// symlinks, devices, etc.)
return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String())
}
destinationFileStat, err := os.Stat(destinationFile)
if err != nil {
if !os.IsNotExist(err) {
return
return err
}
} else {
if !(destinationFileStat.Mode().IsRegular()) {
return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String())
}
if os.SameFile(sourceFileStat, destinationFileStat) {
return
return err
}
}
if err = os.Link(sourceFile, destinationFile); err != nil {
err = copyFileContents(sourceFile, destinationFile)
}
return
return err
}

168
cmd/crowdsec-cli/dashboard.go
View file

@ -19,15 +19,14 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/metabase"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/metabase"
)
var (
metabaseUser = "crowdsec@crowdsec.net"
metabasePassword string
metabaseDbPath string
metabaseDBPath string
metabaseConfigPath string
metabaseConfigFolder = "metabase/"
metabaseConfigFile = "metabase.yaml"
@ -43,9 +42,18 @@ var (
// information needed to set up a random password on user's behalf
)
func NewDashboardCmd() *cobra.Command {
/* ---- UPDATE COMMAND */
var cmdDashboard = &cobra.Command{
type cliDashboard struct {
cfg configGetter
}
func NewCLIDashboard(cfg configGetter) *cliDashboard {
return &cliDashboard{
cfg: cfg,
}
}
func (cli *cliDashboard) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "dashboard [command]",
Short: "Manage your metabase dashboard container [requires local API]",
Long: `Install/Start/Stop/Remove a metabase container exposing dashboard and metrics.
@ -59,8 +67,9 @@ cscli dashboard start
cscli dashboard stop
cscli dashboard remove
`,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.LAPI(csConfig); err != nil {
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := require.LAPI(cfg); err != nil {
return err
}
@ -68,13 +77,13 @@ cscli dashboard remove
return err
}
metabaseConfigFolderPath := filepath.Join(csConfig.ConfigPaths.ConfigDir, metabaseConfigFolder)
metabaseConfigFolderPath := filepath.Join(cfg.ConfigPaths.ConfigDir, metabaseConfigFolder)
metabaseConfigPath = filepath.Join(metabaseConfigFolderPath, metabaseConfigFile)
if err := os.MkdirAll(metabaseConfigFolderPath, os.ModePerm); err != nil {
return err
}
if err := require.DB(csConfig); err != nil {
if err := require.DB(cfg); err != nil {
return err
}
@ -89,23 +98,24 @@ cscli dashboard remove
metabaseContainerID = oldContainerID
}
}
return nil
},
}
cmdDashboard.AddCommand(NewDashboardSetupCmd())
cmdDashboard.AddCommand(NewDashboardStartCmd())
cmdDashboard.AddCommand(NewDashboardStopCmd())
cmdDashboard.AddCommand(NewDashboardShowPasswordCmd())
cmdDashboard.AddCommand(NewDashboardRemoveCmd())
cmd.AddCommand(cli.newSetupCmd())
cmd.AddCommand(cli.newStartCmd())
cmd.AddCommand(cli.newStopCmd())
cmd.AddCommand(cli.newShowPasswordCmd())
cmd.AddCommand(cli.newRemoveCmd())
return cmdDashboard
return cmd
}
func NewDashboardSetupCmd() *cobra.Command {
func (cli *cliDashboard) newSetupCmd() *cobra.Command {
var force bool
var cmdDashSetup = &cobra.Command{
cmd := &cobra.Command{
Use: "setup",
Short: "Setup a metabase container.",
Long: `Perform a metabase docker setup, download standard dashboards, create a fresh user and start the container`,
@ -116,9 +126,9 @@ cscli dashboard setup
cscli dashboard setup --listen 0.0.0.0
cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
`,
RunE: func(cmd *cobra.Command, args []string) error {
if metabaseDbPath == "" {
metabaseDbPath = csConfig.ConfigPaths.DataDir
RunE: func(_ *cobra.Command, _ []string) error {
if metabaseDBPath == "" {
metabaseDBPath = cli.cfg().ConfigPaths.DataDir
}
if metabasePassword == "" {
@ -139,10 +149,10 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
if err != nil {
return err
}
if err = chownDatabase(dockerGroup.Gid); err != nil {
if err = cli.chownDatabase(dockerGroup.Gid); err != nil {
return err
}
mb, err := metabase.SetupMetabase(csConfig.API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDbPath, dockerGroup.Gid, metabaseContainerID, metabaseImage)
mb, err := metabase.SetupMetabase(cli.cfg().API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDBPath, dockerGroup.Gid, metabaseContainerID, metabaseImage)
if err != nil {
return err
}
@ -155,29 +165,32 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
fmt.Printf("\tURL : '%s'\n", mb.Config.ListenURL)
fmt.Printf("\tusername : '%s'\n", mb.Config.Username)
fmt.Printf("\tpassword : '%s'\n", mb.Config.Password)
return nil
},
}
cmdDashSetup.Flags().BoolVarP(&force, "force", "f", false, "Force setup : override existing files")
cmdDashSetup.Flags().StringVarP(&metabaseDbPath, "dir", "d", "", "Shared directory with metabase container")
cmdDashSetup.Flags().StringVarP(&metabaseListenAddress, "listen", "l", metabaseListenAddress, "Listen address of container")
cmdDashSetup.Flags().StringVar(&metabaseImage, "metabase-image", metabaseImage, "Metabase image to use")
cmdDashSetup.Flags().StringVarP(&metabaseListenPort, "port", "p", metabaseListenPort, "Listen port of container")
cmdDashSetup.Flags().BoolVarP(&forceYes, "yes", "y", false, "force yes")
//cmdDashSetup.Flags().StringVarP(&metabaseUser, "user", "u", "crowdsec@crowdsec.net", "metabase user")
cmdDashSetup.Flags().StringVar(&metabasePassword, "password", "", "metabase password")
return cmdDashSetup
flags := cmd.Flags()
flags.BoolVarP(&force, "force", "f", false, "Force setup : override existing files")
flags.StringVarP(&metabaseDBPath, "dir", "d", "", "Shared directory with metabase container")
flags.StringVarP(&metabaseListenAddress, "listen", "l", metabaseListenAddress, "Listen address of container")
flags.StringVar(&metabaseImage, "metabase-image", metabaseImage, "Metabase image to use")
flags.StringVarP(&metabaseListenPort, "port", "p", metabaseListenPort, "Listen port of container")
flags.BoolVarP(&forceYes, "yes", "y", false, "force yes")
// flags.StringVarP(&metabaseUser, "user", "u", "crowdsec@crowdsec.net", "metabase user")
flags.StringVar(&metabasePassword, "password", "", "metabase password")
return cmd
}
func NewDashboardStartCmd() *cobra.Command {
var cmdDashStart = &cobra.Command{
func (cli *cliDashboard) newStartCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "start",
Short: "Start the metabase container.",
Long: `Stats the metabase container using docker.`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
mb, err := metabase.NewMetabase(metabaseConfigPath, metabaseContainerID)
if err != nil {
return err
@ -191,51 +204,57 @@ func NewDashboardStartCmd() *cobra.Command {
}
log.Infof("Started metabase")
log.Infof("url : http://%s:%s", mb.Config.ListenAddr, mb.Config.ListenPort)
return nil
},
}
cmdDashStart.Flags().BoolVarP(&forceYes, "yes", "y", false, "force yes")
return cmdDashStart
cmd.Flags().BoolVarP(&forceYes, "yes", "y", false, "force yes")
return cmd
}
func NewDashboardStopCmd() *cobra.Command {
var cmdDashStop = &cobra.Command{
func (cli *cliDashboard) newStopCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "stop",
Short: "Stops the metabase container.",
Long: `Stops the metabase container using docker.`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
if err := metabase.StopContainer(metabaseContainerID); err != nil {
return fmt.Errorf("unable to stop container '%s': %s", metabaseContainerID, err)
}
return nil
},
}
return cmdDashStop
return cmd
}
func NewDashboardShowPasswordCmd() *cobra.Command {
var cmdDashShowPassword = &cobra.Command{Use: "show-password",
func (cli *cliDashboard) newShowPasswordCmd() *cobra.Command {
cmd := &cobra.Command{Use: "show-password",
Short: "displays password of metabase.",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
m := metabase.Metabase{}
if err := m.LoadConfig(metabaseConfigPath); err != nil {
return err
}
log.Printf("'%s'", m.Config.Password)
return nil
},
}
return cmdDashShowPassword
return cmd
}
func NewDashboardRemoveCmd() *cobra.Command {
func (cli *cliDashboard) newRemoveCmd() *cobra.Command {
var force bool
var cmdDashRemove = &cobra.Command{
cmd := &cobra.Command{
Use: "remove",
Short: "removes the metabase container.",
Long: `removes the metabase container using docker.`,
@ -245,7 +264,7 @@ func NewDashboardRemoveCmd() *cobra.Command {
cscli dashboard remove
cscli dashboard remove --force
`,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
if !forceYes {
var answer bool
prompt := &survey.Confirm{
@ -282,8 +301,8 @@ cscli dashboard remove --force
}
log.Infof("container %s stopped & removed", metabaseContainerID)
}
log.Debugf("Removing metabase db %s", csConfig.ConfigPaths.DataDir)
if err := metabase.RemoveDatabase(csConfig.ConfigPaths.DataDir); err != nil {
log.Debugf("Removing metabase db %s", cli.cfg().ConfigPaths.DataDir)
if err := metabase.RemoveDatabase(cli.cfg().ConfigPaths.DataDir); err != nil {
log.Warnf("failed to remove metabase internal db : %s", err)
}
if force {
@ -297,20 +316,25 @@ cscli dashboard remove --force
}
}
}
return nil
},
}
cmdDashRemove.Flags().BoolVarP(&force, "force", "f", false, "Remove also the metabase image")
cmdDashRemove.Flags().BoolVarP(&forceYes, "yes", "y", false, "force yes")
return cmdDashRemove
flags := cmd.Flags()
flags.BoolVarP(&force, "force", "f", false, "Remove also the metabase image")
flags.BoolVarP(&forceYes, "yes", "y", false, "force yes")
return cmd
}
func passwordIsValid(password string) bool {
hasDigit := false
for _, j := range password {
if unicode.IsDigit(j) {
hasDigit = true
break
}
}
@ -318,8 +342,8 @@ func passwordIsValid(password string) bool {
if !hasDigit || len(password) < 6 {
return false
}
return true
return true
}
func checkSystemMemory(forceYes *bool) error {
@ -327,8 +351,10 @@ func checkSystemMemory(forceYes *bool) error {
if totMem >= uint64(math.Pow(2, 30)) {
return nil
}
if !*forceYes {
var answer bool
prompt := &survey.Confirm{
Message: "Metabase requires 1-2GB of RAM, your system is below this requirement continue ?",
Default: true,
@ -336,12 +362,16 @@ func checkSystemMemory(forceYes *bool) error {
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask about RAM check: %s", err)
}
if !answer {
return fmt.Errorf("user stated no to continue")
}
return nil
}
log.Warn("Metabase requires 1-2GB of RAM, your system is below this requirement")
return nil
}
@ -349,25 +379,32 @@ func warnIfNotLoopback(addr string) {
if addr == "127.0.0.1" || addr == "::1" {
return
}
log.Warnf("You are potentially exposing your metabase port to the internet (addr: %s), please consider using a reverse proxy", addr)
}
func disclaimer(forceYes *bool) error {
if !*forceYes {
var answer bool
prompt := &survey.Confirm{
Message: "CrowdSec takes no responsibility for the security of your metabase instance. Do you accept these responsibilities ?",
Default: true,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask to question: %s", err)
}
if !answer {
return fmt.Errorf("user stated no to responsibilities")
}
return nil
}
log.Warn("CrowdSec takes no responsibility for the security of your metabase instance. You used force yes, so you accept this disclaimer")
return nil
}
@ -376,19 +413,24 @@ func checkGroups(forceYes *bool) (*user.Group, error) {
if err == nil {
return dockerGroup, nil
}
if !*forceYes {
var answer bool
prompt := &survey.Confirm{
Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup),
Default: true,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return dockerGroup, fmt.Errorf("unable to ask to question: %s", err)
}
if !answer {
return dockerGroup, fmt.Errorf("unable to continue without creating '%s' group", crowdsecGroup)
}
}
groupAddCmd, err := exec.LookPath("groupadd")
if err != nil {
return dockerGroup, fmt.Errorf("unable to find 'groupadd' command, can't continue")
@ -398,23 +440,28 @@ func checkGroups(forceYes *bool) (*user.Group, error) {
if err := groupAdd.Run(); err != nil {
return dockerGroup, fmt.Errorf("unable to add group '%s': %s", dockerGroup, err)
}
return user.LookupGroup(crowdsecGroup)
}
func chownDatabase(gid string) error {
func (cli *cliDashboard) chownDatabase(gid string) error {
cfg := cli.cfg()
intID, err := strconv.Atoi(gid)
if err != nil {
return fmt.Errorf("unable to convert group ID to int: %s", err)
}
if stat, err := os.Stat(csConfig.DbConfig.DbPath); !os.IsNotExist(err) {
if stat, err := os.Stat(cfg.DbConfig.DbPath); !os.IsNotExist(err) {
info := stat.Sys()
if err := os.Chown(csConfig.DbConfig.DbPath, int(info.(*syscall.Stat_t).Uid), intID); err != nil {
return fmt.Errorf("unable to chown sqlite db file '%s': %s", csConfig.DbConfig.DbPath, err)
if err := os.Chown(cfg.DbConfig.DbPath, int(info.(*syscall.Stat_t).Uid), intID); err != nil {
return fmt.Errorf("unable to chown sqlite db file '%s': %s", cfg.DbConfig.DbPath, err)
}
}
if csConfig.DbConfig.Type == "sqlite" && csConfig.DbConfig.UseWal != nil && *csConfig.DbConfig.UseWal {
if cfg.DbConfig.Type == "sqlite" && cfg.DbConfig.UseWal != nil && *cfg.DbConfig.UseWal {
for _, ext := range []string{"-wal", "-shm"} {
file := csConfig.DbConfig.DbPath + ext
file := cfg.DbConfig.DbPath + ext
if stat, err := os.Stat(file); !os.IsNotExist(err) {
info := stat.Sys()
if err := os.Chown(file, int(info.(*syscall.Stat_t).Uid), intID); err != nil {
@ -423,5 +470,6 @@ func chownDatabase(gid string) error {
}
}
}
return nil
}

18
cmd/crowdsec-cli/dashboard_unsupported.go
View file

@ -9,14 +9,24 @@ import (
"github.com/spf13/cobra"
)
func NewDashboardCmd() *cobra.Command {
var cmdDashboard = &cobra.Command{
type cliDashboard struct{
cfg configGetter
}
func NewCLIDashboard(cfg configGetter) *cliDashboard {
return &cliDashboard{
cfg: cfg,
}
}
func (cli cliDashboard) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "dashboard",
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
Run: func(_ *cobra.Command, _ []string) {
log.Infof("Dashboard command is disabled on %s", runtime.GOOS)
},
}
return cmdDashboard
return cmd
}

185
cmd/crowdsec-cli/decisions.go
View file

@ -25,7 +25,7 @@ import (
var Client *apiclient.ApiClient
func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
func (cli *cliDecisions) decisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
/*here we cheat a bit : to make it more readable for the user, we dedup some entries*/
spamLimit := make(map[string]bool)
skipped := 0
@ -33,27 +33,36 @@ func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error
for aIdx := 0; aIdx < len(*alerts); aIdx++ {
alertItem := (*alerts)[aIdx]
newDecisions := make([]*models.Decision, 0)
for _, decisionItem := range alertItem.Decisions {
spamKey := fmt.Sprintf("%t:%s:%s:%s", *decisionItem.Simulated, *decisionItem.Type, *decisionItem.Scope, *decisionItem.Value)
if _, ok := spamLimit[spamKey]; ok {
skipped++
continue
}
spamLimit[spamKey] = true
newDecisions = append(newDecisions, decisionItem)
}
alertItem.Decisions = newDecisions
}
if csConfig.Cscli.Output == "raw" {
switch cli.cfg().Cscli.Output {
case "raw":
csvwriter := csv.NewWriter(os.Stdout)
header := []string{"id", "source", "ip", "reason", "action", "country", "as", "events_count", "expiration", "simulated", "alert_id"}
if printMachine {
header = append(header, "machine")
}
err := csvwriter.Write(header)
if err != nil {
return err
}
for _, alertItem := range *alerts {
for _, decisionItem := range alertItem.Decisions {
raw := []string{
@ -79,31 +88,46 @@ func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error
}
}
}
csvwriter.Flush()
} else if csConfig.Cscli.Output == "json" {
case "json":
if *alerts == nil {
// avoid returning "null" in `json"
// could be cleaner if we used slice of alerts directly
fmt.Println("[]")
return nil
}
x, _ := json.MarshalIndent(alerts, "", " ")
fmt.Printf("%s", string(x))
} else if csConfig.Cscli.Output == "human" {
case "human":
if len(*alerts) == 0 {
fmt.Println("No active decisions")
return nil
}
decisionsTable(color.Output, alerts, printMachine)
cli.decisionsTable(color.Output, alerts, printMachine)
if skipped > 0 {
fmt.Printf("%d duplicated entries skipped\n", skipped)
}
}
return nil
}
func NewDecisionsCmd() *cobra.Command {
var cmdDecisions = &cobra.Command{
type cliDecisions struct {
cfg configGetter
}
func NewCLIDecisions(cfg configGetter) *cliDecisions {
return &cliDecisions{
cfg: cfg,
}
}
func (cli *cliDecisions) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "decisions [action]",
Short: "Manage decisions",
Long: `Add/List/Delete/Import decisions from LAPI`,
@ -112,17 +136,18 @@ func NewDecisionsCmd() *cobra.Command {
/*TBD example*/
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIClient(); err != nil {
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := cfg.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
}
password := strfmt.Password(csConfig.API.Client.Credentials.Password)
apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL)
password := strfmt.Password(cfg.API.Client.Credentials.Password)
apiurl, err := url.Parse(cfg.API.Client.Credentials.URL)
if err != nil {
return fmt.Errorf("parsing api url %s: %w", csConfig.API.Client.Credentials.URL, err)
return fmt.Errorf("parsing api url %s: %w", cfg.API.Client.Credentials.URL, err)
}
Client, err = apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
MachineID: cfg.API.Client.Credentials.Login,
Password: password,
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiurl,
@ -131,19 +156,20 @@ func NewDecisionsCmd() *cobra.Command {
if err != nil {
return fmt.Errorf("creating api client: %w", err)
}
return nil
},
}
cmdDecisions.AddCommand(NewDecisionsListCmd())
cmdDecisions.AddCommand(NewDecisionsAddCmd())
cmdDecisions.AddCommand(NewDecisionsDeleteCmd())
cmdDecisions.AddCommand(NewDecisionsImportCmd())
cmd.AddCommand(cli.newListCmd())
cmd.AddCommand(cli.newAddCmd())
cmd.AddCommand(cli.newDeleteCmd())
cmd.AddCommand(cli.newImportCmd())
return cmdDecisions
return cmd
}
func NewDecisionsListCmd() *cobra.Command {
func (cli *cliDecisions) newListCmd() *cobra.Command {
var filter = apiclient.AlertsListOpts{
ValueEquals: new(string),
ScopeEquals: new(string),
@ -157,11 +183,13 @@ func NewDecisionsListCmd() *cobra.Command {
IncludeCAPI: new(bool),
Limit: new(int),
}
NoSimu := new(bool)
contained := new(bool)
var printMachine bool
var cmdDecisionsList = &cobra.Command{
cmd := &cobra.Command{
Use: "list [options]",
Short: "List decisions from LAPI",
Example: `cscli decisions list -i 1.2.3.4
@ -171,7 +199,7 @@ cscli decisions list -t ban
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(cmd *cobra.Command, _ []string) error {
var err error
/*take care of shorthand options*/
if err = manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil {
@ -243,7 +271,7 @@ cscli decisions list -t ban
return fmt.Errorf("unable to retrieve decisions: %w", err)
}
err = DecisionsToTable(alerts, printMachine)
err = cli.decisionsToTable(alerts, printMachine)
if err != nil {
return fmt.Errorf("unable to print decisions: %w", err)
}
@ -251,26 +279,26 @@ cscli decisions list -t ban
return nil
},
}
cmdDecisionsList.Flags().SortFlags = false
cmdDecisionsList.Flags().BoolVarP(filter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
cmdDecisionsList.Flags().StringVar(filter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
cmdDecisionsList.Flags().StringVar(filter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
cmdDecisionsList.Flags().StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)")
cmdDecisionsList.Flags().StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)")
cmdDecisionsList.Flags().StringVar(filter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmdDecisionsList.Flags().StringVarP(filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)")
cmdDecisionsList.Flags().StringVarP(filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)")
cmdDecisionsList.Flags().StringVarP(filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
cmdDecisionsList.Flags().StringVarP(filter.RangeEquals, "range", "r", "", "restrict to alerts from this source range (shorthand for --scope range --value <RANGE>)")
cmdDecisionsList.Flags().IntVarP(filter.Limit, "limit", "l", 100, "number of alerts to get (use 0 to remove the limit)")
cmdDecisionsList.Flags().BoolVar(NoSimu, "no-simu", false, "exclude decisions in simulation mode")
cmdDecisionsList.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that triggered decisions")
cmdDecisionsList.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
cmd.Flags().SortFlags = false
cmd.Flags().BoolVarP(filter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
cmd.Flags().StringVar(filter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
cmd.Flags().StringVar(filter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
cmd.Flags().StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)")
cmd.Flags().StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)")
cmd.Flags().StringVar(filter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmd.Flags().StringVarP(filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)")
cmd.Flags().StringVarP(filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)")
cmd.Flags().StringVarP(filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
cmd.Flags().StringVarP(filter.RangeEquals, "range", "r", "", "restrict to alerts from this source range (shorthand for --scope range --value <RANGE>)")
cmd.Flags().IntVarP(filter.Limit, "limit", "l", 100, "number of alerts to get (use 0 to remove the limit)")
cmd.Flags().BoolVar(NoSimu, "no-simu", false, "exclude decisions in simulation mode")
cmd.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that triggered decisions")
cmd.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
return cmdDecisionsList
return cmd
}
func NewDecisionsAddCmd() *cobra.Command {
func (cli *cliDecisions) newAddCmd() *cobra.Command {
var (
addIP string
addRange string
@ -281,7 +309,7 @@ func NewDecisionsAddCmd() *cobra.Command {
addType string
)
var cmdDecisionsAdd = &cobra.Command{
cmd := &cobra.Command{
Use: "add [options]",
Short: "Add decision to LAPI",
Example: `cscli decisions add --ip 1.2.3.4
@ -292,7 +320,7 @@ cscli decisions add --scope username --value foobar
/*TBD : fix long and example*/
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(cmd *cobra.Command, _ []string) error {
var err error
alerts := models.AddAlertsRequest{}
origin := types.CscliOrigin
@ -306,7 +334,7 @@ cscli decisions add --scope username --value foobar
createdAt := time.Now().UTC().Format(time.RFC3339)
/*take care of shorthand options*/
if err := manageCliDecisionAlerts(&addIP, &addRange, &addScope, &addValue); err != nil {
if err = manageCliDecisionAlerts(&addIP, &addRange, &addScope, &addValue); err != nil {
return err
}
@ -318,11 +346,11 @@ cscli decisions add --scope username --value foobar
addScope = types.Range
} else if addValue == "" {
printHelp(cmd)
return fmt.Errorf("Missing arguments, a value is required (--ip, --range or --scope and --value)")
return fmt.Errorf("missing arguments, a value is required (--ip, --range or --scope and --value)")
}
if addReason == "" {
addReason = fmt.Sprintf("manual '%s' from '%s'", addType, csConfig.API.Client.Credentials.Login)
addReason = fmt.Sprintf("manual '%s' from '%s'", addType, cli.cfg().API.Client.Credentials.Login)
}
decision := models.Decision{
Duration: &addDuration,
@ -365,23 +393,24 @@ cscli decisions add --scope username --value foobar
}
log.Info("Decision successfully added")
return nil
},
}
cmdDecisionsAdd.Flags().SortFlags = false
cmdDecisionsAdd.Flags().StringVarP(&addIP, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmdDecisionsAdd.Flags().StringVarP(&addRange, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmdDecisionsAdd.Flags().StringVarP(&addDuration, "duration", "d", "4h", "Decision duration (ie. 1h,4h,30m)")
cmdDecisionsAdd.Flags().StringVarP(&addValue, "value", "v", "", "The value (ie. --scope username --value foobar)")
cmdDecisionsAdd.Flags().StringVar(&addScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)")
cmdDecisionsAdd.Flags().StringVarP(&addReason, "reason", "R", "", "Decision reason (ie. scenario-name)")
cmdDecisionsAdd.Flags().StringVarP(&addType, "type", "t", "ban", "Decision type (ie. ban,captcha,throttle)")
cmd.Flags().SortFlags = false
cmd.Flags().StringVarP(&addIP, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmd.Flags().StringVarP(&addRange, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmd.Flags().StringVarP(&addDuration, "duration", "d", "4h", "Decision duration (ie. 1h,4h,30m)")
cmd.Flags().StringVarP(&addValue, "value", "v", "", "The value (ie. --scope username --value foobar)")
cmd.Flags().StringVar(&addScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)")
cmd.Flags().StringVarP(&addReason, "reason", "R", "", "Decision reason (ie. scenario-name)")
cmd.Flags().StringVarP(&addType, "type", "t", "ban", "Decision type (ie. ban,captcha,throttle)")
return cmdDecisionsAdd
return cmd
}
func NewDecisionsDeleteCmd() *cobra.Command {
func (cli *cliDecisions) newDeleteCmd() *cobra.Command {
var delFilter = apiclient.DecisionsDeleteOpts{
ScopeEquals: new(string),
ValueEquals: new(string),
@ -391,11 +420,14 @@ func NewDecisionsDeleteCmd() *cobra.Command {
ScenarioEquals: new(string),
OriginEquals: new(string),
}
var delDecisionId string
var delDecisionID string
var delDecisionAll bool
contained := new(bool)
var cmdDecisionsDelete = &cobra.Command{
cmd := &cobra.Command{
Use: "delete [options]",
Short: "Delete decisions",
DisableAutoGenTag: true,
@ -406,21 +438,21 @@ cscli decisions delete --id 42
cscli decisions delete --type captcha
`,
/*TBD : refaire le Long/Example*/
PreRunE: func(cmd *cobra.Command, args []string) error {
PreRunE: func(cmd *cobra.Command, _ []string) error {
if delDecisionAll {
return nil
}
if *delFilter.ScopeEquals == "" && *delFilter.ValueEquals == "" &&
*delFilter.TypeEquals == "" && *delFilter.IPEquals == "" &&
*delFilter.RangeEquals == "" && *delFilter.ScenarioEquals == "" &&
*delFilter.OriginEquals == "" && delDecisionId == "" {
*delFilter.OriginEquals == "" && delDecisionID == "" {
cmd.Usage()
return fmt.Errorf("at least one filter or --all must be specified")
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
var err error
var decisions *models.DeleteDecisionResponse
@ -453,36 +485,37 @@ cscli decisions delete --type captcha
delFilter.Contains = new(bool)
}
if delDecisionId == "" {
if delDecisionID == "" {
decisions, _, err = Client.Decisions.Delete(context.Background(), delFilter)
if err != nil {
return fmt.Errorf("Unable to delete decisions: %v", err)
return fmt.Errorf("unable to delete decisions: %v", err)
}
} else {
if _, err = strconv.Atoi(delDecisionId); err != nil {
return fmt.Errorf("id '%s' is not an integer: %v", delDecisionId, err)
if _, err = strconv.Atoi(delDecisionID); err != nil {
return fmt.Errorf("id '%s' is not an integer: %v", delDecisionID, err)
}
decisions, _, err = Client.Decisions.DeleteOne(context.Background(), delDecisionId)
decisions, _, err = Client.Decisions.DeleteOne(context.Background(), delDecisionID)
if err != nil {
return fmt.Errorf("Unable to delete decision: %v", err)
return fmt.Errorf("unable to delete decision: %v", err)
}
}
log.Infof("%s decision(s) deleted", decisions.NbDeleted)
return nil
},
}
cmdDecisionsDelete.Flags().SortFlags = false
cmdDecisionsDelete.Flags().StringVarP(delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmdDecisionsDelete.Flags().StringVarP(delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmdDecisionsDelete.Flags().StringVarP(delFilter.TypeEquals, "type", "t", "", "the decision type (ie. ban,captcha)")
cmdDecisionsDelete.Flags().StringVarP(delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmdDecisionsDelete.Flags().StringVarP(delFilter.ScenarioEquals, "scenario", "s", "", "the scenario name (ie. crowdsecurity/ssh-bf)")
cmdDecisionsDelete.Flags().StringVar(delFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmd.Flags().SortFlags = false
cmd.Flags().StringVarP(delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmd.Flags().StringVarP(delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmd.Flags().StringVarP(delFilter.TypeEquals, "type", "t", "", "the decision type (ie. ban,captcha)")
cmd.Flags().StringVarP(delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmd.Flags().StringVarP(delFilter.ScenarioEquals, "scenario", "s", "", "the scenario name (ie. crowdsecurity/ssh-bf)")
cmd.Flags().StringVar(delFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmdDecisionsDelete.Flags().StringVar(&delDecisionId, "id", "", "decision id")
cmdDecisionsDelete.Flags().BoolVar(&delDecisionAll, "all", false, "delete all decisions")
cmdDecisionsDelete.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
cmd.Flags().StringVar(&delDecisionID, "id", "", "decision id")
cmd.Flags().BoolVar(&delDecisionAll, "all", false, "delete all decisions")
cmd.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
return cmdDecisionsDelete
return cmd
}

25
cmd/crowdsec-cli/decisions_import.go
View file

@ -37,21 +37,25 @@ func parseDecisionList(content []byte, format string) ([]decisionRaw, error) {
switch format {
case "values":
log.Infof("Parsing values")
scanner := bufio.NewScanner(bytes.NewReader(content))
for scanner.Scan() {
value := strings.TrimSpace(scanner.Text())
ret = append(ret, decisionRaw{Value: value})
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("unable to parse values: '%s'", err)
}
case "json":
log.Infof("Parsing json")
if err := json.Unmarshal(content, &ret); err != nil {
return nil, err
}
case "csv":
log.Infof("Parsing csv")
if err := csvutil.Unmarshal(content, &ret); err != nil {
return nil, fmt.Errorf("unable to parse csv: '%s'", err)
}
@ -63,7 +67,7 @@ func parseDecisionList(content []byte, format string) ([]decisionRaw, error) {
}
func runDecisionsImport(cmd *cobra.Command, args []string) error {
func (cli *cliDecisions) runImport(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
input, err := flags.GetString("input")
@ -75,6 +79,7 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
if defaultDuration == "" {
return fmt.Errorf("--duration cannot be empty")
}
@ -83,6 +88,7 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
if defaultScope == "" {
return fmt.Errorf("--scope cannot be empty")
}
@ -91,6 +97,7 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
if defaultReason == "" {
return fmt.Errorf("--reason cannot be empty")
}
@ -99,6 +106,7 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
if defaultType == "" {
return fmt.Errorf("--type cannot be empty")
}
@ -152,6 +160,7 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error {
}
decisions := make([]*models.Decision, len(decisionsListRaw))
for i, d := range decisionsListRaw {
if d.Value == "" {
return fmt.Errorf("item %d: missing 'value'", i)
@ -222,17 +231,19 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error {
}
log.Infof("Imported %d decisions", len(decisions))
return nil
}
func NewDecisionsImportCmd() *cobra.Command {
var cmdDecisionsImport = &cobra.Command{
func (cli *cliDecisions) newImportCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "import [options]",
Short: "Import decisions from a file or pipe",
Long: "expected format:\n" +
"csv : any of duration,reason,scope,type,value, with a header line\n" +
"json :" + "`{" + `"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"` + "}`",
Args: cobra.NoArgs,
DisableAutoGenTag: true,
Example: `decisions.csv:
duration,scope,value
@ -250,10 +261,10 @@ Raw values, standard input:
$ echo "1.2.3.4" | cscli decisions import -i - --format values
`,
RunE: runDecisionsImport,
RunE: cli.runImport,
}
flags := cmdDecisionsImport.Flags()
flags := cmd.Flags()
flags.SortFlags = false
flags.StringP("input", "i", "", "Input file")
flags.StringP("duration", "d", "4h", "Decision duration: 1h,4h,30m")
@ -263,7 +274,7 @@ $ echo "1.2.3.4" | cscli decisions import -i - --format values
flags.Int("batch", 0, "Split import in batches of N decisions")
flags.String("format", "", "Input format: 'json', 'csv' or 'values' (each line is a value, no headers)")
cmdDecisionsImport.MarkFlagRequired("input")
cmd.MarkFlagRequired("input")
return cmdDecisionsImport
return cmd
}

6
cmd/crowdsec-cli/decisions_table.go
View file

@ -8,13 +8,15 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/models"
)
func decisionsTable(out io.Writer, alerts *models.GetAlertsResponse, printMachine bool) {
func (cli *cliDecisions) decisionsTable(out io.Writer, alerts *models.GetAlertsResponse, printMachine bool) {
t := newTable(out)
t.SetRowLines(false)
header := []string{"ID", "Source", "Scope:Value", "Reason", "Action", "Country", "AS", "Events", "expiration", "Alert ID"}
if printMachine {
header = append(header, "Machine")
}
t.SetHeaders(header...)
for _, alertItem := range *alerts {
@ -22,6 +24,7 @@ func decisionsTable(out io.Writer, alerts *models.GetAlertsResponse, printMachin
if *alertItem.Simulated {
*decisionItem.Type = fmt.Sprintf("(simul)%s", *decisionItem.Type)
}
row := []string{
strconv.Itoa(int(decisionItem.ID)),
*decisionItem.Origin,
@ -42,5 +45,6 @@ func decisionsTable(out io.Writer, alerts *models.GetAlertsResponse, printMachin
t.AddRow(row...)
}
}
t.Render()
}

49
cmd/crowdsec-cli/doc.go Normal file
View file

@ -0,0 +1,49 @@
package main
import (
"fmt"
"path/filepath"
"strings"
"github.com/spf13/cobra"
"github.com/spf13/cobra/doc"
)
type cliDoc struct{}
func NewCLIDoc() *cliDoc {
return &cliDoc{}
}
func (cli cliDoc) NewCommand(rootCmd *cobra.Command) *cobra.Command {
cmd := &cobra.Command{
Use: "doc",
Short: "Generate the documentation in `./doc/`. Directory must exist.",
Args: cobra.ExactArgs(0),
Hidden: true,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
if err := doc.GenMarkdownTreeCustom(rootCmd, "./doc/", cli.filePrepender, cli.linkHandler); err != nil {
return fmt.Errorf("failed to generate cobra doc: %s", err)
}
return nil
},
}
return cmd
}
func (cli cliDoc) filePrepender(filename string) string {
const header = `---
id: %s
title: %s
---
`
name := filepath.Base(filename)
base := strings.TrimSuffix(name, filepath.Ext(name))
return fmt.Sprintf(header, base, strings.ReplaceAll(base, "_", " "))
}
func (cli cliDoc) linkHandler(name string) string {
return fmt.Sprintf("/cscli/%s", name)
}

236
cmd/crowdsec-cli/explain.go
View file

@ -2,6 +2,7 @@ package main
import (
"bufio"
"errors"
"fmt"
"io"
"os"
@ -11,78 +12,115 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/dumps"
"github.com/crowdsecurity/crowdsec/pkg/hubtest"
)
func GetLineCountForFile(filepath string) (int, error) {
func getLineCountForFile(filepath string) (int, error) {
f, err := os.Open(filepath)
if err != nil {
return 0, err
}
defer f.Close()
lc := 0
fs := bufio.NewReader(f)
for {
input, err := fs.ReadBytes('\n')
if len(input) > 1 {
lc++
}
if err != nil && err == io.EOF {
break
}
}
return lc, nil
}
func runExplain(cmd *cobra.Command, args []string) error {
type cliExplain struct {
cfg configGetter
flags struct {
logFile string
dsn string
logLine string
logType string
details bool
skipOk bool
onlySuccessfulParsers bool
noClean bool
crowdsec string
labels string
}
}
func NewCLIExplain(cfg configGetter) *cliExplain {
return &cliExplain{
cfg: cfg,
}
}
func (cli *cliExplain) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "explain",
Short: "Explain log pipeline",
Long: `
Explain log pipeline
`,
Example: `
cscli explain --file ./myfile.log --type nginx
cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog
cscli explain --dsn "file://myfile.log" --type nginx
tail -n 5 myfile.log | cscli explain --type nginx -f -
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.run()
},
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
fileInfo, _ := os.Stdin.Stat()
if cli.flags.logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) {
return fmt.Errorf("the option -f - is intended to work with pipes")
}
return nil
},
}
flags := cmd.Flags()
logFile, err := flags.GetString("file")
if err != nil {
return err
}
flags.StringVarP(&cli.flags.logFile, "file", "f", "", "Log file to test")
flags.StringVarP(&cli.flags.dsn, "dsn", "d", "", "DSN to test")
flags.StringVarP(&cli.flags.logLine, "log", "l", "", "Log line to test")
flags.StringVarP(&cli.flags.logType, "type", "t", "", "Type of the acquisition to test")
flags.StringVar(&cli.flags.labels, "labels", "", "Additional labels to add to the acquisition format (key:value,key2:value2)")
flags.BoolVarP(&cli.flags.details, "verbose", "v", false, "Display individual changes")
flags.BoolVar(&cli.flags.skipOk, "failures", false, "Only show failed lines")
flags.BoolVar(&cli.flags.onlySuccessfulParsers, "only-successful-parsers", false, "Only show successful parsers")
flags.StringVar(&cli.flags.crowdsec, "crowdsec", "crowdsec", "Path to crowdsec")
flags.BoolVar(&cli.flags.noClean, "no-clean", false, "Don't clean runtime environment after tests")
dsn, err := flags.GetString("dsn")
if err != nil {
return err
}
cmd.MarkFlagRequired("type")
cmd.MarkFlagsOneRequired("log", "file", "dsn")
logLine, err := flags.GetString("log")
if err != nil {
return err
}
return cmd
}
logType, err := flags.GetString("type")
if err != nil {
return err
}
func (cli *cliExplain) run() error {
logFile := cli.flags.logFile
logLine := cli.flags.logLine
logType := cli.flags.logType
dsn := cli.flags.dsn
labels := cli.flags.labels
crowdsec := cli.flags.crowdsec
opts := hubtest.DumpOpts{}
opts.Details, err = flags.GetBool("verbose")
if err != nil {
return err
}
opts.SkipOk, err = flags.GetBool("failures")
if err != nil {
return err
}
opts.ShowNotOkParsers, err = flags.GetBool("only-successful-parsers")
opts.ShowNotOkParsers = !opts.ShowNotOkParsers
if err != nil {
return err
}
crowdsec, err := flags.GetString("crowdsec")
if err != nil {
return err
}
labels, err := flags.GetString("labels")
if err != nil {
return err
opts := dumps.DumpOpts{
Details: cli.flags.details,
SkipOk: cli.flags.skipOk,
ShowNotOkParsers: !cli.flags.onlySuccessfulParsers,
}
var f *os.File
@ -90,19 +128,25 @@ func runExplain(cmd *cobra.Command, args []string) error {
// using empty string fallback to /tmp
dir, err := os.MkdirTemp("", "cscli_explain")
if err != nil {
return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %s", err)
return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %w", err)
}
defer func() {
if cli.flags.noClean {
return
}
if _, err := os.Stat(dir); !os.IsNotExist(err) {
if err := os.RemoveAll(dir); err != nil {
log.Errorf("unable to delete temporary directory '%s': %s", dir, err)
}
}
}()
tmpFile := ""
// we create a temporary log file if a log line/stdin has been provided
if logLine != "" || logFile == "-" {
tmpFile = filepath.Join(dir, "cscli_test_tmp.log")
tmpFile := filepath.Join(dir, "cscli_test_tmp.log")
f, err = os.Create(tmpFile)
if err != nil {
return err
@ -118,7 +162,7 @@ func runExplain(cmd *cobra.Command, args []string) error {
errCount := 0
for {
input, err := reader.ReadBytes('\n')
if err != nil && err == io.EOF {
if err != nil && errors.Is(err, io.EOF) {
break
}
if len(input) > 1 {
@ -132,6 +176,7 @@ func runExplain(cmd *cobra.Command, args []string) error {
log.Warnf("Failed to write %d lines to %s", errCount, tmpFile)
}
}
f.Close()
// this is the file that was going to be read by crowdsec anyway
logFile = tmpFile
@ -142,15 +187,20 @@ func runExplain(cmd *cobra.Command, args []string) error {
if err != nil {
return fmt.Errorf("unable to get absolute path of '%s', exiting", logFile)
}
dsn = fmt.Sprintf("file://%s", absolutePath)
lineCount, err := GetLineCountForFile(absolutePath)
lineCount, err := getLineCountForFile(absolutePath)
if err != nil {
return err
}
log.Debugf("file %s has %d lines", absolutePath, lineCount)
if lineCount == 0 {
return fmt.Errorf("the log file is empty: %s", absolutePath)
}
if lineCount > 100 {
log.Warnf("%s contains %d lines. This may take a lot of resources.", absolutePath, lineCount)
}
@ -161,103 +211,35 @@ func runExplain(cmd *cobra.Command, args []string) error {
}
cmdArgs := []string{"-c", ConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", dir, "-no-api"}
if labels != "" {
log.Debugf("adding labels %s", labels)
cmdArgs = append(cmdArgs, "-label", labels)
}
crowdsecCmd := exec.Command(crowdsec, cmdArgs...)
output, err := crowdsecCmd.CombinedOutput()
if err != nil {
fmt.Println(string(output))
return fmt.Errorf("fail to run crowdsec for test: %v", err)
return fmt.Errorf("fail to run crowdsec for test: %w", err)
}
parserDumpFile := filepath.Join(dir, hubtest.ParserResultFileName)
bucketStateDumpFile := filepath.Join(dir, hubtest.BucketPourResultFileName)
parserDump, err := hubtest.LoadParserDump(parserDumpFile)
parserDump, err := dumps.LoadParserDump(parserDumpFile)
if err != nil {
return fmt.Errorf("unable to load parser dump result: %s", err)
return fmt.Errorf("unable to load parser dump result: %w", err)
}
bucketStateDump, err := hubtest.LoadBucketPourDump(bucketStateDumpFile)
bucketStateDump, err := dumps.LoadBucketPourDump(bucketStateDumpFile)
if err != nil {
return fmt.Errorf("unable to load bucket dump result: %s", err)
return fmt.Errorf("unable to load bucket dump result: %w", err)
}
hubtest.DumpTree(*parserDump, *bucketStateDump, opts)
dumps.DumpTree(*parserDump, *bucketStateDump, opts)
return nil
}
func NewExplainCmd() *cobra.Command {
cmdExplain := &cobra.Command{
Use: "explain",
Short: "Explain log pipeline",
Long: `
Explain log pipeline
`,
Example: `
cscli explain --file ./myfile.log --type nginx
cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog
cscli explain --dsn "file://myfile.log" --type nginx
tail -n 5 myfile.log | cscli explain --type nginx -f -
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: runExplain,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
logFile, err := flags.GetString("file")
if err != nil {
return err
}
dsn, err := flags.GetString("dsn")
if err != nil {
return err
}
logLine, err := flags.GetString("log")
if err != nil {
return err
}
logType, err := flags.GetString("type")
if err != nil {
return err
}
if logLine == "" && logFile == "" && dsn == "" {
printHelp(cmd)
fmt.Println()
return fmt.Errorf("please provide --log, --file or --dsn flag")
}
if logType == "" {
printHelp(cmd)
fmt.Println()
return fmt.Errorf("please provide --type flag")
}
fileInfo, _ := os.Stdin.Stat()
if logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) {
return fmt.Errorf("the option -f - is intended to work with pipes")
}
return nil
},
}
flags := cmdExplain.Flags()
flags.StringP("file", "f", "", "Log file to test")
flags.StringP("dsn", "d", "", "DSN to test")
flags.StringP("log", "l", "", "Log line to test")
flags.StringP("type", "t", "", "Type of the acquisition to test")
flags.String("labels", "", "Additional labels to add to the acquisition format (key:value,key2:value2)")
flags.BoolP("verbose", "v", false, "Display individual changes")
flags.Bool("failures", false, "Only show failed lines")
flags.Bool("only-successful-parsers", false, "Only show successful parsers")
flags.String("crowdsec", "crowdsec", "Path to crowdsec")
return cmdExplain
}

29
cmd/crowdsec-cli/flag.go Normal file
View file

@ -0,0 +1,29 @@
package main
// Custom types for flag validation and conversion.
import (
"errors"
)
type MachinePassword string
func (p *MachinePassword) String() string {
return string(*p)
}
func (p *MachinePassword) Set(v string) error {
// a password can't be more than 72 characters
// due to bcrypt limitations
if len(v) > 72 {
return errors.New("password too long (max 72 characters)")
}
*p = MachinePassword(v)
return nil
}
func (p *MachinePassword) Type() string {
return "string"
}

116
cmd/crowdsec-cli/hub.go
View file

@ -13,8 +13,18 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewHubCmd() *cobra.Command {
cmdHub := &cobra.Command{
type cliHub struct {
cfg configGetter
}
func NewCLIHub(cfg configGetter) *cliHub {
return &cliHub{
cfg: cfg,
}
}
func (cli *cliHub) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "hub [action]",
Short: "Manage hub index",
Long: `Hub management
@ -28,23 +38,16 @@ cscli hub upgrade`,
DisableAutoGenTag: true,
}
cmdHub.AddCommand(NewHubListCmd())
cmdHub.AddCommand(NewHubUpdateCmd())
cmdHub.AddCommand(NewHubUpgradeCmd())
cmdHub.AddCommand(NewHubTypesCmd())
cmd.AddCommand(cli.newListCmd())
cmd.AddCommand(cli.newUpdateCmd())
cmd.AddCommand(cli.newUpgradeCmd())
cmd.AddCommand(cli.newTypesCmd())
return cmdHub
return cmd
}
func runHubList(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
all, err := flags.GetBool("all")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, nil)
func (cli *cliHub) list(all bool) error {
hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger())
if err != nil {
return err
}
@ -74,27 +77,31 @@ func runHubList(cmd *cobra.Command, args []string) error {
return nil
}
func NewHubListCmd() *cobra.Command {
cmdHubList := &cobra.Command{
func (cli *cliHub) newListCmd() *cobra.Command {
var all bool
cmd := &cobra.Command{
Use: "list [-a]",
Short: "List all installed configurations",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: runHubList,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.list(all)
},
}
flags := cmdHubList.Flags()
flags.BoolP("all", "a", false, "List disabled items as well")
flags := cmd.Flags()
flags.BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmdHubList
return cmd
}
func runHubUpdate(cmd *cobra.Command, args []string) error {
local := csConfig.Hub
remote := require.RemoteHub(csConfig)
func (cli *cliHub) update() error {
local := cli.cfg().Hub
remote := require.RemoteHub(cli.cfg())
// don't use require.Hub because if there is no index file, it would fail
hub, err := cwhub.NewHub(local, remote, true)
hub, err := cwhub.NewHub(local, remote, true, log.StandardLogger())
if err != nil {
return fmt.Errorf("failed to update hub: %w", err)
}
@ -106,8 +113,8 @@ func runHubUpdate(cmd *cobra.Command, args []string) error {
return nil
}
func NewHubUpdateCmd() *cobra.Command {
cmdHubUpdate := &cobra.Command{
func (cli *cliHub) newUpdateCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "update",
Short: "Download the latest index (catalog of available configurations)",
Long: `
@ -115,21 +122,16 @@ Fetches the .index.json file from the hub, containing the list of available conf
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: runHubUpdate,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.update()
},
}
return cmdHubUpdate
return cmd
}
func runHubUpgrade(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
force, err := flags.GetBool("force")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
func (cli *cliHub) upgrade(force bool) error {
hub, err := require.Hub(cli.cfg(), require.RemoteHub(cli.cfg()), log.StandardLogger())
if err != nil {
return err
}
@ -143,23 +145,28 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error {
updated := 0
log.Infof("Upgrading %s", itemType)
for _, item := range items {
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
updated++
}
}
log.Infof("Upgraded %d %s", updated, itemType)
}
return nil
}
func NewHubUpgradeCmd() *cobra.Command {
cmdHubUpgrade := &cobra.Command{
func (cli *cliHub) newUpgradeCmd() *cobra.Command {
var force bool
cmd := &cobra.Command{
Use: "upgrade",
Short: "Upgrade all configurations to their latest version",
Long: `
@ -167,39 +174,44 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: runHubUpgrade,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.upgrade(force)
},
}
flags := cmdHubUpgrade.Flags()
flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
flags := cmd.Flags()
flags.BoolVar(&force, "force", false, "Force upgrade: overwrite tainted and outdated files")
return cmdHubUpgrade
return cmd
}
func runHubTypes(cmd *cobra.Command, args []string) error {
switch csConfig.Cscli.Output {
func (cli *cliHub) types() error {
switch cli.cfg().Cscli.Output {
case "human":
s, err := yaml.Marshal(cwhub.ItemTypes)
if err != nil {
return err
}
fmt.Print(string(s))
case "json":
jsonStr, err := json.Marshal(cwhub.ItemTypes)
if err != nil {
return err
}
fmt.Println(string(jsonStr))
case "raw":
for _, itemType := range cwhub.ItemTypes {
fmt.Println(itemType)
}
}
return nil
}
func NewHubTypesCmd() *cobra.Command {
cmdHubTypes := &cobra.Command{
func (cli *cliHub) newTypesCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "types",
Short: "List supported item types",
Long: `
@ -207,8 +219,10 @@ List the types of supported hub items.
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: runHubTypes,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.types()
},
}
return cmdHubTypes
return cmd
}

121
cmd/crowdsec-cli/hubappsec.go Normal file
View file

@ -0,0 +1,121 @@
package main
import (
"fmt"
"os"
"golang.org/x/text/cases"
"golang.org/x/text/language"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/pkg/appsec"
"github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIAppsecConfig() *cliItem {
return &cliItem{
name: cwhub.APPSEC_CONFIGS,
singular: "appsec-config",
oneOrMore: "appsec-config(s)",
help: cliHelp{
example: `cscli appsec-configs list -a
cscli appsec-configs install crowdsecurity/vpatch
cscli appsec-configs inspect crowdsecurity/vpatch
cscli appsec-configs upgrade crowdsecurity/vpatch
cscli appsec-configs remove crowdsecurity/vpatch
`,
},
installHelp: cliHelp{
example: `cscli appsec-configs install crowdsecurity/vpatch`,
},
removeHelp: cliHelp{
example: `cscli appsec-configs remove crowdsecurity/vpatch`,
},
upgradeHelp: cliHelp{
example: `cscli appsec-configs upgrade crowdsecurity/vpatch`,
},
inspectHelp: cliHelp{
example: `cscli appsec-configs inspect crowdsecurity/vpatch`,
},
listHelp: cliHelp{
example: `cscli appsec-configs list
cscli appsec-configs list -a
cscli appsec-configs list crowdsecurity/vpatch`,
},
}
}
func NewCLIAppsecRule() *cliItem {
inspectDetail := func(item *cwhub.Item) error {
// Only show the converted rules in human mode
if csConfig.Cscli.Output != "human" {
return nil
}
appsecRule := appsec.AppsecCollectionConfig{}
yamlContent, err := os.ReadFile(item.State.LocalPath)
if err != nil {
return fmt.Errorf("unable to read file %s : %s", item.State.LocalPath, err)
}
if err := yaml.Unmarshal(yamlContent, &appsecRule); err != nil {
return fmt.Errorf("unable to unmarshal yaml file %s : %s", item.State.LocalPath, err)
}
for _, ruleType := range appsec_rule.SupportedTypes() {
fmt.Printf("\n%s format:\n", cases.Title(language.Und, cases.NoLower).String(ruleType))
for _, rule := range appsecRule.Rules {
convertedRule, _, err := rule.Convert(ruleType, appsecRule.Name)
if err != nil {
return fmt.Errorf("unable to convert rule %s : %s", rule.Name, err)
}
fmt.Println(convertedRule)
}
switch ruleType { //nolint:gocritic
case appsec_rule.ModsecurityRuleType:
for _, rule := range appsecRule.SecLangRules {
fmt.Println(rule)
}
}
}
return nil
}
return &cliItem{
name: "appsec-rules",
singular: "appsec-rule",
oneOrMore: "appsec-rule(s)",
help: cliHelp{
example: `cscli appsec-rules list -a
cscli appsec-rules install crowdsecurity/crs
cscli appsec-rules inspect crowdsecurity/crs
cscli appsec-rules upgrade crowdsecurity/crs
cscli appsec-rules remove crowdsecurity/crs
`,
},
installHelp: cliHelp{
example: `cscli appsec-rules install crowdsecurity/crs`,
},
removeHelp: cliHelp{
example: `cscli appsec-rules remove crowdsecurity/crs`,
},
upgradeHelp: cliHelp{
example: `cscli appsec-rules upgrade crowdsecurity/crs`,
},
inspectHelp: cliHelp{
example: `cscli appsec-rules inspect crowdsecurity/crs`,
},
inspectDetail: inspectDetail,
listHelp: cliHelp{
example: `cscli appsec-rules list
cscli appsec-rules list -a
cscli appsec-rules list crowdsecurity/crs`,
},
}
}

40
cmd/crowdsec-cli/hubcollection.go Normal file
View file

@ -0,0 +1,40 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLICollection() *cliItem {
return &cliItem{
name: cwhub.COLLECTIONS,
singular: "collection",
oneOrMore: "collection(s)",
help: cliHelp{
example: `cscli collections list -a
cscli collections install crowdsecurity/http-cve crowdsecurity/iptables
cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables
cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables
cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
`,
},
installHelp: cliHelp{
example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`,
},
removeHelp: cliHelp{
example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`,
},
upgradeHelp: cliHelp{
example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`,
},
inspectHelp: cliHelp{
example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`,
},
listHelp: cliHelp{
example: `cscli collections list
cscli collections list -a
cscli collections list crowdsecurity/http-cve crowdsecurity/iptables
List only enabled collections unless "-a" or names are specified.`,
},
}
}

40
cmd/crowdsec-cli/hubcontext.go Normal file
View file

@ -0,0 +1,40 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIContext() *cliItem {
return &cliItem{
name: cwhub.CONTEXTS,
singular: "context",
oneOrMore: "context(s)",
help: cliHelp{
example: `cscli contexts list -a
cscli contexts install crowdsecurity/yyy crowdsecurity/zzz
cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz
cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz
cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz
`,
},
installHelp: cliHelp{
example: `cscli contexts install crowdsecurity/yyy crowdsecurity/zzz`,
},
removeHelp: cliHelp{
example: `cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz`,
},
upgradeHelp: cliHelp{
example: `cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz`,
},
inspectHelp: cliHelp{
example: `cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz`,
},
listHelp: cliHelp{
example: `cscli contexts list
cscli contexts list -a
cscli contexts list crowdsecurity/yyy crowdsecurity/zzz
List only enabled contexts unless "-a" or names are specified.`,
},
}
}

40
cmd/crowdsec-cli/hubparser.go Normal file
View file

@ -0,0 +1,40 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIParser() *cliItem {
return &cliItem{
name: cwhub.PARSERS,
singular: "parser",
oneOrMore: "parser(s)",
help: cliHelp{
example: `cscli parsers list -a
cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
`,
},
installHelp: cliHelp{
example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
removeHelp: cliHelp{
example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
upgradeHelp: cliHelp{
example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
inspectHelp: cliHelp{
example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`,
},
listHelp: cliHelp{
example: `cscli parsers list
cscli parsers list -a
cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs
List only enabled parsers unless "-a" or names are specified.`,
},
}
}

40
cmd/crowdsec-cli/hubpostoverflow.go Normal file
View file

@ -0,0 +1,40 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIPostOverflow() *cliItem {
return &cliItem{
name: cwhub.POSTOVERFLOWS,
singular: "postoverflow",
oneOrMore: "postoverflow(s)",
help: cliHelp{
example: `cscli postoverflows list -a
cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
`,
},
installHelp: cliHelp{
example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
removeHelp: cliHelp{
example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
upgradeHelp: cliHelp{
example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
inspectHelp: cliHelp{
example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
listHelp: cliHelp{
example: `cscli postoverflows list
cscli postoverflows list -a
cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns
List only enabled postoverflows unless "-a" or names are specified.`,
},
}
}

40
cmd/crowdsec-cli/hubscenario.go Normal file
View file

@ -0,0 +1,40 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIScenario() *cliItem {
return &cliItem{
name: cwhub.SCENARIOS,
singular: "scenario",
oneOrMore: "scenario(s)",
help: cliHelp{
example: `cscli scenarios list -a
cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
`,
},
installHelp: cliHelp{
example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
removeHelp: cliHelp{
example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
upgradeHelp: cliHelp{
example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
inspectHelp: cliHelp{
example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
listHelp: cliHelp{
example: `cscli scenarios list
cscli scenarios list -a
cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing
List only enabled scenarios unless "-a" or names are specified.`,
},
}
}

363
cmd/crowdsec-cli/hubtest.go
View file

@ -7,6 +7,7 @@ import (
"os"
"path/filepath"
"strings"
"text/template"
"github.com/AlecAivazis/survey/v2"
"github.com/enescakir/emoji"
@ -15,50 +16,70 @@ import (
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/dumps"
"github.com/crowdsecurity/crowdsec/pkg/hubtest"
)
var HubTest hubtest.HubTest
var HubAppsecTests hubtest.HubTest
var hubPtr *hubtest.HubTest
var isAppsecTest bool
func NewHubTestCmd() *cobra.Command {
type cliHubTest struct{}
func NewCLIHubTest() *cliHubTest {
return &cliHubTest{}
}
func (cli cliHubTest) NewCommand() *cobra.Command {
var hubPath string
var crowdsecPath string
var cscliPath string
var cmdHubTest = &cobra.Command{
cmd := &cobra.Command{
Use: "hubtest",
Short: "Run functional tests on hub configurations",
Long: "Run functional tests on hub configurations (parsers, scenarios, collections...)",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
var err error
HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath)
HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, false)
if err != nil {
return fmt.Errorf("unable to load hubtest: %+v", err)
}
HubAppsecTests, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, true)
if err != nil {
return fmt.Errorf("unable to load appsec specific hubtest: %+v", err)
}
/*commands will use the hubPtr, will point to the default hubTest object, or the one dedicated to appsec tests*/
hubPtr = &HubTest
if isAppsecTest {
hubPtr = &HubAppsecTests
}
return nil
},
}
cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
cmd.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
cmd.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
cmd.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
cmd.PersistentFlags().BoolVar(&isAppsecTest, "appsec", false, "Command relates to appsec tests")
cmdHubTest.AddCommand(NewHubTestCreateCmd())
cmdHubTest.AddCommand(NewHubTestRunCmd())
cmdHubTest.AddCommand(NewHubTestCleanCmd())
cmdHubTest.AddCommand(NewHubTestInfoCmd())
cmdHubTest.AddCommand(NewHubTestListCmd())
cmdHubTest.AddCommand(NewHubTestCoverageCmd())
cmdHubTest.AddCommand(NewHubTestEvalCmd())
cmdHubTest.AddCommand(NewHubTestExplainCmd())
cmd.AddCommand(cli.NewCreateCmd())
cmd.AddCommand(cli.NewRunCmd())
cmd.AddCommand(cli.NewCleanCmd())
cmd.AddCommand(cli.NewInfoCmd())
cmd.AddCommand(cli.NewListCmd())
cmd.AddCommand(cli.NewCoverageCmd())
cmd.AddCommand(cli.NewEvalCmd())
cmd.AddCommand(cli.NewExplainCmd())
return cmdHubTest
return cmd
}
func NewHubTestCreateCmd() *cobra.Command {
func (cli cliHubTest) NewCreateCmd() *cobra.Command {
parsers := []string{}
postoverflows := []string{}
scenarios := []string{}
@ -66,7 +87,7 @@ func NewHubTestCreateCmd() *cobra.Command {
var labels map[string]string
var logType string
var cmdHubTestCreate = &cobra.Command{
cmd := &cobra.Command{
Use: "create",
Short: "create [test_name]",
Example: `cscli hubtest create my-awesome-test --type syslog
@ -74,13 +95,17 @@ cscli hubtest create my-nginx-custom-test --type nginx
cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios crowdsecurity/http-probing`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, args []string) error {
testName := args[0]
testPath := filepath.Join(HubTest.HubTestPath, testName)
testPath := filepath.Join(hubPtr.HubTestPath, testName)
if _, err := os.Stat(testPath); os.IsExist(err) {
return fmt.Errorf("test '%s' already exists in '%s', exiting", testName, testPath)
}
if isAppsecTest {
logType = "appsec"
}
if logType == "" {
return fmt.Errorf("please provide a type (--type) for the test")
}
@ -89,53 +114,83 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
return fmt.Errorf("unable to create folder '%s': %+v", testPath, err)
}
// create empty log file
logFileName := fmt.Sprintf("%s.log", testName)
logFilePath := filepath.Join(testPath, logFileName)
logFile, err := os.Create(logFilePath)
if err != nil {
return err
}
logFile.Close()
// create empty parser assertion file
parserAssertFilePath := filepath.Join(testPath, hubtest.ParserAssertFileName)
parserAssertFile, err := os.Create(parserAssertFilePath)
if err != nil {
return err
}
parserAssertFile.Close()
// create empty scenario assertion file
scenarioAssertFilePath := filepath.Join(testPath, hubtest.ScenarioAssertFileName)
scenarioAssertFile, err := os.Create(scenarioAssertFilePath)
if err != nil {
return err
}
scenarioAssertFile.Close()
parsers = append(parsers, "crowdsecurity/syslog-logs")
parsers = append(parsers, "crowdsecurity/dateparse-enrich")
if len(scenarios) == 0 {
scenarios = append(scenarios, "")
}
if len(postoverflows) == 0 {
postoverflows = append(postoverflows, "")
}
configFileData := &hubtest.HubTestItemConfig{
Parsers: parsers,
Scenarios: scenarios,
PostOVerflows: postoverflows,
LogFile: logFileName,
LogType: logType,
IgnoreParsers: ignoreParsers,
Labels: labels,
}
configFilePath := filepath.Join(testPath, "config.yaml")
configFileData := &hubtest.HubTestItemConfig{}
if logType == "appsec" {
//create empty nuclei template file
nucleiFileName := fmt.Sprintf("%s.yaml", testName)
nucleiFilePath := filepath.Join(testPath, nucleiFileName)
nucleiFile, err := os.OpenFile(nucleiFilePath, os.O_RDWR|os.O_CREATE, 0755)
if err != nil {
return err
}
ntpl := template.Must(template.New("nuclei").Parse(hubtest.TemplateNucleiFile))
if ntpl == nil {
return fmt.Errorf("unable to parse nuclei template")
}
ntpl.ExecuteTemplate(nucleiFile, "nuclei", struct{ TestName string }{TestName: testName})
nucleiFile.Close()
configFileData.AppsecRules = []string{"./appsec-rules/<author>/your_rule_here.yaml"}
configFileData.NucleiTemplate = nucleiFileName
fmt.Println()
fmt.Printf(" Test name : %s\n", testName)
fmt.Printf(" Test path : %s\n", testPath)
fmt.Printf(" Config File : %s\n", configFilePath)
fmt.Printf(" Nuclei Template : %s\n", nucleiFilePath)
} else {
// create empty log file
logFileName := fmt.Sprintf("%s.log", testName)
logFilePath := filepath.Join(testPath, logFileName)
logFile, err := os.Create(logFilePath)
if err != nil {
return err
}
logFile.Close()
// create empty parser assertion file
parserAssertFilePath := filepath.Join(testPath, hubtest.ParserAssertFileName)
parserAssertFile, err := os.Create(parserAssertFilePath)
if err != nil {
return err
}
parserAssertFile.Close()
// create empty scenario assertion file
scenarioAssertFilePath := filepath.Join(testPath, hubtest.ScenarioAssertFileName)
scenarioAssertFile, err := os.Create(scenarioAssertFilePath)
if err != nil {
return err
}
scenarioAssertFile.Close()
parsers = append(parsers, "crowdsecurity/syslog-logs")
parsers = append(parsers, "crowdsecurity/dateparse-enrich")
if len(scenarios) == 0 {
scenarios = append(scenarios, "")
}
if len(postoverflows) == 0 {
postoverflows = append(postoverflows, "")
}
configFileData.Parsers = parsers
configFileData.Scenarios = scenarios
configFileData.PostOverflows = postoverflows
configFileData.LogFile = logFileName
configFileData.LogType = logType
configFileData.IgnoreParsers = ignoreParsers
configFileData.Labels = labels
fmt.Println()
fmt.Printf(" Test name : %s\n", testName)
fmt.Printf(" Test path : %s\n", testPath)
fmt.Printf(" Log file : %s (please fill it with logs)\n", logFilePath)
fmt.Printf(" Parser assertion file : %s (please fill it with assertion)\n", parserAssertFilePath)
fmt.Printf(" Scenario assertion file : %s (please fill it with assertion)\n", scenarioAssertFilePath)
fmt.Printf(" Configuration File : %s (please fill it with parsers, scenarios...)\n", configFilePath)
}
fd, err := os.Create(configFilePath)
if err != nil {
return fmt.Errorf("open: %s", err)
@ -151,33 +206,26 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
if err := fd.Close(); err != nil {
return fmt.Errorf("close: %s", err)
}
fmt.Println()
fmt.Printf(" Test name : %s\n", testName)
fmt.Printf(" Test path : %s\n", testPath)
fmt.Printf(" Log file : %s (please fill it with logs)\n", logFilePath)
fmt.Printf(" Parser assertion file : %s (please fill it with assertion)\n", parserAssertFilePath)
fmt.Printf(" Scenario assertion file : %s (please fill it with assertion)\n", scenarioAssertFilePath)
fmt.Printf(" Configuration File : %s (please fill it with parsers, scenarios...)\n", configFilePath)
return nil
},
}
cmdHubTestCreate.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test")
cmdHubTestCreate.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test")
cmdHubTestCreate.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test")
cmdHubTestCreate.Flags().StringSliceVarP(&scenarios, "scenarios", "s", scenarios, "Scenarios to add to test")
cmdHubTestCreate.PersistentFlags().BoolVar(&ignoreParsers, "ignore-parsers", false, "Don't run test on parsers")
cmd.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test")
cmd.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test")
cmd.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test")
cmd.Flags().StringSliceVarP(&scenarios, "scenarios", "s", scenarios, "Scenarios to add to test")
cmd.PersistentFlags().BoolVar(&ignoreParsers, "ignore-parsers", false, "Don't run test on parsers")
return cmdHubTestCreate
return cmd
}
func NewHubTestRunCmd() *cobra.Command {
func (cli cliHubTest) NewRunCmd() *cobra.Command {
var noClean bool
var runAll bool
var forceClean bool
var cmdHubTestRun = &cobra.Command{
var NucleiTargetHost string
var AppSecHost string
var cmd = &cobra.Command{
Use: "run",
Short: "run [test_name]",
DisableAutoGenTag: true,
@ -186,14 +234,15 @@ func NewHubTestRunCmd() *cobra.Command {
printHelp(cmd)
return fmt.Errorf("please provide test to run or --all flag")
}
hubPtr.NucleiTargetHost = NucleiTargetHost
hubPtr.AppSecHost = AppSecHost
if runAll {
if err := HubTest.LoadAllTests(); err != nil {
if err := hubPtr.LoadAllTests(); err != nil {
return fmt.Errorf("unable to load all tests: %+v", err)
}
} else {
for _, testName := range args {
_, err := HubTest.LoadTestItem(testName)
_, err := hubPtr.LoadTestItem(testName)
if err != nil {
return fmt.Errorf("unable to load test '%s': %s", testName, err)
}
@ -202,8 +251,7 @@ func NewHubTestRunCmd() *cobra.Command {
// set timezone to avoid DST issues
os.Setenv("TZ", "UTC")
for _, test := range HubTest.Tests {
for _, test := range hubPtr.Tests {
if csConfig.Cscli.Output == "human" {
log.Infof("Running test '%s'", test.Name)
}
@ -215,11 +263,11 @@ func NewHubTestRunCmd() *cobra.Command {
return nil
},
PersistentPostRunE: func(cmd *cobra.Command, args []string) error {
PersistentPostRunE: func(_ *cobra.Command, _ []string) error {
success := true
testResult := make(map[string]bool)
for _, test := range HubTest.Tests {
if test.AutoGen {
for _, test := range hubPtr.Tests {
if test.AutoGen && !isAppsecTest {
if test.ParserAssert.AutoGenAssert {
log.Warningf("Assert file '%s' is empty, generating assertion:", test.ParserAssert.File)
fmt.Println()
@ -326,22 +374,24 @@ func NewHubTestRunCmd() *cobra.Command {
},
}
cmdHubTestRun.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
cmdHubTestRun.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
cmdHubTestRun.Flags().BoolVar(&runAll, "all", false, "Run all tests")
cmd.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
cmd.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
cmd.Flags().StringVar(&NucleiTargetHost, "target", hubtest.DefaultNucleiTarget, "Target for AppSec Test")
cmd.Flags().StringVar(&AppSecHost, "host", hubtest.DefaultAppsecHost, "Address to expose AppSec for hubtest")
cmd.Flags().BoolVar(&runAll, "all", false, "Run all tests")
return cmdHubTestRun
return cmd
}
func NewHubTestCleanCmd() *cobra.Command {
var cmdHubTestClean = &cobra.Command{
func (cli cliHubTest) NewCleanCmd() *cobra.Command {
var cmd = &cobra.Command{
Use: "clean",
Short: "clean [test_name]",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, args []string) error {
for _, testName := range args {
test, err := HubTest.LoadTestItem(testName)
test, err := hubPtr.LoadTestItem(testName)
if err != nil {
return fmt.Errorf("unable to load test '%s': %s", testName, err)
}
@ -354,27 +404,32 @@ func NewHubTestCleanCmd() *cobra.Command {
},
}
return cmdHubTestClean
return cmd
}
func NewHubTestInfoCmd() *cobra.Command {
var cmdHubTestInfo = &cobra.Command{
func (cli cliHubTest) NewInfoCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "info",
Short: "info [test_name]",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, args []string) error {
for _, testName := range args {
test, err := HubTest.LoadTestItem(testName)
test, err := hubPtr.LoadTestItem(testName)
if err != nil {
return fmt.Errorf("unable to load test '%s': %s", testName, err)
}
fmt.Println()
fmt.Printf(" Test name : %s\n", test.Name)
fmt.Printf(" Test path : %s\n", test.Path)
fmt.Printf(" Log file : %s\n", filepath.Join(test.Path, test.Config.LogFile))
fmt.Printf(" Parser assertion file : %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName))
fmt.Printf(" Scenario assertion file : %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName))
if isAppsecTest {
fmt.Printf(" Nuclei Template : %s\n", test.Config.NucleiTemplate)
fmt.Printf(" Appsec Rules : %s\n", strings.Join(test.Config.AppsecRules, ", "))
} else {
fmt.Printf(" Log file : %s\n", filepath.Join(test.Path, test.Config.LogFile))
fmt.Printf(" Parser assertion file : %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName))
fmt.Printf(" Scenario assertion file : %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName))
}
fmt.Printf(" Configuration File : %s\n", filepath.Join(test.Path, "config.yaml"))
}
@ -382,24 +437,24 @@ func NewHubTestInfoCmd() *cobra.Command {
},
}
return cmdHubTestInfo
return cmd
}
func NewHubTestListCmd() *cobra.Command {
var cmdHubTestList = &cobra.Command{
func (cli cliHubTest) NewListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "list",
Short: "list",
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if err := HubTest.LoadAllTests(); err != nil {
RunE: func(_ *cobra.Command, _ []string) error {
if err := hubPtr.LoadAllTests(); err != nil {
return fmt.Errorf("unable to load all tests: %s", err)
}
switch csConfig.Cscli.Output {
case "human":
hubTestListTable(color.Output, HubTest.Tests)
hubTestListTable(color.Output, hubPtr.Tests)
case "json":
j, err := json.MarshalIndent(HubTest.Tests, " ", " ")
j, err := json.MarshalIndent(hubPtr.Tests, " ", " ")
if err != nil {
return err
}
@ -412,30 +467,34 @@ func NewHubTestListCmd() *cobra.Command {
},
}
return cmdHubTestList
return cmd
}
func NewHubTestCoverageCmd() *cobra.Command {
func (cli cliHubTest) NewCoverageCmd() *cobra.Command {
var showParserCov bool
var showScenarioCov bool
var showOnlyPercent bool
var showAppsecCov bool
var cmdHubTestCoverage = &cobra.Command{
cmd := &cobra.Command{
Use: "coverage",
Short: "coverage",
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
//for this one we explicitly don't do for appsec
if err := HubTest.LoadAllTests(); err != nil {
return fmt.Errorf("unable to load all tests: %+v", err)
}
var err error
scenarioCoverage := []hubtest.Coverage{}
parserCoverage := []hubtest.Coverage{}
appsecRuleCoverage := []hubtest.Coverage{}
scenarioCoveragePercent := 0
parserCoveragePercent := 0
appsecRuleCoveragePercent := 0
// if both are false (flag by default), show both
showAll := !showScenarioCov && !showParserCov
showAll := !showScenarioCov && !showParserCov && !showAppsecCov
if showParserCov || showAll {
parserCoverage, err = HubTest.GetParsersCoverage()
@ -467,13 +526,30 @@ func NewHubTestCoverageCmd() *cobra.Command {
scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100)))
}
if showAppsecCov || showAll {
appsecRuleCoverage, err = HubTest.GetAppsecCoverage()
if err != nil {
return fmt.Errorf("while getting scenario coverage: %s", err)
}
appsecRuleTested := 0
for _, test := range appsecRuleCoverage {
if test.TestsCount > 0 {
appsecRuleTested++
}
}
appsecRuleCoveragePercent = int(math.Round((float64(appsecRuleTested) / float64(len(appsecRuleCoverage)) * 100)))
}
if showOnlyPercent {
if showAll {
fmt.Printf("parsers=%d%%\nscenarios=%d%%", parserCoveragePercent, scenarioCoveragePercent)
fmt.Printf("parsers=%d%%\nscenarios=%d%%\nappsec_rules=%d%%", parserCoveragePercent, scenarioCoveragePercent, appsecRuleCoveragePercent)
} else if showParserCov {
fmt.Printf("parsers=%d%%", parserCoveragePercent)
} else if showScenarioCov {
fmt.Printf("scenarios=%d%%", scenarioCoveragePercent)
} else if showAppsecCov {
fmt.Printf("appsec_rules=%d%%", appsecRuleCoveragePercent)
}
os.Exit(0)
}
@ -487,6 +563,11 @@ func NewHubTestCoverageCmd() *cobra.Command {
if showScenarioCov || showAll {
hubTestScenarioCoverageTable(color.Output, scenarioCoverage)
}
if showAppsecCov || showAll {
hubTestAppsecRuleCoverageTable(color.Output, appsecRuleCoverage)
}
fmt.Println()
if showParserCov || showAll {
fmt.Printf("PARSERS : %d%% of coverage\n", parserCoveragePercent)
@ -494,6 +575,9 @@ func NewHubTestCoverageCmd() *cobra.Command {
if showScenarioCov || showAll {
fmt.Printf("SCENARIOS : %d%% of coverage\n", scenarioCoveragePercent)
}
if showAppsecCov || showAll {
fmt.Printf("APPSEC RULES : %d%% of coverage\n", appsecRuleCoveragePercent)
}
case "json":
dump, err := json.MarshalIndent(parserCoverage, "", " ")
if err != nil {
@ -505,6 +589,11 @@ func NewHubTestCoverageCmd() *cobra.Command {
return err
}
fmt.Printf("%s", dump)
dump, err = json.MarshalIndent(appsecRuleCoverage, "", " ")
if err != nil {
return err
}
fmt.Printf("%s", dump)
default:
return fmt.Errorf("only human/json output modes are supported")
}
@ -513,23 +602,25 @@ func NewHubTestCoverageCmd() *cobra.Command {
},
}
cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
cmd.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
cmd.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
cmd.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
cmd.PersistentFlags().BoolVar(&showAppsecCov, "appsec", false, "Show only appsec coverage")
return cmdHubTestCoverage
return cmd
}
func NewHubTestEvalCmd() *cobra.Command {
func (cli cliHubTest) NewEvalCmd() *cobra.Command {
var evalExpression string
var cmdHubTestEval = &cobra.Command{
cmd := &cobra.Command{
Use: "eval",
Short: "eval [test_name]",
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, args []string) error {
for _, testName := range args {
test, err := HubTest.LoadTestItem(testName)
test, err := hubPtr.LoadTestItem(testName)
if err != nil {
return fmt.Errorf("can't load test: %+v", err)
}
@ -551,18 +642,18 @@ func NewHubTestEvalCmd() *cobra.Command {
},
}
cmdHubTestEval.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval")
cmd.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval")
return cmdHubTestEval
return cmd
}
func NewHubTestExplainCmd() *cobra.Command {
var cmdHubTestExplain = &cobra.Command{
func (cli cliHubTest) NewExplainCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "explain",
Short: "explain [test_name]",
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, args []string) error {
for _, testName := range args {
test, err := HubTest.LoadTestItem(testName)
if err != nil {
@ -589,13 +680,13 @@ func NewHubTestExplainCmd() *cobra.Command {
return fmt.Errorf("unable to load scenario result after run: %s", err)
}
}
opts := hubtest.DumpOpts{}
hubtest.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData, opts)
opts := dumps.DumpOpts{}
dumps.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData, opts)
}
return nil
},
}
return cmdHubTestExplain
return cmd
}

20
cmd/crowdsec-cli/hubtest_table.go
View file

@ -61,6 +61,26 @@ func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
t.Render()
}
func hubTestAppsecRuleCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
t := newLightTable(out)
t.SetHeaders("Appsec Rule", "Status", "Number of tests")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
parserTested := 0
for _, test := range coverage {
status := emoji.RedCircle.String()
if test.TestsCount > 0 {
status = emoji.GreenCircle.String()
parserTested++
}
t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
}
t.Render()
}
func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
t := newLightTable(out)
t.SetHeaders("Scenario", "Status", "Number of tests")

65
cmd/crowdsec-cli/item_metrics.go
View file

@ -32,8 +32,10 @@ func ShowMetrics(hubItem *cwhub.Item) error {
return err
}
}
default:
// no metrics for this item type
case cwhub.APPSEC_RULES:
metrics := GetAppsecRuleMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
appsecMetricsTable(color.Output, hubItem.Name, metrics)
default: // no metrics for this item type
}
return nil
}
@ -175,6 +177,63 @@ func GetScenarioMetric(url string, itemName string) map[string]int {
return stats
}
func GetAppsecRuleMetric(url string, itemName string) map[string]int {
stats := make(map[string]int)
stats["inband_hits"] = 0
stats["outband_hits"] = 0
results := GetPrometheusMetric(url)
for idx, fam := range results {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["rule_name"]
if !ok {
log.Debugf("no rule_name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
band, ok := metric.Labels["type"]
if !ok {
log.Debugf("no type in Metric %v", metric.Labels)
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_appsec_rule_hits":
switch band {
case "inband":
stats["inband_hits"] += ival
case "outband":
stats["outband_hits"] += ival
default:
continue
}
default:
continue
}
}
}
return stats
}
func GetPrometheusMetric(url string) []*prom2json.Family {
mfChan := make(chan *dto.MetricFamily, 1024)
@ -220,6 +279,7 @@ var ranges = []unit{
func formatNumber(num int) string {
goodUnit := unit{}
for _, u := range ranges {
if int64(num) >= u.value {
goodUnit = u
@ -232,5 +292,6 @@ func formatNumber(num int) string {
}
res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
}

6
cmd/crowdsec-cli/item_suggest.go
View file

@ -2,11 +2,11 @@ package main
import (
"fmt"
"slices"
"strings"
"github.com/agext/levenshtein"
"github.com/spf13/cobra"
"slices"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
@ -37,7 +37,7 @@ func suggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) str
}
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
hub, err := require.Hub(csConfig, nil)
hub, err := require.Hub(csConfig, nil, nil)
if err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
@ -56,7 +56,7 @@ func compAllItems(itemType string, args []string, toComplete string) ([]string,
}
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
hub, err := require.Hub(csConfig, nil)
hub, err := require.Hub(csConfig, nil, nil)
if err != nil {
return nil, cobra.ShellCompDirectiveDefault
}

534
cmd/crowdsec-cli/itemcli.go Normal file
View file

@ -0,0 +1,534 @@
package main
import (
"fmt"
"os"
"strings"
"github.com/fatih/color"
"github.com/hexops/gotextdiff"
"github.com/hexops/gotextdiff/myers"
"github.com/hexops/gotextdiff/span"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/coalesce"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
type cliHelp struct {
// Example is required, the others have a default value
// generated from the item type
use string
short string
long string
example string
}
type cliItem struct {
name string // plural, as used in the hub index
singular string
oneOrMore string // parenthetical pluralizaion: "parser(s)"
help cliHelp
installHelp cliHelp
removeHelp cliHelp
upgradeHelp cliHelp
inspectHelp cliHelp
inspectDetail func(item *cwhub.Item) error
listHelp cliHelp
}
func (cli cliItem) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: coalesce.String(cli.help.use, fmt.Sprintf("%s <action> [item]...", cli.name)),
Short: coalesce.String(cli.help.short, fmt.Sprintf("Manage hub %s", cli.name)),
Long: cli.help.long,
Example: cli.help.example,
Args: cobra.MinimumNArgs(1),
Aliases: []string{cli.singular},
DisableAutoGenTag: true,
}
cmd.AddCommand(cli.newInstallCmd())
cmd.AddCommand(cli.newRemoveCmd())
cmd.AddCommand(cli.newUpgradeCmd())
cmd.AddCommand(cli.newInspectCmd())
cmd.AddCommand(cli.newListCmd())
return cmd
}
func (cli cliItem) install(args []string, downloadOnly bool, force bool, ignoreError bool) error {
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), log.StandardLogger())
if err != nil {
return err
}
for _, name := range args {
item := hub.GetItem(cli.name, name)
if item == nil {
msg := suggestNearestMessage(hub, cli.name, name)
if !ignoreError {
return fmt.Errorf(msg)
}
log.Errorf(msg)
continue
}
if err := item.Install(force, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", item.Name, err)
}
log.Errorf("Error while installing '%s': %s", item.Name, err)
}
}
log.Infof(ReloadMessage())
return nil
}
func (cli cliItem) newInstallCmd() *cobra.Command {
var (
downloadOnly bool
force bool
ignoreError bool
)
cmd := &cobra.Command{
Use: coalesce.String(cli.installHelp.use, "install [item]..."),
Short: coalesce.String(cli.installHelp.short, fmt.Sprintf("Install given %s", cli.oneOrMore)),
Long: coalesce.String(cli.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", cli.name)),
Example: cli.installHelp.example,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cli.name, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
return cli.install(args, downloadOnly, force, ignoreError)
},
}
flags := cmd.Flags()
flags.BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
flags.BoolVar(&force, "force", false, "Force install: overwrite tainted and outdated files")
flags.BoolVar(&ignoreError, "ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", cli.name))
return cmd
}
// return the names of the installed parents of an item, used to check if we can remove it
func istalledParentNames(item *cwhub.Item) []string {
ret := make([]string, 0)
for _, parent := range item.Ancestors() {
if parent.State.Installed {
ret = append(ret, parent.Name)
}
}
return ret
}
func (cli cliItem) remove(args []string, purge bool, force bool, all bool) error {
hub, err := require.Hub(csConfig, nil, log.StandardLogger())
if err != nil {
return err
}
if all {
getter := hub.GetInstalledItems
if purge {
getter = hub.GetAllItems
}
items, err := getter(cli.name)
if err != nil {
return err
}
removed := 0
for _, item := range items {
didRemove, err := item.Remove(purge, force)
if err != nil {
return err
}
if didRemove {
log.Infof("Removed %s", item.Name)
removed++
}
}
log.Infof("Removed %d %s", removed, cli.name)
if removed > 0 {
log.Infof(ReloadMessage())
}
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one %s to remove or '--all'", cli.singular)
}
removed := 0
for _, itemName := range args {
item := hub.GetItem(cli.name, itemName)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", itemName, cli.name)
}
parents := istalledParentNames(item)
if !force && len(parents) > 0 {
log.Warningf("%s belongs to collections: %s", item.Name, parents)
log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, cli.singular)
continue
}
didRemove, err := item.Remove(purge, force)
if err != nil {
return err
}
if didRemove {
log.Infof("Removed %s", item.Name)
removed++
}
}
log.Infof("Removed %d %s", removed, cli.name)
if removed > 0 {
log.Infof(ReloadMessage())
}
return nil
}
func (cli cliItem) newRemoveCmd() *cobra.Command {
var (
purge bool
force bool
all bool
)
cmd := &cobra.Command{
Use: coalesce.String(cli.removeHelp.use, "remove [item]..."),
Short: coalesce.String(cli.removeHelp.short, fmt.Sprintf("Remove given %s", cli.oneOrMore)),
Long: coalesce.String(cli.removeHelp.long, fmt.Sprintf("Remove one or more %s", cli.name)),
Example: cli.removeHelp.example,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cli.name, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
return cli.remove(args, purge, force, all)
},
}
flags := cmd.Flags()
flags.BoolVar(&purge, "purge", false, "Delete source file too")
flags.BoolVar(&force, "force", false, "Force remove: remove tainted and outdated files")
flags.BoolVar(&all, "all", false, fmt.Sprintf("Remove all the %s", cli.name))
return cmd
}
func (cli cliItem) upgrade(args []string, force bool, all bool) error {
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), log.StandardLogger())
if err != nil {
return err
}
if all {
items, err := hub.GetInstalledItems(cli.name)
if err != nil {
return err
}
updated := 0
for _, item := range items {
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
updated++
}
}
log.Infof("Updated %d %s", updated, cli.name)
if updated > 0 {
log.Infof(ReloadMessage())
}
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one %s to upgrade or '--all'", cli.singular)
}
updated := 0
for _, itemName := range args {
item := hub.GetItem(cli.name, itemName)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", itemName, cli.name)
}
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
log.Infof("Updated %s", item.Name)
updated++
}
}
if updated > 0 {
log.Infof(ReloadMessage())
}
return nil
}
func (cli cliItem) newUpgradeCmd() *cobra.Command {
var (
all bool
force bool
)
cmd := &cobra.Command{
Use: coalesce.String(cli.upgradeHelp.use, "upgrade [item]..."),
Short: coalesce.String(cli.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", cli.oneOrMore)),
Long: coalesce.String(cli.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", cli.name)),
Example: cli.upgradeHelp.example,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cli.name, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
return cli.upgrade(args, force, all)
},
}
flags := cmd.Flags()
flags.BoolVarP(&all, "all", "a", false, fmt.Sprintf("Upgrade all the %s", cli.name))
flags.BoolVar(&force, "force", false, "Force upgrade: overwrite tainted and outdated files")
return cmd
}
func (cli cliItem) inspect(args []string, url string, diff bool, rev bool, noMetrics bool) error {
if rev && !diff {
return fmt.Errorf("--rev can only be used with --diff")
}
if url != "" {
csConfig.Cscli.PrometheusUrl = url
}
remote := (*cwhub.RemoteHubCfg)(nil)
if diff {
remote = require.RemoteHub(csConfig)
}
hub, err := require.Hub(csConfig, remote, log.StandardLogger())
if err != nil {
return err
}
for _, name := range args {
item := hub.GetItem(cli.name, name)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", name, cli.name)
}
if diff {
fmt.Println(cli.whyTainted(hub, item, rev))
continue
}
if err = inspectItem(item, !noMetrics); err != nil {
return err
}
if cli.inspectDetail != nil {
if err = cli.inspectDetail(item); err != nil {
return err
}
}
}
return nil
}
func (cli cliItem) newInspectCmd() *cobra.Command {
var (
url string
diff bool
rev bool
noMetrics bool
)
cmd := &cobra.Command{
Use: coalesce.String(cli.inspectHelp.use, "inspect [item]..."),
Short: coalesce.String(cli.inspectHelp.short, fmt.Sprintf("Inspect given %s", cli.oneOrMore)),
Long: coalesce.String(cli.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", cli.name)),
Example: cli.inspectHelp.example,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cli.name, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
return cli.inspect(args, url, diff, rev, noMetrics)
},
}
flags := cmd.Flags()
flags.StringVarP(&url, "url", "u", "", "Prometheus url")
flags.BoolVar(&diff, "diff", false, "Show diff with latest version (for tainted items)")
flags.BoolVar(&rev, "rev", false, "Reverse diff output")
flags.BoolVar(&noMetrics, "no-metrics", false, "Don't show metrics (when cscli.output=human)")
return cmd
}
func (cli cliItem) list(args []string, all bool) error {
hub, err := require.Hub(csConfig, nil, log.StandardLogger())
if err != nil {
return err
}
items := make(map[string][]*cwhub.Item)
items[cli.name], err = selectItems(hub, cli.name, args, !all)
if err != nil {
return err
}
if err = listItems(color.Output, []string{cli.name}, items, false); err != nil {
return err
}
return nil
}
func (cli cliItem) newListCmd() *cobra.Command {
var all bool
cmd := &cobra.Command{
Use: coalesce.String(cli.listHelp.use, "list [item... | -a]"),
Short: coalesce.String(cli.listHelp.short, fmt.Sprintf("List %s", cli.oneOrMore)),
Long: coalesce.String(cli.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", cli.name)),
Example: cli.listHelp.example,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
return cli.list(args, all)
},
}
flags := cmd.Flags()
flags.BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmd
}
// return the diff between the installed version and the latest version
func (cli cliItem) itemDiff(item *cwhub.Item, reverse bool) (string, error) {
if !item.State.Installed {
return "", fmt.Errorf("'%s' is not installed", item.FQName())
}
latestContent, remoteURL, err := item.FetchLatest()
if err != nil {
return "", err
}
localContent, err := os.ReadFile(item.State.LocalPath)
if err != nil {
return "", fmt.Errorf("while reading %s: %w", item.State.LocalPath, err)
}
file1 := item.State.LocalPath
file2 := remoteURL
content1 := string(localContent)
content2 := string(latestContent)
if reverse {
file1, file2 = file2, file1
content1, content2 = content2, content1
}
edits := myers.ComputeEdits(span.URIFromPath(file1), content1, content2)
diff := gotextdiff.ToUnified(file1, file2, content1, edits)
return fmt.Sprintf("%s", diff), nil
}
func (cli cliItem) whyTainted(hub *cwhub.Hub, item *cwhub.Item, reverse bool) string {
if !item.State.Installed {
return fmt.Sprintf("# %s is not installed", item.FQName())
}
if !item.State.Tainted {
return fmt.Sprintf("# %s is not tainted", item.FQName())
}
if len(item.State.TaintedBy) == 0 {
return fmt.Sprintf("# %s is tainted but we don't know why. please report this as a bug", item.FQName())
}
ret := []string{
fmt.Sprintf("# Let's see why %s is tainted.", item.FQName()),
}
for _, fqsub := range item.State.TaintedBy {
ret = append(ret, fmt.Sprintf("\n-> %s\n", fqsub))
sub, err := hub.GetItemFQ(fqsub)
if err != nil {
ret = append(ret, err.Error())
}
diff, err := cli.itemDiff(sub, reverse)
if err != nil {
ret = append(ret, err.Error())
}
if diff != "" {
ret = append(ret, diff)
} else if len(sub.State.TaintedBy) > 0 {
taintList := strings.Join(sub.State.TaintedBy, ", ")
if sub.FQName() == taintList {
// hack: avoid message "item is tainted by itself"
continue
}
ret = append(ret, fmt.Sprintf("# %s is tainted by %s", sub.FQName(), taintList))
}
}
return strings.Join(ret, "\n")
}

609
cmd/crowdsec-cli/itemcommands.go
View file

@ -1,609 +0,0 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/coalesce"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
type cmdHelp struct {
// Example is required, the others have a default value
// generated from the item type
use string
short string
long string
example string
}
type hubItemType struct {
name string // plural, as used in the hub index
singular string
oneOrMore string // parenthetical pluralizaion: "parser(s)"
help cmdHelp
installHelp cmdHelp
removeHelp cmdHelp
upgradeHelp cmdHelp
inspectHelp cmdHelp
listHelp cmdHelp
}
var hubItemTypes = map[string]hubItemType{
"parsers": {
name: cwhub.PARSERS,
singular: "parser",
oneOrMore: "parser(s)",
help: cmdHelp{
example: `cscli parsers list -a
cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
`,
},
installHelp: cmdHelp{
example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
removeHelp: cmdHelp{
example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
upgradeHelp: cmdHelp{
example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
inspectHelp: cmdHelp{
example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`,
},
listHelp: cmdHelp{
example: `cscli parsers list
cscli parsers list -a
cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs
List only enabled parsers unless "-a" or names are specified.`,
},
},
"postoverflows": {
name: cwhub.POSTOVERFLOWS,
singular: "postoverflow",
oneOrMore: "postoverflow(s)",
help: cmdHelp{
example: `cscli postoverflows list -a
cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
`,
},
installHelp: cmdHelp{
example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
removeHelp: cmdHelp{
example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
upgradeHelp: cmdHelp{
example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
inspectHelp: cmdHelp{
example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
listHelp: cmdHelp{
example: `cscli postoverflows list
cscli postoverflows list -a
cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns
List only enabled postoverflows unless "-a" or names are specified.`,
},
},
"scenarios": {
name: cwhub.SCENARIOS,
singular: "scenario",
oneOrMore: "scenario(s)",
help: cmdHelp{
example: `cscli scenarios list -a
cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
`,
},
installHelp: cmdHelp{
example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
removeHelp: cmdHelp{
example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
upgradeHelp: cmdHelp{
example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
inspectHelp: cmdHelp{
example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
listHelp: cmdHelp{
example: `cscli scenarios list
cscli scenarios list -a
cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing
List only enabled scenarios unless "-a" or names are specified.`,
},
},
"collections": {
name: cwhub.COLLECTIONS,
singular: "collection",
oneOrMore: "collection(s)",
help: cmdHelp{
example: `cscli collections list -a
cscli collections install crowdsecurity/http-cve crowdsecurity/iptables
cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables
cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables
cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
`,
},
installHelp: cmdHelp{
example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`,
},
removeHelp: cmdHelp{
example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`,
},
upgradeHelp: cmdHelp{
example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`,
},
inspectHelp: cmdHelp{
example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`,
},
listHelp: cmdHelp{
example: `cscli collections list
cscli collections list -a
cscli collections list crowdsecurity/http-cve crowdsecurity/iptables
List only enabled collections unless "-a" or names are specified.`,
},
},
}
func NewItemsCmd(typeName string) *cobra.Command {
it := hubItemTypes[typeName]
cmd := &cobra.Command{
Use: coalesce.String(it.help.use, fmt.Sprintf("%s <action> [item]...", it.name)),
Short: coalesce.String(it.help.short, fmt.Sprintf("Manage hub %s", it.name)),
Long: it.help.long,
Example: it.help.example,
Args: cobra.MinimumNArgs(1),
Aliases: []string{it.singular},
DisableAutoGenTag: true,
}
cmd.AddCommand(NewItemsInstallCmd(typeName))
cmd.AddCommand(NewItemsRemoveCmd(typeName))
cmd.AddCommand(NewItemsUpgradeCmd(typeName))
cmd.AddCommand(NewItemsInspectCmd(typeName))
cmd.AddCommand(NewItemsListCmd(typeName))
return cmd
}
func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
run := func(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
downloadOnly, err := flags.GetBool("download-only")
if err != nil {
return err
}
force, err := flags.GetBool("force")
if err != nil {
return err
}
ignoreError, err := flags.GetBool("ignore")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
for _, name := range args {
item := hub.GetItem(it.name, name)
if item == nil {
msg := suggestNearestMessage(hub, it.name, name)
if !ignoreError {
return fmt.Errorf(msg)
}
log.Errorf(msg)
continue
}
if err := item.Install(force, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", item.Name, err)
}
log.Errorf("Error while installing '%s': %s", item.Name, err)
}
}
log.Infof(ReloadMessage())
return nil
}
return run
}
func NewItemsInstallCmd(typeName string) *cobra.Command {
it := hubItemTypes[typeName]
cmd := &cobra.Command{
Use: coalesce.String(it.installHelp.use, "install [item]..."),
Short: coalesce.String(it.installHelp.short, fmt.Sprintf("Install given %s", it.oneOrMore)),
Long: coalesce.String(it.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", it.name)),
Example: it.installHelp.example,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(typeName, args, toComplete)
},
RunE: itemsInstallRunner(it),
}
flags := cmd.Flags()
flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
flags.Bool("ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", it.name))
return cmd
}
// return the names of the installed parents of an item, used to check if we can remove it
func istalledParentNames(item *cwhub.Item) []string {
ret := make([]string, 0)
for _, parent := range item.Ancestors() {
if parent.State.Installed {
ret = append(ret, parent.Name)
}
}
return ret
}
func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
run := func(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
purge, err := flags.GetBool("purge")
if err != nil {
return err
}
force, err := flags.GetBool("force")
if err != nil {
return err
}
all, err := flags.GetBool("all")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
if all {
getter := hub.GetInstalledItems
if purge {
getter = hub.GetAllItems
}
items, err := getter(it.name)
if err != nil {
return err
}
removed := 0
for _, item := range items {
didRemove, err := item.Remove(purge, force)
if err != nil {
return err
}
if didRemove {
log.Infof("Removed %s", item.Name)
removed++
}
}
log.Infof("Removed %d %s", removed, it.name)
if removed > 0 {
log.Infof(ReloadMessage())
}
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular)
}
removed := 0
for _, itemName := range args {
item := hub.GetItem(it.name, itemName)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
}
parents := istalledParentNames(item)
if !force && len(parents) > 0 {
log.Warningf("%s belongs to collections: %s", item.Name, parents)
log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular)
continue
}
didRemove, err := item.Remove(purge, force)
if err != nil {
return err
}
if didRemove {
log.Infof("Removed %s", item.Name)
removed++
}
}
log.Infof("Removed %d %s", removed, it.name)
if removed > 0 {
log.Infof(ReloadMessage())
}
return nil
}
return run
}
func NewItemsRemoveCmd(typeName string) *cobra.Command {
it := hubItemTypes[typeName]
cmd := &cobra.Command{
Use: coalesce.String(it.removeHelp.use, "remove [item]..."),
Short: coalesce.String(it.removeHelp.short, fmt.Sprintf("Remove given %s", it.oneOrMore)),
Long: coalesce.String(it.removeHelp.long, fmt.Sprintf("Remove one or more %s", it.name)),
Example: it.removeHelp.example,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(it.name, args, toComplete)
},
RunE: itemsRemoveRunner(it),
}
flags := cmd.Flags()
flags.Bool("purge", false, "Delete source file too")
flags.Bool("force", false, "Force remove: remove tainted and outdated files")
flags.Bool("all", false, fmt.Sprintf("Remove all the %s", it.name))
return cmd
}
func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
run := func(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
force, err := flags.GetBool("force")
if err != nil {
return err
}
all, err := flags.GetBool("all")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
if all {
items, err := hub.GetInstalledItems(it.name)
if err != nil {
return err
}
updated := 0
for _, item := range items {
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
updated++
}
}
log.Infof("Updated %d %s", updated, it.name)
if updated > 0 {
log.Infof(ReloadMessage())
}
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular)
}
updated := 0
for _, itemName := range args {
item := hub.GetItem(it.name, itemName)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
}
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
log.Infof("Updated %s", item.Name)
updated++
}
}
if updated > 0 {
log.Infof(ReloadMessage())
}
return nil
}
return run
}
func NewItemsUpgradeCmd(typeName string) *cobra.Command {
it := hubItemTypes[typeName]
cmd := &cobra.Command{
Use: coalesce.String(it.upgradeHelp.use, "upgrade [item]..."),
Short: coalesce.String(it.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", it.oneOrMore)),
Long: coalesce.String(it.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", it.name)),
Example: it.upgradeHelp.example,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(it.name, args, toComplete)
},
RunE: itemsUpgradeRunner(it),
}
flags := cmd.Flags()
flags.BoolP("all", "a", false, fmt.Sprintf("Upgrade all the %s", it.name))
flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
return cmd
}
func itemsInspectRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
run := func(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
url, err := flags.GetString("url")
if err != nil {
return err
}
if url != "" {
csConfig.Cscli.PrometheusUrl = url
}
noMetrics, err := flags.GetBool("no-metrics")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
for _, name := range args {
item := hub.GetItem(it.name, name)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", name, it.name)
}
if err = InspectItem(item, !noMetrics); err != nil {
return err
}
}
return nil
}
return run
}
func NewItemsInspectCmd(typeName string) *cobra.Command {
it := hubItemTypes[typeName]
cmd := &cobra.Command{
Use: coalesce.String(it.inspectHelp.use, "inspect [item]..."),
Short: coalesce.String(it.inspectHelp.short, fmt.Sprintf("Inspect given %s", it.oneOrMore)),
Long: coalesce.String(it.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", it.name)),
Example: it.inspectHelp.example,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(it.name, args, toComplete)
},
RunE: itemsInspectRunner(it),
}
flags := cmd.Flags()
flags.StringP("url", "u", "", "Prometheus url")
flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
return cmd
}
func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) error {
run := func(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
all, err := flags.GetBool("all")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
items := make(map[string][]*cwhub.Item)
items[it.name], err = selectItems(hub, it.name, args, !all)
if err != nil {
return err
}
if err = listItems(color.Output, []string{it.name}, items, false); err != nil {
return err
}
return nil
}
return run
}
func NewItemsListCmd(typeName string) *cobra.Command {
it := hubItemTypes[typeName]
cmd := &cobra.Command{
Use: coalesce.String(it.listHelp.use, "list [item... | -a]"),
Short: coalesce.String(it.listHelp.short, fmt.Sprintf("List %s", it.oneOrMore)),
Long: coalesce.String(it.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", it.name)),
Example: it.listHelp.example,
DisableAutoGenTag: true,
RunE: itemsListRunner(it),
}
flags := cmd.Flags()
flags.BoolP("all", "a", false, "List disabled items as well")
return cmd
}

32
cmd/crowdsec-cli/items.go
View file

@ -6,10 +6,11 @@ import (
"fmt"
"io"
"os"
"path/filepath"
"slices"
"strings"
"gopkg.in/yaml.v3"
"slices"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
@ -57,13 +58,17 @@ func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item
switch csConfig.Cscli.Output {
case "human":
nothingToDisplay := true
for _, itemType := range itemTypes {
if omitIfEmpty && len(items[itemType]) == 0 {
continue
}
listHubItemTable(out, "\n"+strings.ToUpper(itemType), items[itemType])
nothingToDisplay = false
}
if nothingToDisplay {
fmt.Println("No items to display")
}
@ -84,14 +89,14 @@ func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item
for i, item := range items[itemType] {
status := item.State.Text()
status_emo := item.State.Emoji()
statusEmo := item.State.Emoji()
hubStatus[itemType][i] = itemHubStatus{
Name: item.Name,
LocalVersion: item.State.LocalVersion,
LocalPath: item.State.LocalPath,
Description: item.Description,
Status: status,
UTF8Status: fmt.Sprintf("%v %s", status_emo, status),
UTF8Status: fmt.Sprintf("%v %s", statusEmo, status),
}
}
}
@ -125,24 +130,25 @@ func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item
if len(itemTypes) > 1 {
row = append(row, itemType)
}
if err := csvwriter.Write(row); err != nil {
return fmt.Errorf("failed to write raw output: %s", err)
}
}
}
csvwriter.Flush()
default:
return fmt.Errorf("unknown output format '%s'", csConfig.Cscli.Output)
}
return nil
}
func InspectItem(item *cwhub.Item, showMetrics bool) error {
func inspectItem(item *cwhub.Item, showMetrics bool) error {
switch csConfig.Cscli.Output {
case "human", "raw":
enc := yaml.NewEncoder(os.Stdout)
enc.SetIndent(2)
if err := enc.Encode(item); err != nil {
return fmt.Errorf("unable to encode item: %s", err)
}
@ -151,11 +157,23 @@ func InspectItem(item *cwhub.Item, showMetrics bool) error {
if err != nil {
return fmt.Errorf("unable to marshal item: %s", err)
}
fmt.Print(string(b))
}
if csConfig.Cscli.Output == "human" && showMetrics {
if csConfig.Cscli.Output != "human" {
return nil
}
if item.State.Tainted {
fmt.Println()
fmt.Printf(`This item is tainted. Use "%s %s inspect --diff %s" to see why.`, filepath.Base(os.Args[0]), item.Type, item.Name)
fmt.Println()
}
if showMetrics {
fmt.Printf("\nCurrent metrics: \n")
if err := ShowMetrics(item); err != nil {
return err
}

408
cmd/crowdsec-cli/lapi.go
View file

@ -2,6 +2,7 @@ package main
import (
"context"
"errors"
"fmt"
"net/url"
"os"
@ -26,26 +27,36 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/parser"
)
var LAPIURLPrefix string = "v1"
const LAPIURLPrefix = "v1"
func runLapiStatus(cmd *cobra.Command, args []string) error {
var err error
type cliLapi struct {
cfg configGetter
}
password := strfmt.Password(csConfig.API.Client.Credentials.Password)
apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL)
login := csConfig.API.Client.Credentials.Login
func NewCLILapi(cfg configGetter) *cliLapi {
return &cliLapi{
cfg: cfg,
}
}
func (cli *cliLapi) status() error {
cfg := cli.cfg()
password := strfmt.Password(cfg.API.Client.Credentials.Password)
login := cfg.API.Client.Credentials.Login
apiurl, err := url.Parse(cfg.API.Client.Credentials.URL)
if err != nil {
log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
return fmt.Errorf("parsing api url: %w", err)
}
hub, err := require.Hub(csConfig, nil)
hub, err := require.Hub(cfg, nil, nil)
if err != nil {
log.Fatal(err)
return err
}
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
log.Fatalf("failed to get scenarios : %s", err)
return fmt.Errorf("failed to get scenarios: %w", err)
}
Client, err = apiclient.NewDefaultClient(apiurl,
@ -53,58 +64,49 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
fmt.Sprintf("crowdsec/%s", version.String()),
nil)
if err != nil {
log.Fatalf("init default client: %s", err)
return fmt.Errorf("init default client: %w", err)
}
t := models.WatcherAuthRequest{
MachineID: &login,
Password: &password,
Scenarios: scenarios,
}
log.Infof("Loaded credentials from %s", csConfig.API.Client.CredentialsFilePath)
log.Infof("Loaded credentials from %s", cfg.API.Client.CredentialsFilePath)
log.Infof("Trying to authenticate with username %s on %s", login, apiurl)
_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
if err != nil {
log.Fatalf("Failed to authenticate to Local API (LAPI) : %s", err)
} else {
log.Infof("You can successfully interact with Local API (LAPI)")
return fmt.Errorf("failed to authenticate to Local API (LAPI): %w", err)
}
log.Infof("You can successfully interact with Local API (LAPI)")
return nil
}
func runLapiRegister(cmd *cobra.Command, args []string) error {
func (cli *cliLapi) register(apiURL string, outputFile string, machine string) error {
var err error
flags := cmd.Flags()
apiURL, err := flags.GetString("url")
if err != nil {
return err
}
outputFile, err := flags.GetString("file")
if err != nil {
return err
}
lapiUser, err := flags.GetString("machine")
if err != nil {
return err
}
lapiUser := machine
cfg := cli.cfg()
if lapiUser == "" {
lapiUser, err = generateID("")
if err != nil {
log.Fatalf("unable to generate machine id: %s", err)
return fmt.Errorf("unable to generate machine id: %w", err)
}
}
password := strfmt.Password(generatePassword(passwordLength))
if apiURL == "" {
if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" {
apiURL = csConfig.API.Client.Credentials.URL
} else {
log.Fatalf("No Local API URL. Please provide it in your configuration or with the -u parameter")
if cfg.API.Client == nil || cfg.API.Client.Credentials == nil || cfg.API.Client.Credentials.URL == "" {
return fmt.Errorf("no Local API URL. Please provide it in your configuration or with the -u parameter")
}
apiURL = cfg.API.Client.Credentials.URL
}
/*URL needs to end with /, but user doesn't care*/
if !strings.HasSuffix(apiURL, "/") {
@ -114,10 +116,12 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
if !strings.HasPrefix(apiURL, "http://") && !strings.HasPrefix(apiURL, "https://") {
apiURL = "http://" + apiURL
}
apiurl, err := url.Parse(apiURL)
if err != nil {
log.Fatalf("parsing api url: %s", err)
return fmt.Errorf("parsing api url: %w", err)
}
_, err = apiclient.RegisterClient(&apiclient.Config{
MachineID: lapiUser,
Password: password,
@ -127,143 +131,148 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
}, nil)
if err != nil {
log.Fatalf("api client register: %s", err)
return fmt.Errorf("api client register: %w", err)
}
log.Printf("Successfully registered to Local API (LAPI)")
var dumpFile string
if outputFile != "" {
dumpFile = outputFile
} else if csConfig.API.Client.CredentialsFilePath != "" {
dumpFile = csConfig.API.Client.CredentialsFilePath
} else if cfg.API.Client.CredentialsFilePath != "" {
dumpFile = cfg.API.Client.CredentialsFilePath
} else {
dumpFile = ""
}
apiCfg := csconfig.ApiCredentialsCfg{
Login: lapiUser,
Password: password.String(),
URL: apiURL,
}
apiConfigDump, err := yaml.Marshal(apiCfg)
if err != nil {
log.Fatalf("unable to marshal api credentials: %s", err)
return fmt.Errorf("unable to marshal api credentials: %w", err)
}
if dumpFile != "" {
err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
if err != nil {
log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
return fmt.Errorf("write api credentials to '%s' failed: %w", dumpFile, err)
}
log.Printf("Local API credentials written to '%s'", dumpFile)
} else {
fmt.Printf("%s\n", string(apiConfigDump))
}
log.Warning(ReloadMessage())
return nil
}
func NewLapiStatusCmd() *cobra.Command {
func (cli *cliLapi) newStatusCmd() *cobra.Command {
cmdLapiStatus := &cobra.Command{
Use: "status",
Short: "Check authentication to Local API (LAPI)",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
RunE: runLapiStatus,
RunE: func(cmd *cobra.Command, args []string) error {
return cli.status()
},
}
return cmdLapiStatus
}
func NewLapiRegisterCmd() *cobra.Command {
cmdLapiRegister := &cobra.Command{
func (cli *cliLapi) newRegisterCmd() *cobra.Command {
var (
apiURL string
outputFile string
machine string
)
cmd := &cobra.Command{
Use: "register",
Short: "Register a machine to Local API (LAPI)",
Long: `Register your machine to the Local API (LAPI).
Keep in mind the machine needs to be validated by an administrator on LAPI side to be effective.`,
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
RunE: runLapiRegister,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.register(apiURL, outputFile, machine)
},
}
flags := cmdLapiRegister.Flags()
flags.StringP("url", "u", "", "URL of the API (ie. http://127.0.0.1)")
flags.StringP("file", "f", "", "output file destination")
flags.String("machine", "", "Name of the machine to register with")
flags := cmd.Flags()
flags.StringVarP(&apiURL, "url", "u", "", "URL of the API (ie. http://127.0.0.1)")
flags.StringVarP(&outputFile, "file", "f", "", "output file destination")
flags.StringVar(&machine, "machine", "", "Name of the machine to register with")
return cmdLapiRegister
return cmd
}
func NewLapiCmd() *cobra.Command {
var cmdLapi = &cobra.Command{
func (cli *cliLapi) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "lapi [action]",
Short: "Manage interaction with Local API (LAPI)",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIClient(); err != nil {
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
if err := cli.cfg().LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
}
return nil
},
}
cmdLapi.AddCommand(NewLapiRegisterCmd())
cmdLapi.AddCommand(NewLapiStatusCmd())
cmdLapi.AddCommand(NewLapiContextCmd())
cmd.AddCommand(cli.newRegisterCmd())
cmd.AddCommand(cli.newStatusCmd())
cmd.AddCommand(cli.newContextCmd())
return cmdLapi
return cmd
}
func AddContext(key string, values []string) error {
func (cli *cliLapi) addContext(key string, values []string) error {
cfg := cli.cfg()
if err := alertcontext.ValidateContextExpr(key, values); err != nil {
return fmt.Errorf("invalid context configuration :%s", err)
return fmt.Errorf("invalid context configuration: %w", err)
}
if _, ok := csConfig.Crowdsec.ContextToSend[key]; !ok {
csConfig.Crowdsec.ContextToSend[key] = make([]string, 0)
if _, ok := cfg.Crowdsec.ContextToSend[key]; !ok {
cfg.Crowdsec.ContextToSend[key] = make([]string, 0)
log.Infof("key '%s' added", key)
}
data := csConfig.Crowdsec.ContextToSend[key]
data := cfg.Crowdsec.ContextToSend[key]
for _, val := range values {
if !slices.Contains(data, val) {
log.Infof("value '%s' added to key '%s'", val, key)
data = append(data, val)
}
csConfig.Crowdsec.ContextToSend[key] = data
cfg.Crowdsec.ContextToSend[key] = data
}
if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
if err := cfg.Crowdsec.DumpContextConfigFile(); err != nil {
return err
}
return nil
}
func NewLapiContextCmd() *cobra.Command {
cmdContext := &cobra.Command{
Use: "context [command]",
Short: "Manage context to send with alerts",
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadCrowdsec(); err != nil {
fileNotFoundMessage := fmt.Sprintf("failed to open context file: open %s: no such file or directory", csConfig.Crowdsec.ConsoleContextPath)
if err.Error() != fileNotFoundMessage {
log.Fatalf("Unable to load CrowdSec Agent: %s", err)
}
}
if csConfig.DisableAgent {
log.Fatalf("Agent is disabled and lapi context can only be used on the agent")
}
func (cli *cliLapi) newContextAddCmd() *cobra.Command {
var (
keyToAdd string
valuesToAdd []string
)
return nil
},
Run: func(cmd *cobra.Command, args []string) {
printHelp(cmd)
},
}
var keyToAdd string
var valuesToAdd []string
cmdContextAdd := &cobra.Command{
cmd := &cobra.Command{
Use: "add",
Short: "Add context to send with alerts. You must specify the output key with the expr value you want",
Example: `cscli lapi context add --key source_ip --value evt.Meta.source_ip
@ -271,61 +280,91 @@ cscli lapi context add --key file_source --value evt.Line.Src
cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(_ *cobra.Command, _ []string) error {
hub, err := require.Hub(cli.cfg(), nil, nil)
if err != nil {
return err
}
if err = alertcontext.LoadConsoleContext(cli.cfg(), hub); err != nil {
return fmt.Errorf("while loading context: %w", err)
}
if keyToAdd != "" {
if err := AddContext(keyToAdd, valuesToAdd); err != nil {
log.Fatalf(err.Error())
if err := cli.addContext(keyToAdd, valuesToAdd); err != nil {
return err
}
return
return nil
}
for _, v := range valuesToAdd {
keySlice := strings.Split(v, ".")
key := keySlice[len(keySlice)-1]
value := []string{v}
if err := AddContext(key, value); err != nil {
log.Fatalf(err.Error())
if err := cli.addContext(key, value); err != nil {
return err
}
}
return nil
},
}
cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
cmdContextAdd.Flags().StringSliceVar(&valuesToAdd, "value", []string{}, "The expr fields to associate with the key")
cmdContextAdd.MarkFlagRequired("value")
cmdContext.AddCommand(cmdContextAdd)
cmdContextStatus := &cobra.Command{
flags := cmd.Flags()
flags.StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
flags.StringSliceVar(&valuesToAdd, "value", []string{}, "The expr fields to associate with the key")
cmd.MarkFlagRequired("value")
return cmd
}
func (cli *cliLapi) newContextStatusCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "status",
Short: "List context to send with alerts",
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if len(csConfig.Crowdsec.ContextToSend) == 0 {
fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.")
return
}
dump, err := yaml.Marshal(csConfig.Crowdsec.ContextToSend)
RunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
hub, err := require.Hub(cfg, nil, nil)
if err != nil {
log.Fatalf("unable to show context status: %s", err)
return err
}
fmt.Println(string(dump))
if err = alertcontext.LoadConsoleContext(cfg, hub); err != nil {
return fmt.Errorf("while loading context: %w", err)
}
if len(cfg.Crowdsec.ContextToSend) == 0 {
fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.")
return nil
}
dump, err := yaml.Marshal(cfg.Crowdsec.ContextToSend)
if err != nil {
return fmt.Errorf("unable to show context status: %w", err)
}
fmt.Print(string(dump))
return nil
},
}
cmdContext.AddCommand(cmdContextStatus)
return cmd
}
func (cli *cliLapi) newContextDetectCmd() *cobra.Command {
var detectAll bool
cmdContextDetect := &cobra.Command{
cmd := &cobra.Command{
Use: "detect",
Short: "Detect available fields from the installed parsers",
Example: `cscli lapi context detect --all
cscli lapi context detect crowdsecurity/sshd-logs
`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
var err error
RunE: func(cmd *cobra.Command, args []string) error {
cfg := cli.cfg()
if !detectAll && len(args) == 0 {
log.Infof("Please provide parsers to detect or --all flag.")
printHelp(cmd)
@ -334,19 +373,18 @@ cscli lapi context detect crowdsecurity/sshd-logs
// to avoid all the log.Info from the loaders functions
log.SetLevel(log.WarnLevel)
err = exprhelpers.Init(nil)
if err != nil {
log.Fatalf("Failed to init expr helpers : %s", err)
if err := exprhelpers.Init(nil); err != nil {
return fmt.Errorf("failed to init expr helpers: %w", err)
}
hub, err := require.Hub(csConfig, nil)
hub, err := require.Hub(cfg, nil, nil)
if err != nil {
log.Fatal(err)
return err
}
csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil {
log.Fatalf("unable to load parsers: %s", err)
if csParsers, err = parser.LoadParsers(cfg, csParsers); err != nil {
return fmt.Errorf("unable to load parsers: %w", err)
}
fieldByParsers := make(map[string][]string)
@ -366,7 +404,6 @@ cscli lapi context detect crowdsecurity/sshd-logs
fieldByParsers[node.Name] = append(fieldByParsers[node.Name], field)
}
}
}
fmt.Printf("Acquisition :\n\n")
@ -399,84 +436,89 @@ cscli lapi context detect crowdsecurity/sshd-logs
log.Errorf("parser '%s' not found, can't detect fields", parserNotFound)
}
}
return nil
},
}
cmdContextDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser")
cmdContext.AddCommand(cmdContextDetect)
cmd.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser")
var keysToDelete []string
var valuesToDelete []string
cmdContextDelete := &cobra.Command{
Use: "delete",
Short: "Delete context to send with alerts",
Example: `cscli lapi context delete --key source_ip
cscli lapi context delete --value evt.Line.Src
`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if len(keysToDelete) == 0 && len(valuesToDelete) == 0 {
log.Fatalf("please provide at least a key or a value to delete")
}
for _, key := range keysToDelete {
if _, ok := csConfig.Crowdsec.ContextToSend[key]; ok {
delete(csConfig.Crowdsec.ContextToSend, key)
log.Infof("key '%s' has been removed", key)
} else {
log.Warningf("key '%s' doesn't exist", key)
}
}
for _, value := range valuesToDelete {
valueFound := false
for key, context := range csConfig.Crowdsec.ContextToSend {
if slices.Contains(context, value) {
valueFound = true
csConfig.Crowdsec.ContextToSend[key] = removeFromSlice(value, context)
log.Infof("value '%s' has been removed from key '%s'", value, key)
}
if len(csConfig.Crowdsec.ContextToSend[key]) == 0 {
delete(csConfig.Crowdsec.ContextToSend, key)
}
}
if !valueFound {
log.Warningf("value '%s' not found", value)
}
}
if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
log.Fatalf(err.Error())
}
},
}
cmdContextDelete.Flags().StringSliceVarP(&keysToDelete, "key", "k", []string{}, "The keys to delete")
cmdContextDelete.Flags().StringSliceVar(&valuesToDelete, "value", []string{}, "The expr fields to delete")
cmdContext.AddCommand(cmdContextDelete)
return cmdContext
return cmd
}
func detectStaticField(GrokStatics []parser.ExtraField) []string {
func (cli *cliLapi) newContextDeleteCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "delete",
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
filePath := cli.cfg().Crowdsec.ConsoleContextPath
if filePath == "" {
filePath = "the context file"
}
fmt.Printf("Command 'delete' is deprecated, please manually edit %s.", filePath)
return nil
},
}
return cmd
}
func (cli *cliLapi) newContextCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "context [command]",
Short: "Manage context to send with alerts",
DisableAutoGenTag: true,
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := cfg.LoadCrowdsec(); err != nil {
fileNotFoundMessage := fmt.Sprintf("failed to open context file: open %s: no such file or directory", cfg.Crowdsec.ConsoleContextPath)
if err.Error() != fileNotFoundMessage {
return fmt.Errorf("unable to load CrowdSec agent configuration: %w", err)
}
}
if cfg.DisableAgent {
return errors.New("agent is disabled and lapi context can only be used on the agent")
}
return nil
},
Run: func(cmd *cobra.Command, _ []string) {
printHelp(cmd)
},
}
cmd.AddCommand(cli.newContextAddCmd())
cmd.AddCommand(cli.newContextStatusCmd())
cmd.AddCommand(cli.newContextDetectCmd())
cmd.AddCommand(cli.newContextDeleteCmd())
return cmd
}
func detectStaticField(grokStatics []parser.ExtraField) []string {
ret := make([]string, 0)
for _, static := range GrokStatics {
for _, static := range grokStatics {
if static.Parsed != "" {
fieldName := fmt.Sprintf("evt.Parsed.%s", static.Parsed)
if !slices.Contains(ret, fieldName) {
ret = append(ret, fieldName)
}
}
if static.Meta != "" {
fieldName := fmt.Sprintf("evt.Meta.%s", static.Meta)
if !slices.Contains(ret, fieldName) {
ret = append(ret, fieldName)
}
}
if static.TargetByName != "" {
fieldName := static.TargetByName
if !strings.HasPrefix(fieldName, "evt.") {
fieldName = "evt." + fieldName
}
if !slices.Contains(ret, fieldName) {
ret = append(ret, fieldName)
}
@ -487,7 +529,8 @@ func detectStaticField(GrokStatics []parser.ExtraField) []string {
}
func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
var ret = make([]string, 0)
ret := make([]string, 0)
if node.Grok.RunTimeRegexp != nil {
for _, capturedField := range node.Grok.RunTimeRegexp.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
@ -543,6 +586,7 @@ func detectSubNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
}
}
}
if subnode.Grok.RegexpName != "" {
grokCompiled, err := parserCTX.Grok.Get(subnode.Grok.RegexpName)
if err == nil {

537
cmd/crowdsec-cli/machines.go
View file

@ -5,7 +5,6 @@ import (
"encoding/csv"
"encoding/json"
"fmt"
"io"
"math/big"
"os"
"slices"
@ -18,16 +17,15 @@ import (
"github.com/google/uuid"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/machineid"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/database/ent"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
const passwordLength = 64
@ -47,6 +45,7 @@ func generatePassword(length int) string {
if err != nil {
log.Fatalf("failed getting data from prng for password generation : %s", err)
}
buf[i] = charset[rInt.Int64()]
}
@ -61,12 +60,14 @@ func generateIDPrefix() (string, error) {
if err == nil {
return prefix, nil
}
log.Debugf("failed to get machine-id with usual files: %s", err)
bId, err := uuid.NewRandom()
bID, err := uuid.NewRandom()
if err == nil {
return bId.String(), nil
return bID.String(), nil
}
return "", fmt.Errorf("generating machine id: %w", err)
}
@ -77,11 +78,14 @@ func generateID(prefix string) (string, error) {
if prefix == "" {
prefix, err = generateIDPrefix()
}
if err != nil {
return "", err
}
prefix = strings.ReplaceAll(prefix, "-", "")[:32]
suffix := generatePassword(16)
return prefix + suffix, nil
}
@ -102,135 +106,160 @@ func getLastHeartbeat(m *ent.Machine) (string, bool) {
return hb, true
}
func getAgents(out io.Writer, dbClient *database.Client) error {
machines, err := dbClient.ListMachines()
if err != nil {
return fmt.Errorf("unable to list machines: %s", err)
}
if csConfig.Cscli.Output == "human" {
getAgentsTable(out, machines)
} else if csConfig.Cscli.Output == "json" {
enc := json.NewEncoder(out)
enc.SetIndent("", " ")
if err := enc.Encode(machines); err != nil {
return fmt.Errorf("failed to marshal")
}
return nil
} else if csConfig.Cscli.Output == "raw" {
csvwriter := csv.NewWriter(out)
err := csvwriter.Write([]string{"machine_id", "ip_address", "updated_at", "validated", "version", "auth_type", "last_heartbeat"})
if err != nil {
return fmt.Errorf("failed to write header: %s", err)
}
for _, m := range machines {
var validated string
if m.IsValidated {
validated = "true"
} else {
validated = "false"
}
hb, _ := getLastHeartbeat(m)
err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb})
if err != nil {
return fmt.Errorf("failed to write raw output: %w", err)
}
}
csvwriter.Flush()
} else {
log.Errorf("unknown output '%s'", csConfig.Cscli.Output)
}
return nil
type cliMachines struct {
db *database.Client
cfg configGetter
}
func NewMachinesListCmd() *cobra.Command {
cmdMachinesList := &cobra.Command{
Use: "list",
Short: "list all machines in the database",
Long: `list all machines in the database with their status and last heartbeat`,
Example: `cscli machines list`,
Args: cobra.NoArgs,
func NewCLIMachines(cfg configGetter) *cliMachines {
return &cliMachines{
cfg: cfg,
}
}
func (cli *cliMachines) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "machines [action]",
Short: "Manage local API machines [requires local API]",
Long: `To list/add/delete/validate/prune machines.
Note: This command requires database direct access, so is intended to be run on the local API machine.
`,
Example: `cscli machines [action]`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
err := getAgents(color.Output, dbClient)
Aliases: []string{"machine"},
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
var err error
if err = require.LAPI(cli.cfg()); err != nil {
return err
}
cli.db, err = database.NewClient(cli.cfg().DbConfig)
if err != nil {
return fmt.Errorf("unable to list machines: %s", err)
return fmt.Errorf("unable to create new database client: %s", err)
}
return nil
},
}
return cmdMachinesList
cmd.AddCommand(cli.newListCmd())
cmd.AddCommand(cli.newAddCmd())
cmd.AddCommand(cli.newDeleteCmd())
cmd.AddCommand(cli.newValidateCmd())
cmd.AddCommand(cli.newPruneCmd())
return cmd
}
func NewMachinesAddCmd() *cobra.Command {
cmdMachinesAdd := &cobra.Command{
func (cli *cliMachines) list() error {
out := color.Output
machines, err := cli.db.ListMachines()
if err != nil {
return fmt.Errorf("unable to list machines: %s", err)
}
switch cli.cfg().Cscli.Output {
case "human":
getAgentsTable(out, machines)
case "json":
enc := json.NewEncoder(out)
enc.SetIndent("", " ")
if err := enc.Encode(machines); err != nil {
return fmt.Errorf("failed to marshal")
}
return nil
case "raw":
csvwriter := csv.NewWriter(out)
err := csvwriter.Write([]string{"machine_id", "ip_address", "updated_at", "validated", "version", "auth_type", "last_heartbeat"})
if err != nil {
return fmt.Errorf("failed to write header: %s", err)
}
for _, m := range machines {
validated := "false"
if m.IsValidated {
validated = "true"
}
hb, _ := getLastHeartbeat(m)
if err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb}); err != nil {
return fmt.Errorf("failed to write raw output: %w", err)
}
}
csvwriter.Flush()
}
return nil
}
func (cli *cliMachines) newListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "list",
Short: "list all machines in the database",
Long: `list all machines in the database with their status and last heartbeat`,
Example: `cscli machines list`,
Args: cobra.NoArgs,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.list()
},
}
return cmd
}
func (cli *cliMachines) newAddCmd() *cobra.Command {
var (
password MachinePassword
dumpFile string
apiURL string
interactive bool
autoAdd bool
force bool
)
cmd := &cobra.Command{
Use: "add",
Short: "add a single machine to the database",
DisableAutoGenTag: true,
Long: `Register a new machine in the database. cscli should be on the same machine as LAPI.`,
Example: `
cscli machines add --auto
Example: `cscli machines add --auto
cscli machines add MyTestMachine --auto
cscli machines add MyTestMachine --password MyPassword
`,
RunE: runMachinesAdd,
cscli machines add -f- --auto > /tmp/mycreds.yaml`,
RunE: func(_ *cobra.Command, args []string) error {
return cli.add(args, string(password), dumpFile, apiURL, interactive, autoAdd, force)
},
}
flags := cmdMachinesAdd.Flags()
flags.StringP("password", "p", "", "machine password to login to the API")
flags.StringP("file", "f", "", "output file destination (defaults to "+csconfig.DefaultConfigPath("local_api_credentials.yaml")+")")
flags.StringP("url", "u", "", "URL of the local API")
flags.BoolP("interactive", "i", false, "interfactive mode to enter the password")
flags.BoolP("auto", "a", false, "automatically generate password (and username if not provided)")
flags.Bool("force", false, "will force add the machine if it already exist")
return cmdMachinesAdd
}
func runMachinesAdd(cmd *cobra.Command, args []string) error {
var err error
flags := cmd.Flags()
flags.VarP(&password, "password", "p", "machine password to login to the API")
flags.StringVarP(&dumpFile, "file", "f", "", "output file destination (defaults to "+csconfig.DefaultConfigPath("local_api_credentials.yaml")+")")
flags.StringVarP(&apiURL, "url", "u", "", "URL of the local API")
flags.BoolVarP(&interactive, "interactive", "i", false, "interfactive mode to enter the password")
flags.BoolVarP(&autoAdd, "auto", "a", false, "automatically generate password (and username if not provided)")
flags.BoolVar(&force, "force", false, "will force add the machine if it already exist")
machinePassword, err := flags.GetString("password")
if err != nil {
return err
}
return cmd
}
dumpFile, err := flags.GetString("file")
if err != nil {
return err
}
apiURL, err := flags.GetString("url")
if err != nil {
return err
}
interactive, err := flags.GetBool("interactive")
if err != nil {
return err
}
autoAdd, err := flags.GetBool("auto")
if err != nil {
return err
}
force, err := flags.GetBool("force")
if err != nil {
return err
}
var machineID string
func (cli *cliMachines) add(args []string, machinePassword string, dumpFile string, apiURL string, interactive bool, autoAdd bool, force bool) error {
var (
err error
machineID string
)
// create machineID if not specified by user
if len(args) == 0 {
if !autoAdd {
printHelp(cmd)
return nil
return fmt.Errorf("please specify a machine name to add, or use --auto")
}
machineID, err = generateID("")
if err != nil {
return fmt.Errorf("unable to generate machine id: %s", err)
@ -239,14 +268,18 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
machineID = args[0]
}
clientCfg := cli.cfg().API.Client
serverCfg := cli.cfg().API.Server
/*check if file already exists*/
if dumpFile == "" && csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" {
credFile := csConfig.API.Client.CredentialsFilePath
if dumpFile == "" && clientCfg != nil && clientCfg.CredentialsFilePath != "" {
credFile := clientCfg.CredentialsFilePath
// use the default only if the file does not exist
_, err := os.Stat(credFile)
_, err = os.Stat(credFile)
switch {
case os.IsNotExist(err) || force:
dumpFile = csConfig.API.Client.CredentialsFilePath
dumpFile = credFile
case err != nil:
return fmt.Errorf("unable to stat '%s': %s", credFile, err)
default:
@ -263,94 +296,166 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
if !autoAdd {
return fmt.Errorf("please specify a password with --password or use --auto")
}
machinePassword = generatePassword(passwordLength)
} else if machinePassword == "" && interactive {
qs := &survey.Password{
Message: "Please provide a password for the machine",
Message: "Please provide a password for the machine:",
}
survey.AskOne(qs, &machinePassword)
}
password := strfmt.Password(machinePassword)
_, err = dbClient.CreateMachine(&machineID, &password, "", true, force, types.PasswordAuthType)
_, err = cli.db.CreateMachine(&machineID, &password, "", true, force, types.PasswordAuthType)
if err != nil {
return fmt.Errorf("unable to create machine: %s", err)
}
log.Infof("Machine '%s' successfully added to the local API", machineID)
fmt.Fprintf(os.Stderr, "Machine '%s' successfully added to the local API.\n", machineID)
if apiURL == "" {
if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" {
apiURL = csConfig.API.Client.Credentials.URL
} else if csConfig.API.Server != nil && csConfig.API.Server.ListenURI != "" {
apiURL = "http://" + csConfig.API.Server.ListenURI
if clientCfg != nil && clientCfg.Credentials != nil && clientCfg.Credentials.URL != "" {
apiURL = clientCfg.Credentials.URL
} else if serverCfg != nil && serverCfg.ListenURI != "" {
apiURL = "http://" + serverCfg.ListenURI
} else {
return fmt.Errorf("unable to dump an api URL. Please provide it in your configuration or with the -u parameter")
}
}
apiCfg := csconfig.ApiCredentialsCfg{
Login: machineID,
Password: password.String(),
URL: apiURL,
}
apiConfigDump, err := yaml.Marshal(apiCfg)
if err != nil {
return fmt.Errorf("unable to marshal api credentials: %s", err)
}
if dumpFile != "" && dumpFile != "-" {
err = os.WriteFile(dumpFile, apiConfigDump, 0644)
if err != nil {
if err = os.WriteFile(dumpFile, apiConfigDump, 0o600); err != nil {
return fmt.Errorf("write api credentials in '%s' failed: %s", dumpFile, err)
}
log.Printf("API credentials written to '%s'", dumpFile)
fmt.Fprintf(os.Stderr, "API credentials written to '%s'.\n", dumpFile)
} else {
fmt.Printf("%s\n", string(apiConfigDump))
fmt.Print(string(apiConfigDump))
}
return nil
}
func NewMachinesDeleteCmd() *cobra.Command {
cmdMachinesDelete := &cobra.Command{
Use: "delete [machine_name]...",
Short: "delete machine(s) by name",
Example: `cscli machines delete "machine1" "machine2"`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"remove"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
machines, err := dbClient.ListMachines()
if err != nil {
cobra.CompError("unable to list machines " + err.Error())
}
ret := make([]string, 0)
for _, machine := range machines {
if strings.Contains(machine.MachineId, toComplete) && !slices.Contains(args, machine.MachineId) {
ret = append(ret, machine.MachineId)
}
}
return ret, cobra.ShellCompDirectiveNoFileComp
},
RunE: runMachinesDelete,
func (cli *cliMachines) deleteValid(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
machines, err := cli.db.ListMachines()
if err != nil {
cobra.CompError("unable to list machines " + err.Error())
}
return cmdMachinesDelete
ret := []string{}
for _, machine := range machines {
if strings.Contains(machine.MachineId, toComplete) && !slices.Contains(args, machine.MachineId) {
ret = append(ret, machine.MachineId)
}
}
return ret, cobra.ShellCompDirectiveNoFileComp
}
func runMachinesDelete(cmd *cobra.Command, args []string) error {
for _, machineID := range args {
err := dbClient.DeleteWatcher(machineID)
if err != nil {
func (cli *cliMachines) delete(machines []string) error {
for _, machineID := range machines {
if err := cli.db.DeleteWatcher(machineID); err != nil {
log.Errorf("unable to delete machine '%s': %s", machineID, err)
return nil
}
log.Infof("machine '%s' deleted successfully", machineID)
}
return nil
}
func NewMachinesPruneCmd() *cobra.Command {
var parsedDuration time.Duration
cmdMachinesPrune := &cobra.Command{
func (cli *cliMachines) newDeleteCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "delete [machine_name]...",
Short: "delete machine(s) by name",
Example: `cscli machines delete "machine1" "machine2"`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"remove"},
DisableAutoGenTag: true,
ValidArgsFunction: cli.deleteValid,
RunE: func(_ *cobra.Command, args []string) error {
return cli.delete(args)
},
}
return cmd
}
func (cli *cliMachines) prune(duration time.Duration, notValidOnly bool, force bool) error {
if duration < 2*time.Minute && !notValidOnly {
if yes, err := askYesNo(
"The duration you provided is less than 2 minutes. " +
"This can break installations if the machines are only temporarily disconnected. Continue?", false); err != nil {
return err
} else if !yes {
fmt.Println("User aborted prune. No changes were made.")
return nil
}
}
machines := []*ent.Machine{}
if pending, err := cli.db.QueryPendingMachine(); err == nil {
machines = append(machines, pending...)
}
if !notValidOnly {
if pending, err := cli.db.QueryLastValidatedHeartbeatLT(time.Now().UTC().Add(duration)); err == nil {
machines = append(machines, pending...)
}
}
if len(machines) == 0 {
fmt.Println("no machines to prune")
return nil
}
getAgentsTable(color.Output, machines)
if !force {
if yes, err := askYesNo(
"You are about to PERMANENTLY remove the above machines from the database. " +
"These will NOT be recoverable. Continue?", false); err != nil {
return err
} else if !yes {
fmt.Println("User aborted prune. No changes were made.")
return nil
}
}
deleted, err := cli.db.BulkDeleteWatchers(machines)
if err != nil {
return fmt.Errorf("unable to prune machines: %s", err)
}
fmt.Fprintf(os.Stderr, "successfully delete %d machines\n", deleted)
return nil
}
func (cli *cliMachines) newPruneCmd() *cobra.Command {
var (
duration time.Duration
notValidOnly bool
force bool
)
const defaultDuration = 10 * time.Minute
cmd := &cobra.Command{
Use: "prune",
Short: "prune multiple machines from the database",
Long: `prune multiple machines that are not validated or have not connected to the local API in a given duration.`,
@ -359,77 +464,31 @@ cscli machines prune --duration 1h
cscli machines prune --not-validated-only --force`,
Args: cobra.NoArgs,
DisableAutoGenTag: true,
PreRunE: func(cmd *cobra.Command, args []string) error {
dur, _ := cmd.Flags().GetString("duration")
var err error
parsedDuration, err = time.ParseDuration(fmt.Sprintf("-%s", dur))
if err != nil {
return fmt.Errorf("unable to parse duration '%s': %s", dur, err)
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
notValidOnly, _ := cmd.Flags().GetBool("not-validated-only")
force, _ := cmd.Flags().GetBool("force")
if parsedDuration >= 0-60*time.Second && !notValidOnly {
var answer bool
prompt := &survey.Confirm{
Message: "The duration you provided is less than or equal 60 seconds this can break installations do you want to continue ?",
Default: false,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask about prune check: %s", err)
}
if !answer {
fmt.Println("user aborted prune no changes were made")
return nil
}
}
machines := make([]*ent.Machine, 0)
if pending, err := dbClient.QueryPendingMachine(); err == nil {
machines = append(machines, pending...)
}
if !notValidOnly {
if pending, err := dbClient.QueryLastValidatedHeartbeatLT(time.Now().UTC().Add(parsedDuration)); err == nil {
machines = append(machines, pending...)
}
}
if len(machines) == 0 {
fmt.Println("no machines to prune")
return nil
}
getAgentsTable(color.Output, machines)
if !force {
var answer bool
prompt := &survey.Confirm{
Message: "You are about to PERMANENTLY remove the above machines from the database these will NOT be recoverable, continue ?",
Default: false,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask about prune check: %s", err)
}
if !answer {
fmt.Println("user aborted prune no changes were made")
return nil
}
}
nbDeleted, err := dbClient.BulkDeleteWatchers(machines)
if err != nil {
return fmt.Errorf("unable to prune machines: %s", err)
}
fmt.Printf("successfully delete %d machines\n", nbDeleted)
return nil
RunE: func(_ *cobra.Command, _ []string) error {
return cli.prune(duration, notValidOnly, force)
},
}
cmdMachinesPrune.Flags().StringP("duration", "d", "10m", "duration of time since validated machine last heartbeat")
cmdMachinesPrune.Flags().Bool("not-validated-only", false, "only prune machines that are not validated")
cmdMachinesPrune.Flags().Bool("force", false, "force prune without asking for confirmation")
return cmdMachinesPrune
flags := cmd.Flags()
flags.DurationVarP(&duration, "duration", "d", defaultDuration, "duration of time since validated machine last heartbeat")
flags.BoolVar(&notValidOnly, "not-validated-only", false, "only prune machines that are not validated")
flags.BoolVar(&force, "force", false, "force prune without asking for confirmation")
return cmd
}
func NewMachinesValidateCmd() *cobra.Command {
cmdMachinesValidate := &cobra.Command{
func (cli *cliMachines) validate(machineID string) error {
if err := cli.db.ValidateMachine(machineID); err != nil {
return fmt.Errorf("unable to validate machine '%s': %s", machineID, err)
}
log.Infof("machine '%s' validated successfully", machineID)
return nil
}
func (cli *cliMachines) newValidateCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "validate",
Short: "validate a machine to access the local API",
Long: `validate a machine to access the local API.`,
@ -437,47 +496,9 @@ func NewMachinesValidateCmd() *cobra.Command {
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
machineID := args[0]
if err := dbClient.ValidateMachine(machineID); err != nil {
return fmt.Errorf("unable to validate machine '%s': %s", machineID, err)
}
log.Infof("machine '%s' validated successfully", machineID)
return nil
return cli.validate(args[0])
},
}
return cmdMachinesValidate
}
func NewMachinesCmd() *cobra.Command {
var cmdMachines = &cobra.Command{
Use: "machines [action]",
Short: "Manage local API machines [requires local API]",
Long: `To list/add/delete/validate/prune machines.
Note: This command requires database direct access, so is intended to be run on the local API machine.
`,
Example: `cscli machines [action]`,
DisableAutoGenTag: true,
Aliases: []string{"machine"},
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
var err error
if err := require.LAPI(csConfig); err != nil {
return err
}
dbClient, err = database.NewClient(csConfig.DbConfig)
if err != nil {
return fmt.Errorf("unable to create new database client: %s", err)
}
return nil
},
}
cmdMachines.AddCommand(NewMachinesListCmd())
cmdMachines.AddCommand(NewMachinesAddCmd())
cmdMachines.AddCommand(NewMachinesDeleteCmd())
cmdMachines.AddCommand(NewMachinesValidateCmd())
cmdMachines.AddCommand(NewMachinesPruneCmd())
return cmdMachines
return cmd
}

307
cmd/crowdsec-cli/main.go
View file

@ -1,60 +1,102 @@
package main
import (
"fmt"
"os"
"path/filepath"
"strings"
"slices"
"time"
"github.com/fatih/color"
cc "github.com/ivanpirog/coloredcobra"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/cobra/doc"
"slices"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
)
var trace_lvl, dbg_lvl, nfo_lvl, wrn_lvl, err_lvl bool
var ConfigFilePath string
var csConfig *csconfig.Config
var dbClient *database.Client
var OutputFormat string
var OutputColor string
type configGetter func() *csconfig.Config
var mergedConfig string
// flagBranch overrides the value in csConfig.Cscli.HubBranch
var flagBranch = ""
type cliRoot struct {
logTrace bool
logDebug bool
logInfo bool
logWarn bool
logErr bool
outputColor string
outputFormat string
// flagBranch overrides the value in csConfig.Cscli.HubBranch
flagBranch string
}
func initConfig() {
var err error
if trace_lvl {
log.SetLevel(log.TraceLevel)
} else if dbg_lvl {
log.SetLevel(log.DebugLevel)
} else if nfo_lvl {
log.SetLevel(log.InfoLevel)
} else if wrn_lvl {
log.SetLevel(log.WarnLevel)
} else if err_lvl {
log.SetLevel(log.ErrorLevel)
func newCliRoot() *cliRoot {
return &cliRoot{}
}
// cfg() is a helper function to get the configuration loaded from config.yaml,
// we pass it to subcommands because the file is not read until the Execute() call
func (cli *cliRoot) cfg() *csconfig.Config {
return csConfig
}
// wantedLogLevel returns the log level requested in the command line flags.
func (cli *cliRoot) wantedLogLevel() log.Level {
switch {
case cli.logTrace:
return log.TraceLevel
case cli.logDebug:
return log.DebugLevel
case cli.logInfo:
return log.InfoLevel
case cli.logWarn:
return log.WarnLevel
case cli.logErr:
return log.ErrorLevel
default:
return log.InfoLevel
}
}
// loadConfigFor loads the configuration file for the given sub-command.
// If the sub-command does not need it, it returns a default configuration.
func loadConfigFor(command string) (*csconfig.Config, string, error) {
noNeedConfig := []string{
"doc",
"help",
"completion",
"version",
"hubtest",
}
if !slices.Contains(NoNeedConfig, os.Args[1]) {
if !slices.Contains(noNeedConfig, command) {
log.Debugf("Using %s as configuration file", ConfigFilePath)
csConfig, mergedConfig, err = csconfig.NewConfig(ConfigFilePath, false, false, true)
config, merged, err := csconfig.NewConfig(ConfigFilePath, false, false, true)
if err != nil {
log.Fatal(err)
return nil, "", err
}
} else {
csConfig = csconfig.NewDefaultConfig()
return config, merged, nil
}
return csconfig.NewDefaultConfig(), "", nil
}
// initialize is called before the subcommand is executed.
func (cli *cliRoot) initialize() {
var err error
log.SetLevel(cli.wantedLogLevel())
csConfig, mergedConfig, err = loadConfigFor(os.Args[1])
if err != nil {
log.Fatal(err)
}
// recap of the enabled feature flags, because logging
@ -63,19 +105,22 @@ func initConfig() {
log.Debugf("Enabled feature flags: %s", fflist)
}
if flagBranch != "" {
csConfig.Cscli.HubBranch = flagBranch
if cli.flagBranch != "" {
csConfig.Cscli.HubBranch = cli.flagBranch
}
if OutputFormat != "" {
csConfig.Cscli.Output = OutputFormat
if OutputFormat != "json" && OutputFormat != "raw" && OutputFormat != "human" {
log.Fatalf("output format %s unknown", OutputFormat)
}
if cli.outputFormat != "" {
csConfig.Cscli.Output = cli.outputFormat
}
if csConfig.Cscli.Output == "" {
csConfig.Cscli.Output = "human"
}
if csConfig.Cscli.Output != "human" && csConfig.Cscli.Output != "json" && csConfig.Cscli.Output != "raw" {
log.Fatalf("output format '%s' not supported: must be one of human, json, raw", csConfig.Cscli.Output)
}
if csConfig.Cscli.Output == "json" {
log.SetFormatter(&log.JSONFormatter{})
log.SetLevel(log.ErrorLevel)
@ -83,47 +128,44 @@ func initConfig() {
log.SetLevel(log.ErrorLevel)
}
if OutputColor != "" {
csConfig.Cscli.Color = OutputColor
if OutputColor != "yes" && OutputColor != "no" && OutputColor != "auto" {
log.Fatalf("output color %s unknown", OutputColor)
if cli.outputColor != "" {
csConfig.Cscli.Color = cli.outputColor
if cli.outputColor != "yes" && cli.outputColor != "no" && cli.outputColor != "auto" {
log.Fatalf("output color %s unknown", cli.outputColor)
}
}
}
// list of valid subcommands for the shell completion
var validArgs = []string{
"scenarios", "parsers", "collections", "capi", "lapi", "postoverflows", "machines",
"metrics", "bouncers", "alerts", "decisions", "simulation", "hub", "dashboard",
"config", "completion", "version", "console", "notifications", "support",
"alerts", "appsec-configs", "appsec-rules", "bouncers", "capi", "collections",
"completion", "config", "console", "contexts", "dashboard", "decisions", "explain",
"hub", "hubtest", "lapi", "machines", "metrics", "notifications", "parsers",
"postoverflows", "scenarios", "simulation", "support", "version",
}
func prepender(filename string) string {
const header = `---
id: %s
title: %s
---
`
name := filepath.Base(filename)
base := strings.TrimSuffix(name, filepath.Ext(name))
return fmt.Sprintf(header, base, strings.ReplaceAll(base, "_", " "))
func (cli *cliRoot) colorize(cmd *cobra.Command) {
cc.Init(&cc.Config{
RootCmd: cmd,
Headings: cc.Yellow,
Commands: cc.Green + cc.Bold,
CmdShortDescr: cc.Cyan,
Example: cc.Italic,
ExecName: cc.Bold,
Aliases: cc.Bold + cc.Italic,
FlagsDataType: cc.White,
Flags: cc.Green,
FlagsDescr: cc.Cyan,
NoExtraNewlines: true,
NoBottomNewline: true,
})
cmd.SetOut(color.Output)
}
func linkHandler(name string) string {
return fmt.Sprintf("/cscli/%s", name)
}
var (
NoNeedConfig = []string{
"help",
"completion",
"version",
"hubtest",
}
)
func main() {
func (cli *cliRoot) NewCommand() *cobra.Command {
// set the formatter asap and worry about level later
logFormatter := &log.TextFormatter{TimestampFormat: "2006-01-02 15:04:05", FullTimestamp: true}
logFormatter := &log.TextFormatter{TimestampFormat: time.RFC3339, FullTimestamp: true}
log.SetFormatter(logFormatter)
if err := fflag.RegisterAllFeatures(); err != nil {
@ -134,7 +176,7 @@ func main() {
log.Fatalf("failed to set feature flags from env: %s", err)
}
var rootCmd = &cobra.Command{
cmd := &cobra.Command{
Use: "cscli",
Short: "cscli allows you to manage crowdsec",
Long: `cscli is the main command to interact with your crowdsec service, scenarios & db.
@ -146,57 +188,25 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
/*TBD examples*/
}
cc.Init(&cc.Config{
RootCmd: rootCmd,
Headings: cc.Yellow,
Commands: cc.Green + cc.Bold,
CmdShortDescr: cc.Cyan,
Example: cc.Italic,
ExecName: cc.Bold,
Aliases: cc.Bold + cc.Italic,
FlagsDataType: cc.White,
Flags: cc.Green,
FlagsDescr: cc.Cyan,
})
rootCmd.SetOut(color.Output)
cli.colorize(cmd)
var cmdDocGen = &cobra.Command{
Use: "doc",
Short: "Generate the documentation in `./doc/`. Directory must exist.",
Args: cobra.ExactArgs(0),
Hidden: true,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if err := doc.GenMarkdownTreeCustom(rootCmd, "./doc/", prepender, linkHandler); err != nil {
return fmt.Errorf("Failed to generate cobra doc: %s", err)
}
return nil
},
}
rootCmd.AddCommand(cmdDocGen)
/*usage*/
var cmdVersion = &cobra.Command{
Use: "version",
Short: "Display version",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
cwversion.Show()
},
}
rootCmd.AddCommand(cmdVersion)
/*don't sort flags so we can enforce order*/
cmd.Flags().SortFlags = false
rootCmd.PersistentFlags().StringVarP(&ConfigFilePath, "config", "c", csconfig.DefaultConfigPath("config.yaml"), "path to crowdsec config file")
rootCmd.PersistentFlags().StringVarP(&OutputFormat, "output", "o", "", "Output format: human, json, raw")
rootCmd.PersistentFlags().StringVarP(&OutputColor, "color", "", "auto", "Output color: yes, no, auto")
rootCmd.PersistentFlags().BoolVar(&dbg_lvl, "debug", false, "Set logging to debug")
rootCmd.PersistentFlags().BoolVar(&nfo_lvl, "info", false, "Set logging to info")
rootCmd.PersistentFlags().BoolVar(&wrn_lvl, "warning", false, "Set logging to warning")
rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error")
rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace")
pflags := cmd.PersistentFlags()
pflags.SortFlags = false
rootCmd.PersistentFlags().StringVar(&flagBranch, "branch", "", "Override hub branch on github")
if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil {
pflags.StringVarP(&ConfigFilePath, "config", "c", csconfig.DefaultConfigPath("config.yaml"), "path to crowdsec config file")
pflags.StringVarP(&cli.outputFormat, "output", "o", "", "Output format: human, json, raw")
pflags.StringVarP(&cli.outputColor, "color", "", "auto", "Output color: yes, no, auto")
pflags.BoolVar(&cli.logDebug, "debug", false, "Set logging to debug")
pflags.BoolVar(&cli.logInfo, "info", false, "Set logging to info")
pflags.BoolVar(&cli.logWarn, "warning", false, "Set logging to warning")
pflags.BoolVar(&cli.logErr, "error", false, "Set logging to error")
pflags.BoolVar(&cli.logTrace, "trace", false, "Set logging to trace")
pflags.StringVar(&cli.flagBranch, "branch", "", "Override hub branch on github")
if err := pflags.MarkHidden("branch"); err != nil {
log.Fatalf("failed to hide flag: %s", err)
}
@ -216,44 +226,47 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
}
if len(os.Args) > 1 {
cobra.OnInitialize(initConfig)
cobra.OnInitialize(cli.initialize)
}
/*don't sort flags so we can enforce order*/
rootCmd.Flags().SortFlags = false
rootCmd.PersistentFlags().SortFlags = false
rootCmd.AddCommand(NewConfigCmd())
rootCmd.AddCommand(NewHubCmd())
rootCmd.AddCommand(NewMetricsCmd())
rootCmd.AddCommand(NewDashboardCmd())
rootCmd.AddCommand(NewDecisionsCmd())
rootCmd.AddCommand(NewAlertsCmd())
rootCmd.AddCommand(NewSimulationCmds())
rootCmd.AddCommand(NewBouncersCmd())
rootCmd.AddCommand(NewMachinesCmd())
rootCmd.AddCommand(NewCapiCmd())
rootCmd.AddCommand(NewLapiCmd())
rootCmd.AddCommand(NewCompletionCmd())
rootCmd.AddCommand(NewConsoleCmd())
rootCmd.AddCommand(NewExplainCmd())
rootCmd.AddCommand(NewHubTestCmd())
rootCmd.AddCommand(NewNotificationsCmd())
rootCmd.AddCommand(NewSupportCmd())
rootCmd.AddCommand(NewItemsCmd("collections"))
rootCmd.AddCommand(NewItemsCmd("parsers"))
rootCmd.AddCommand(NewItemsCmd("scenarios"))
rootCmd.AddCommand(NewItemsCmd("postoverflows"))
cmd.AddCommand(NewCLIDoc().NewCommand(cmd))
cmd.AddCommand(NewCLIVersion().NewCommand())
cmd.AddCommand(NewConfigCmd())
cmd.AddCommand(NewCLIHub(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIMetrics(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIDashboard(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIDecisions(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIAlerts(cli.cfg).NewCommand())
cmd.AddCommand(NewCLISimulation(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIBouncers(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIMachines(cli.cfg).NewCommand())
cmd.AddCommand(NewCLICapi().NewCommand())
cmd.AddCommand(NewCLILapi(cli.cfg).NewCommand())
cmd.AddCommand(NewCompletionCmd())
cmd.AddCommand(NewCLIConsole(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIExplain(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIHubTest().NewCommand())
cmd.AddCommand(NewCLINotifications(cli.cfg).NewCommand())
cmd.AddCommand(NewCLISupport().NewCommand())
cmd.AddCommand(NewCLIPapi(cli.cfg).NewCommand())
cmd.AddCommand(NewCLICollection().NewCommand())
cmd.AddCommand(NewCLIParser().NewCommand())
cmd.AddCommand(NewCLIScenario().NewCommand())
cmd.AddCommand(NewCLIPostOverflow().NewCommand())
cmd.AddCommand(NewCLIContext().NewCommand())
cmd.AddCommand(NewCLIAppsecConfig().NewCommand())
cmd.AddCommand(NewCLIAppsecRule().NewCommand())
if fflag.CscliSetup.IsEnabled() {
rootCmd.AddCommand(NewSetupCmd())
cmd.AddCommand(NewSetupCmd())
}
if fflag.PapiClient.IsEnabled() {
rootCmd.AddCommand(NewPapiCmd())
}
return cmd
}
if err := rootCmd.Execute(); err != nil {
func main() {
cmd := newCliRoot().NewCommand()
if err := cmd.Execute(); err != nil {
log.Fatal(err)
}
}

512
cmd/crowdsec-cli/metrics.go
View file

@ -2,6 +2,7 @@ package main
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@ -16,11 +17,64 @@ import (
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/go-cs-lib/maptools"
"github.com/crowdsecurity/go-cs-lib/trace"
)
// FormatPrometheusMetrics is a complete rip from prom2json
func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error {
type (
statAcquis map[string]map[string]int
statParser map[string]map[string]int
statBucket map[string]map[string]int
statWhitelist map[string]map[string]map[string]int
statLapi map[string]map[string]int
statLapiMachine map[string]map[string]map[string]int
statLapiBouncer map[string]map[string]map[string]int
statLapiDecision map[string]struct {
NonEmpty int
Empty int
}
statDecision map[string]map[string]map[string]int
statAppsecEngine map[string]map[string]int
statAppsecRule map[string]map[string]map[string]int
statAlert map[string]int
statStash map[string]struct {
Type string
Count int
}
)
var (
ErrMissingConfig = errors.New("prometheus section missing, can't show metrics")
ErrMetricsDisabled = errors.New("prometheus is not enabled, can't show metrics")
)
type metricSection interface {
Table(out io.Writer, noUnit bool, showEmpty bool)
Description() (string, string)
}
type metricStore map[string]metricSection
func NewMetricStore() metricStore {
return metricStore{
"acquisition": statAcquis{},
"buckets": statBucket{},
"parsers": statParser{},
"lapi": statLapi{},
"lapi-machine": statLapiMachine{},
"lapi-bouncer": statLapiBouncer{},
"lapi-decisions": statLapiDecision{},
"decisions": statDecision{},
"alerts": statAlert{},
"stash": statStash{},
"appsec-engine": statAppsecEngine{},
"appsec-rule": statAppsecRule{},
"whitelists": statWhitelist{},
}
}
func (ms metricStore) Fetch(url string) error {
mfChan := make(chan *dto.MetricFamily, 1024)
errChan := make(chan error, 1)
@ -33,9 +87,10 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
transport.ResponseHeaderTimeout = time.Minute
go func() {
defer trace.CatchPanic("crowdsec/ShowPrometheus")
err := prom2json.FetchMetricFamilies(url, mfChan, transport)
if err != nil {
errChan <- fmt.Errorf("failed to fetch prometheus metrics: %w", err)
errChan <- fmt.Errorf("failed to fetch metrics: %w", err)
return
}
errChan <- nil
@ -50,40 +105,42 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
return err
}
log.Debugf("Finished reading prometheus output, %d entries", len(result))
log.Debugf("Finished reading metrics output, %d entries", len(result))
/*walk*/
lapi_decisions_stats := map[string]struct {
NonEmpty int
Empty int
}{}
acquis_stats := map[string]map[string]int{}
parsers_stats := map[string]map[string]int{}
buckets_stats := map[string]map[string]int{}
lapi_stats := map[string]map[string]int{}
lapi_machine_stats := map[string]map[string]map[string]int{}
lapi_bouncer_stats := map[string]map[string]map[string]int{}
decisions_stats := map[string]map[string]map[string]int{}
alerts_stats := map[string]int{}
stash_stats := map[string]struct {
Type string
Count int
}{}
mAcquis := ms["acquisition"].(statAcquis)
mParser := ms["parsers"].(statParser)
mBucket := ms["buckets"].(statBucket)
mLapi := ms["lapi"].(statLapi)
mLapiMachine := ms["lapi-machine"].(statLapiMachine)
mLapiBouncer := ms["lapi-bouncer"].(statLapiBouncer)
mLapiDecision := ms["lapi-decisions"].(statLapiDecision)
mDecision := ms["decisions"].(statDecision)
mAppsecEngine := ms["appsec-engine"].(statAppsecEngine)
mAppsecRule := ms["appsec-rule"].(statAppsecRule)
mAlert := ms["alerts"].(statAlert)
mStash := ms["stash"].(statStash)
mWhitelist := ms["whitelists"].(statWhitelist)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
source, ok := metric.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v for %s", metric.Labels, fam.Name)
@ -104,173 +161,142 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
origin := metric.Labels["origin"]
action := metric.Labels["action"]
appsecEngine := metric.Labels["appsec_engine"]
appsecRule := metric.Labels["rule_name"]
mtype := metric.Labels["type"]
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
}
ival := int(fval)
switch fam.Name {
/*buckets*/
//
// buckets
//
case "cs_bucket_created_total":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
buckets_stats[name]["instantiation"] += ival
mBucket.Process(name, "instantiation", ival)
case "cs_buckets":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
buckets_stats[name]["curr_count"] += ival
mBucket.Process(name, "curr_count", ival)
case "cs_bucket_overflowed_total":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
buckets_stats[name]["overflow"] += ival
mBucket.Process(name, "overflow", ival)
case "cs_bucket_poured_total":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
}
buckets_stats[name]["pour"] += ival
acquis_stats[source]["pour"] += ival
mBucket.Process(name, "pour", ival)
mAcquis.Process(source, "pour", ival)
case "cs_bucket_underflowed_total":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
buckets_stats[name]["underflow"] += ival
/*acquis*/
mBucket.Process(name, "underflow", ival)
//
// parsers
//
case "cs_parser_hits_total":
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
}
acquis_stats[source]["reads"] += ival
mAcquis.Process(source, "reads", ival)
case "cs_parser_hits_ok_total":
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
}
acquis_stats[source]["parsed"] += ival
mAcquis.Process(source, "parsed", ival)
case "cs_parser_hits_ko_total":
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
}
acquis_stats[source]["unparsed"] += ival
mAcquis.Process(source, "unparsed", ival)
case "cs_node_hits_total":
if _, ok := parsers_stats[name]; !ok {
parsers_stats[name] = make(map[string]int)
}
parsers_stats[name]["hits"] += ival
mParser.Process(name, "hits", ival)
case "cs_node_hits_ok_total":
if _, ok := parsers_stats[name]; !ok {
parsers_stats[name] = make(map[string]int)
}
parsers_stats[name]["parsed"] += ival
mParser.Process(name, "parsed", ival)
case "cs_node_hits_ko_total":
if _, ok := parsers_stats[name]; !ok {
parsers_stats[name] = make(map[string]int)
}
parsers_stats[name]["unparsed"] += ival
mParser.Process(name, "unparsed", ival)
//
// whitelists
//
case "cs_node_wl_hits_total":
mWhitelist.Process(name, reason, "hits", ival)
case "cs_node_wl_hits_ok_total":
mWhitelist.Process(name, reason, "whitelisted", ival)
// track as well whitelisted lines at acquis level
mAcquis.Process(source, "whitelisted", ival)
//
// lapi
//
case "cs_lapi_route_requests_total":
if _, ok := lapi_stats[route]; !ok {
lapi_stats[route] = make(map[string]int)
}
lapi_stats[route][method] += ival
mLapi.Process(route, method, ival)
case "cs_lapi_machine_requests_total":
if _, ok := lapi_machine_stats[machine]; !ok {
lapi_machine_stats[machine] = make(map[string]map[string]int)
}
if _, ok := lapi_machine_stats[machine][route]; !ok {
lapi_machine_stats[machine][route] = make(map[string]int)
}
lapi_machine_stats[machine][route][method] += ival
mLapiMachine.Process(machine, route, method, ival)
case "cs_lapi_bouncer_requests_total":
if _, ok := lapi_bouncer_stats[bouncer]; !ok {
lapi_bouncer_stats[bouncer] = make(map[string]map[string]int)
}
if _, ok := lapi_bouncer_stats[bouncer][route]; !ok {
lapi_bouncer_stats[bouncer][route] = make(map[string]int)
}
lapi_bouncer_stats[bouncer][route][method] += ival
mLapiBouncer.Process(bouncer, route, method, ival)
case "cs_lapi_decisions_ko_total", "cs_lapi_decisions_ok_total":
if _, ok := lapi_decisions_stats[bouncer]; !ok {
lapi_decisions_stats[bouncer] = struct {
NonEmpty int
Empty int
}{}
}
x := lapi_decisions_stats[bouncer]
if fam.Name == "cs_lapi_decisions_ko_total" {
x.Empty += ival
} else if fam.Name == "cs_lapi_decisions_ok_total" {
x.NonEmpty += ival
}
lapi_decisions_stats[bouncer] = x
mLapiDecision.Process(bouncer, fam.Name, ival)
//
// decisions
//
case "cs_active_decisions":
if _, ok := decisions_stats[reason]; !ok {
decisions_stats[reason] = make(map[string]map[string]int)
}
if _, ok := decisions_stats[reason][origin]; !ok {
decisions_stats[reason][origin] = make(map[string]int)
}
decisions_stats[reason][origin][action] += ival
mDecision.Process(reason, origin, action, ival)
case "cs_alerts":
/*if _, ok := alerts_stats[scenario]; !ok {
alerts_stats[scenario] = make(map[string]int)
}*/
alerts_stats[reason] += ival
mAlert.Process(reason, ival)
//
// stash
//
case "cs_cache_size":
stash_stats[name] = struct {
Type string
Count int
}{Type: mtype, Count: ival}
mStash.Process(name, mtype, ival)
//
// appsec
//
case "cs_appsec_reqs_total":
mAppsecEngine.Process(appsecEngine, "processed", ival)
case "cs_appsec_block_total":
mAppsecEngine.Process(appsecEngine, "blocked", ival)
case "cs_appsec_rule_hits":
mAppsecRule.Process(appsecEngine, appsecRule, "triggered", ival)
default:
log.Debugf("unknown: %+v", fam.Name)
continue
}
}
}
if formatType == "human" {
acquisStatsTable(out, acquis_stats)
bucketStatsTable(out, buckets_stats)
parserStatsTable(out, parsers_stats)
lapiStatsTable(out, lapi_stats)
lapiMachineStatsTable(out, lapi_machine_stats)
lapiBouncerStatsTable(out, lapi_bouncer_stats)
lapiDecisionStatsTable(out, lapi_decisions_stats)
decisionStatsTable(out, decisions_stats)
alertStatsTable(out, alerts_stats)
stashStatsTable(out, stash_stats)
return nil
return nil
}
type cliMetrics struct {
cfg configGetter
}
func NewCLIMetrics(cfg configGetter) *cliMetrics {
return &cliMetrics{
cfg: cfg,
}
}
func (ms metricStore) Format(out io.Writer, sections []string, formatType string, noUnit bool) error {
// copy only the sections we want
want := map[string]metricSection{}
// if explicitly asking for sections, we want to show empty tables
showEmpty := len(sections) > 0
// if no sections are specified, we want all of them
if len(sections) == 0 {
for section := range ms {
sections = append(sections, section)
}
}
stats := make(map[string]any)
stats["acquisition"] = acquis_stats
stats["buckets"] = buckets_stats
stats["parsers"] = parsers_stats
stats["lapi"] = lapi_stats
stats["lapi_machine"] = lapi_machine_stats
stats["lapi_bouncer"] = lapi_bouncer_stats
stats["lapi_decisions"] = lapi_decisions_stats
stats["decisions"] = decisions_stats
stats["alerts"] = alerts_stats
stats["stash"] = stash_stats
for _, section := range sections {
want[section] = ms[section]
}
switch formatType {
case "human":
for section := range want {
want[section].Table(out, noUnit, showEmpty)
}
case "json":
x, err := json.MarshalIndent(stats, "", " ")
x, err := json.MarshalIndent(want, "", " ")
if err != nil {
return fmt.Errorf("failed to unmarshal metrics : %v", err)
return fmt.Errorf("failed to marshal metrics: %w", err)
}
out.Write(x)
case "raw":
x, err := yaml.Marshal(stats)
x, err := yaml.Marshal(want)
if err != nil {
return fmt.Errorf("failed to unmarshal metrics : %v", err)
return fmt.Errorf("failed to marshal metrics: %w", err)
}
out.Write(x)
default:
@ -280,54 +306,190 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
return nil
}
var noUnit bool
func runMetrics(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
url, err := flags.GetString("url")
if err != nil {
return err
}
func (cli *cliMetrics) show(sections []string, url string, noUnit bool) error {
cfg := cli.cfg()
if url != "" {
csConfig.Cscli.PrometheusUrl = url
cfg.Cscli.PrometheusUrl = url
}
noUnit, err = flags.GetBool("no-unit")
if err != nil {
if cfg.Prometheus == nil {
return ErrMissingConfig
}
if !cfg.Prometheus.Enabled {
return ErrMetricsDisabled
}
ms := NewMetricStore()
if err := ms.Fetch(cfg.Cscli.PrometheusUrl); err != nil {
return err
}
if csConfig.Prometheus == nil {
return fmt.Errorf("prometheus section missing, can't show metrics")
// any section that we don't have in the store is an error
for _, section := range sections {
if _, ok := ms[section]; !ok {
return fmt.Errorf("unknown metrics type: %s", section)
}
}
if !csConfig.Prometheus.Enabled {
return fmt.Errorf("prometheus is not enabled, can't show metrics")
}
if err = FormatPrometheusMetrics(color.Output, csConfig.Cscli.PrometheusUrl, csConfig.Cscli.Output); err != nil {
if err := ms.Format(color.Output, sections, cfg.Cscli.Output, noUnit); err != nil {
return err
}
return nil
}
func (cli *cliMetrics) NewCommand() *cobra.Command {
var (
url string
noUnit bool
)
func NewMetricsCmd() *cobra.Command {
cmdMetrics := &cobra.Command{
Use: "metrics",
Short: "Display crowdsec prometheus metrics.",
Long: `Fetch metrics from the prometheus server and display them in a human-friendly way`,
cmd := &cobra.Command{
Use: "metrics",
Short: "Display crowdsec prometheus metrics.",
Long: `Fetch metrics from a Local API server and display them`,
Example: `# Show all Metrics, skip empty tables (same as "cecli metrics show")
cscli metrics
# Show only some metrics, connect to a different url
cscli metrics --url http://lapi.local:6060/metrics show acquisition parsers
# List available metric types
cscli metrics list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: runMetrics,
RunE: func(cmd *cobra.Command, args []string) error {
return cli.show(nil, url, noUnit)
},
}
flags := cmdMetrics.PersistentFlags()
flags.StringP("url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
flags.Bool("no-unit", false, "Show the real number instead of formatted with units")
flags := cmd.Flags()
flags.StringVarP(&url, "url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
flags.BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
return cmdMetrics
cmd.AddCommand(cli.newShowCmd())
cmd.AddCommand(cli.newListCmd())
return cmd
}
// expandAlias returns a list of sections. The input can be a list of sections or alias.
func (cli *cliMetrics) expandSectionGroups(args []string) []string {
ret := []string{}
for _, section := range args {
switch section {
case "engine":
ret = append(ret, "acquisition", "parsers", "buckets", "stash", "whitelists")
case "lapi":
ret = append(ret, "alerts", "decisions", "lapi", "lapi-bouncer", "lapi-decisions", "lapi-machine")
case "appsec":
ret = append(ret, "appsec-engine", "appsec-rule")
default:
ret = append(ret, section)
}
}
return ret
}
func (cli *cliMetrics) newShowCmd() *cobra.Command {
var (
url string
noUnit bool
)
cmd := &cobra.Command{
Use: "show [type]...",
Short: "Display all or part of the available metrics.",
Long: `Fetch metrics from a Local API server and display them, optionally filtering on specific types.`,
Example: `# Show all Metrics, skip empty tables
cscli metrics show
# Use an alias: "engine", "lapi" or "appsec" to show a group of metrics
cscli metrics show engine
# Show some specific metrics, show empty tables, connect to a different url
cscli metrics show acquisition parsers buckets stash --url http://lapi.local:6060/metrics
# Show metrics in json format
cscli metrics show acquisition parsers buckets stash -o json`,
// Positional args are optional
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
args = cli.expandSectionGroups(args)
return cli.show(args, url, noUnit)
},
}
flags := cmd.Flags()
flags.StringVarP(&url, "url", "u", "", "Metrics url (http://<ip>:<port>/metrics)")
flags.BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
return cmd
}
func (cli *cliMetrics) list() error {
type metricType struct {
Type string `json:"type" yaml:"type"`
Title string `json:"title" yaml:"title"`
Description string `json:"description" yaml:"description"`
}
var allMetrics []metricType
ms := NewMetricStore()
for _, section := range maptools.SortedKeys(ms) {
title, description := ms[section].Description()
allMetrics = append(allMetrics, metricType{
Type: section,
Title: title,
Description: description,
})
}
switch cli.cfg().Cscli.Output {
case "human":
t := newTable(color.Output)
t.SetRowLines(true)
t.SetHeaders("Type", "Title", "Description")
for _, metric := range allMetrics {
t.AddRow(metric.Type, metric.Title, metric.Description)
}
t.Render()
case "json":
x, err := json.MarshalIndent(allMetrics, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal metric types: %w", err)
}
fmt.Println(string(x))
case "raw":
x, err := yaml.Marshal(allMetrics)
if err != nil {
return fmt.Errorf("failed to marshal metric types: %w", err)
}
fmt.Println(string(x))
}
return nil
}
func (cli *cliMetrics) newListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "list",
Short: "List available types of metrics.",
Long: `List available types of metrics.`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.list()
},
}
return cmd
}

464
cmd/crowdsec-cli/metrics_table.go
View file

@ -4,22 +4,29 @@ import (
"fmt"
"io"
"sort"
"strconv"
"github.com/aquasecurity/table"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/maptools"
)
// ErrNilTable means a nil pointer was passed instead of a table instance. This is a programming error.
var ErrNilTable = fmt.Errorf("nil table")
func lapiMetricsToTable(t *table.Table, stats map[string]map[string]map[string]int) int {
// stats: machine -> route -> method -> count
// sort keys to keep consistent order when printing
machineKeys := []string{}
for k := range stats {
machineKeys = append(machineKeys, k)
}
sort.Strings(machineKeys)
numRows := 0
for _, machine := range machineKeys {
// oneRow: route -> method -> count
machineRow := stats[machine]
@ -31,41 +38,77 @@ func lapiMetricsToTable(t *table.Table, stats map[string]map[string]map[string]i
methodName,
}
if count != 0 {
row = append(row, fmt.Sprintf("%d", count))
row = append(row, strconv.Itoa(count))
} else {
row = append(row, "-")
}
t.AddRow(row...)
numRows++
}
}
}
return numRows
}
func metricsToTable(t *table.Table, stats map[string]map[string]int, keys []string) (int, error) {
func wlMetricsToTable(t *table.Table, stats map[string]map[string]map[string]int, noUnit bool) (int, error) {
if t == nil {
return 0, fmt.Errorf("nil table")
return 0, ErrNilTable
}
// sort keys to keep consistent order when printing
sortedKeys := []string{}
for k := range stats {
sortedKeys = append(sortedKeys, k)
}
sort.Strings(sortedKeys)
numRows := 0
for _, alabel := range sortedKeys {
for _, name := range maptools.SortedKeys(stats) {
for _, reason := range maptools.SortedKeys(stats[name]) {
row := []string{
name,
reason,
"-",
"-",
}
for _, action := range maptools.SortedKeys(stats[name][reason]) {
value := stats[name][reason][action]
switch action {
case "whitelisted":
row[3] = strconv.Itoa(value)
case "hits":
row[2] = strconv.Itoa(value)
default:
log.Debugf("unexpected counter '%s' for whitelists = %d", action, value)
}
}
t.AddRow(row...)
numRows++
}
}
return numRows, nil
}
func metricsToTable(t *table.Table, stats map[string]map[string]int, keys []string, noUnit bool) (int, error) {
if t == nil {
return 0, ErrNilTable
}
numRows := 0
for _, alabel := range maptools.SortedKeys(stats) {
astats, ok := stats[alabel]
if !ok {
continue
}
row := []string{
alabel,
}
for _, sl := range keys {
if v, ok := astats[sl]; ok && v != 0 {
numberToShow := fmt.Sprintf("%d", v)
numberToShow := strconv.Itoa(v)
if !noUnit {
numberToShow = formatNumber(v)
}
@ -75,13 +118,29 @@ func metricsToTable(t *table.Table, stats map[string]map[string]int, keys []stri
row = append(row, "-")
}
}
t.AddRow(row...)
numRows++
}
return numRows, nil
}
func bucketStatsTable(out io.Writer, stats map[string]map[string]int) {
func (s statBucket) Description() (string, string) {
return "Bucket Metrics",
`Measure events in different scenarios. Current count is the number of buckets during metrics collection. ` +
`Overflows are past event-producing buckets, while Expired are the ones that didn’t receive enough events to Overflow.`
}
func (s statBucket) Process(bucket, metric string, val int) {
if _, ok := s[bucket]; !ok {
s[bucket] = make(map[string]int)
}
s[bucket][metric] += val
}
func (s statBucket) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Bucket", "Current Count", "Overflows", "Instantiated", "Poured", "Expired")
@ -89,31 +148,161 @@ func bucketStatsTable(out io.Writer, stats map[string]map[string]int) {
keys := []string{"curr_count", "overflow", "instantiation", "pour", "underflow"}
if numRows, err := metricsToTable(t, stats, keys); err != nil {
log.Warningf("while collecting acquis stats: %s", err)
} else if numRows > 0 {
renderTableTitle(out, "\nBucket Metrics:")
if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
log.Warningf("while collecting bucket stats: %s", err)
} else if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func acquisStatsTable(out io.Writer, stats map[string]map[string]int) {
func (s statAcquis) Description() (string, string) {
return "Acquisition Metrics",
`Measures the lines read, parsed, and unparsed per datasource. ` +
`Zero read lines indicate a misconfigured or inactive datasource. ` +
`Zero parsed lines mean the parser(s) failed. ` +
`Non-zero parsed lines are fine as crowdsec selects relevant lines.`
}
func (s statAcquis) Process(source, metric string, val int) {
if _, ok := s[source]; !ok {
s[source] = make(map[string]int)
}
s[source][metric] += val
}
func (s statAcquis) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Source", "Lines read", "Lines parsed", "Lines unparsed", "Lines poured to bucket")
t.SetHeaders("Source", "Lines read", "Lines parsed", "Lines unparsed", "Lines poured to bucket", "Lines whitelisted")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
keys := []string{"reads", "parsed", "unparsed", "pour"}
keys := []string{"reads", "parsed", "unparsed", "pour", "whitelisted"}
if numRows, err := metricsToTable(t, stats, keys); err != nil {
if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
log.Warningf("while collecting acquis stats: %s", err)
} else if numRows > 0 {
renderTableTitle(out, "\nAcquisition Metrics:")
} else if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func parserStatsTable(out io.Writer, stats map[string]map[string]int) {
func (s statAppsecEngine) Description() (string, string) {
return "Appsec Metrics",
`Measures the number of parsed and blocked requests by the AppSec Component.`
}
func (s statAppsecEngine) Process(appsecEngine, metric string, val int) {
if _, ok := s[appsecEngine]; !ok {
s[appsecEngine] = make(map[string]int)
}
s[appsecEngine][metric] += val
}
func (s statAppsecEngine) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Appsec Engine", "Processed", "Blocked")
t.SetAlignment(table.AlignLeft, table.AlignLeft)
keys := []string{"processed", "blocked"}
if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
log.Warningf("while collecting appsec stats: %s", err)
} else if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func (s statAppsecRule) Description() (string, string) {
return "Appsec Rule Metrics",
`Provides “per AppSec Component” information about the number of matches for loaded AppSec Rules.`
}
func (s statAppsecRule) Process(appsecEngine, appsecRule string, metric string, val int) {
if _, ok := s[appsecEngine]; !ok {
s[appsecEngine] = make(map[string]map[string]int)
}
if _, ok := s[appsecEngine][appsecRule]; !ok {
s[appsecEngine][appsecRule] = make(map[string]int)
}
s[appsecEngine][appsecRule][metric] += val
}
func (s statAppsecRule) Table(out io.Writer, noUnit bool, showEmpty bool) {
for appsecEngine, appsecEngineRulesStats := range s {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Rule ID", "Triggered")
t.SetAlignment(table.AlignLeft, table.AlignLeft)
keys := []string{"triggered"}
if numRows, err := metricsToTable(t, appsecEngineRulesStats, keys, noUnit); err != nil {
log.Warningf("while collecting appsec rules stats: %s", err)
} else if numRows > 0 || showEmpty {
renderTableTitle(out, fmt.Sprintf("\nAppsec '%s' Rules Metrics:", appsecEngine))
t.Render()
}
}
}
func (s statWhitelist) Description() (string, string) {
return "Whitelist Metrics",
`Tracks the number of events processed and possibly whitelisted by each parser whitelist.`
}
func (s statWhitelist) Process(whitelist, reason, metric string, val int) {
if _, ok := s[whitelist]; !ok {
s[whitelist] = make(map[string]map[string]int)
}
if _, ok := s[whitelist][reason]; !ok {
s[whitelist][reason] = make(map[string]int)
}
s[whitelist][reason][metric] += val
}
func (s statWhitelist) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Whitelist", "Reason", "Hits", "Whitelisted")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
if numRows, err := wlMetricsToTable(t, s, noUnit); err != nil {
log.Warningf("while collecting parsers stats: %s", err)
} else if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func (s statParser) Description() (string, string) {
return "Parser Metrics",
`Tracks the number of events processed by each parser and indicates success of failure. ` +
`Zero parsed lines means the parer(s) failed. ` +
`Non-zero unparsed lines are fine as crowdsec select relevant lines.`
}
func (s statParser) Process(parser, metric string, val int) {
if _, ok := s[parser]; !ok {
s[parser] = make(map[string]int)
}
s[parser][metric] += val
}
func (s statParser) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Parsers", "Hits", "Parsed", "Unparsed")
@ -121,187 +310,296 @@ func parserStatsTable(out io.Writer, stats map[string]map[string]int) {
keys := []string{"hits", "parsed", "unparsed"}
if numRows, err := metricsToTable(t, stats, keys); err != nil {
log.Warningf("while collecting acquis stats: %s", err)
} else if numRows > 0 {
renderTableTitle(out, "\nParser Metrics:")
if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
log.Warningf("while collecting parsers stats: %s", err)
} else if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func stashStatsTable(out io.Writer, stats map[string]struct {
Type string
Count int
}) {
func (s statStash) Description() (string, string) {
return "Parser Stash Metrics",
`Tracks the status of stashes that might be created by various parsers and scenarios.`
}
func (s statStash) Process(name, mtype string, val int) {
s[name] = struct {
Type string
Count int
}{
Type: mtype,
Count: val,
}
}
func (s statStash) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Name", "Type", "Items")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
// unfortunately, we can't reuse metricsToTable as the structure is too different :/
sortedKeys := []string{}
for k := range stats {
sortedKeys = append(sortedKeys, k)
}
sort.Strings(sortedKeys)
numRows := 0
for _, alabel := range sortedKeys {
astats := stats[alabel]
for _, alabel := range maptools.SortedKeys(s) {
astats := s[alabel]
row := []string{
alabel,
astats.Type,
fmt.Sprintf("%d", astats.Count),
strconv.Itoa(astats.Count),
}
t.AddRow(row...)
numRows++
}
if numRows > 0 {
renderTableTitle(out, "\nParser Stash Metrics:")
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func lapiStatsTable(out io.Writer, stats map[string]map[string]int) {
func (s statLapi) Description() (string, string) {
return "Local API Metrics",
`Monitors the requests made to local API routes.`
}
func (s statLapi) Process(route, method string, val int) {
if _, ok := s[route]; !ok {
s[route] = make(map[string]int)
}
s[route][method] += val
}
func (s statLapi) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Route", "Method", "Hits")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
// unfortunately, we can't reuse metricsToTable as the structure is too different :/
sortedKeys := []string{}
for k := range stats {
sortedKeys = append(sortedKeys, k)
}
sort.Strings(sortedKeys)
numRows := 0
for _, alabel := range sortedKeys {
astats := stats[alabel]
for _, alabel := range maptools.SortedKeys(s) {
astats := s[alabel]
subKeys := []string{}
for skey := range astats {
subKeys = append(subKeys, skey)
}
sort.Strings(subKeys)
for _, sl := range subKeys {
row := []string{
alabel,
sl,
fmt.Sprintf("%d", astats[sl]),
strconv.Itoa(astats[sl]),
}
t.AddRow(row...)
numRows++
}
}
if numRows > 0 {
renderTableTitle(out, "\nLocal API Metrics:")
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func lapiMachineStatsTable(out io.Writer, stats map[string]map[string]map[string]int) {
func (s statLapiMachine) Description() (string, string) {
return "Local API Machines Metrics",
`Tracks the number of calls to the local API from each registered machine.`
}
func (s statLapiMachine) Process(machine, route, method string, val int) {
if _, ok := s[machine]; !ok {
s[machine] = make(map[string]map[string]int)
}
if _, ok := s[machine][route]; !ok {
s[machine][route] = make(map[string]int)
}
s[machine][route][method] += val
}
func (s statLapiMachine) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Machine", "Route", "Method", "Hits")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
numRows := lapiMetricsToTable(t, stats)
numRows := lapiMetricsToTable(t, s)
if numRows > 0 {
renderTableTitle(out, "\nLocal API Machines Metrics:")
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func lapiBouncerStatsTable(out io.Writer, stats map[string]map[string]map[string]int) {
func (s statLapiBouncer) Description() (string, string) {
return "Local API Bouncers Metrics",
`Tracks total hits to remediation component related API routes.`
}
func (s statLapiBouncer) Process(bouncer, route, method string, val int) {
if _, ok := s[bouncer]; !ok {
s[bouncer] = make(map[string]map[string]int)
}
if _, ok := s[bouncer][route]; !ok {
s[bouncer][route] = make(map[string]int)
}
s[bouncer][route][method] += val
}
func (s statLapiBouncer) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Bouncer", "Route", "Method", "Hits")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
numRows := lapiMetricsToTable(t, stats)
numRows := lapiMetricsToTable(t, s)
if numRows > 0 {
renderTableTitle(out, "\nLocal API Bouncers Metrics:")
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func lapiDecisionStatsTable(out io.Writer, stats map[string]struct {
NonEmpty int
Empty int
},
) {
func (s statLapiDecision) Description() (string, string) {
return "Local API Bouncers Decisions",
`Tracks the number of empty/non-empty answers from LAPI to bouncers that are working in "live" mode.`
}
func (s statLapiDecision) Process(bouncer, fam string, val int) {
if _, ok := s[bouncer]; !ok {
s[bouncer] = struct {
NonEmpty int
Empty int
}{}
}
x := s[bouncer]
switch fam {
case "cs_lapi_decisions_ko_total":
x.Empty += val
case "cs_lapi_decisions_ok_total":
x.NonEmpty += val
}
s[bouncer] = x
}
func (s statLapiDecision) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Bouncer", "Empty answers", "Non-empty answers")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
numRows := 0
for bouncer, hits := range stats {
for bouncer, hits := range s {
t.AddRow(
bouncer,
fmt.Sprintf("%d", hits.Empty),
fmt.Sprintf("%d", hits.NonEmpty),
strconv.Itoa(hits.Empty),
strconv.Itoa(hits.NonEmpty),
)
numRows++
}
if numRows > 0 {
renderTableTitle(out, "\nLocal API Bouncers Decisions:")
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func decisionStatsTable(out io.Writer, stats map[string]map[string]map[string]int) {
func (s statDecision) Description() (string, string) {
return "Local API Decisions",
`Provides information about all currently active decisions. ` +
`Includes both local (crowdsec) and global decisions (CAPI), and lists subscriptions (lists).`
}
func (s statDecision) Process(reason, origin, action string, val int) {
if _, ok := s[reason]; !ok {
s[reason] = make(map[string]map[string]int)
}
if _, ok := s[reason][origin]; !ok {
s[reason][origin] = make(map[string]int)
}
s[reason][origin][action] += val
}
func (s statDecision) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Reason", "Origin", "Action", "Count")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
numRows := 0
for reason, origins := range stats {
for reason, origins := range s {
for origin, actions := range origins {
for action, hits := range actions {
t.AddRow(
reason,
origin,
action,
fmt.Sprintf("%d", hits),
strconv.Itoa(hits),
)
numRows++
}
}
}
if numRows > 0 {
renderTableTitle(out, "\nLocal API Decisions:")
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func alertStatsTable(out io.Writer, stats map[string]int) {
func (s statAlert) Description() (string, string) {
return "Local API Alerts",
`Tracks the total number of past and present alerts for the installed scenarios.`
}
func (s statAlert) Process(reason string, val int) {
s[reason] += val
}
func (s statAlert) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Reason", "Count")
t.SetAlignment(table.AlignLeft, table.AlignLeft)
numRows := 0
for scenario, hits := range stats {
for scenario, hits := range s {
t.AddRow(
scenario,
fmt.Sprintf("%d", hits),
strconv.Itoa(hits),
)
numRows++
}
if numRows > 0 {
renderTableTitle(out, "\nLocal API Alerts:")
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}

230
cmd/crowdsec-cli/notifications.go
View file

@ -23,14 +23,13 @@ import (
"github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/csplugin"
"github.com/crowdsecurity/crowdsec/pkg/csprofiles"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
type NotificationsCfg struct {
@ -39,22 +38,33 @@ type NotificationsCfg struct {
ids []uint
}
func NewNotificationsCmd() *cobra.Command {
var cmdNotifications = &cobra.Command{
type cliNotifications struct {
cfg configGetter
}
func NewCLINotifications(cfg configGetter) *cliNotifications {
return &cliNotifications{
cfg: cfg,
}
}
func (cli *cliNotifications) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "notifications [action]",
Short: "Helper for notification plugin configuration",
Long: "To list/inspect/test notification template",
Args: cobra.MinimumNArgs(1),
Aliases: []string{"notifications", "notification"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.LAPI(csConfig); err != nil {
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := require.LAPI(cfg); err != nil {
return err
}
if err := require.Profiles(csConfig); err != nil {
return err
if err := cfg.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
}
if err := require.Notifications(csConfig); err != nil {
if err := require.Notifications(cfg); err != nil {
return err
}
@ -62,108 +72,110 @@ func NewNotificationsCmd() *cobra.Command {
},
}
cmdNotifications.AddCommand(NewNotificationsListCmd())
cmdNotifications.AddCommand(NewNotificationsInspectCmd())
cmdNotifications.AddCommand(NewNotificationsReinjectCmd())
cmdNotifications.AddCommand(NewNotificationsTestCmd())
cmd.AddCommand(cli.NewListCmd())
cmd.AddCommand(cli.NewInspectCmd())
cmd.AddCommand(cli.NewReinjectCmd())
cmd.AddCommand(cli.NewTestCmd())
return cmdNotifications
return cmd
}
func getPluginConfigs() (map[string]csplugin.PluginConfig, error) {
func (cli *cliNotifications) getPluginConfigs() (map[string]csplugin.PluginConfig, error) {
cfg := cli.cfg()
pcfgs := map[string]csplugin.PluginConfig{}
wf := func(path string, info fs.FileInfo, err error) error {
if info == nil {
return fmt.Errorf("error while traversing directory %s: %w", path, err)
}
name := filepath.Join(csConfig.ConfigPaths.NotificationDir, info.Name()) //Avoid calling info.Name() twice
name := filepath.Join(cfg.ConfigPaths.NotificationDir, info.Name()) //Avoid calling info.Name() twice
if (strings.HasSuffix(name, "yaml") || strings.HasSuffix(name, "yml")) && !(info.IsDir()) {
ts, err := csplugin.ParsePluginConfigFile(name)
if err != nil {
return fmt.Errorf("loading notifification plugin configuration with %s: %w", name, err)
}
for _, t := range ts {
csplugin.SetRequiredFields(&t)
pcfgs[t.Name] = t
}
}
return nil
}
if err := filepath.Walk(csConfig.ConfigPaths.NotificationDir, wf); err != nil {
if err := filepath.Walk(cfg.ConfigPaths.NotificationDir, wf); err != nil {
return nil, fmt.Errorf("while loading notifification plugin configuration: %w", err)
}
return pcfgs, nil
}
func getProfilesConfigs() (map[string]NotificationsCfg, error) {
func (cli *cliNotifications) getProfilesConfigs() (map[string]NotificationsCfg, error) {
cfg := cli.cfg()
// A bit of a tricky stuf now: reconcile profiles and notification plugins
pcfgs, err := getPluginConfigs()
pcfgs, err := cli.getPluginConfigs()
if err != nil {
return nil, err
}
ncfgs := map[string]NotificationsCfg{}
profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
for _, pc := range pcfgs {
ncfgs[pc.Name] = NotificationsCfg{
Config: pc,
}
}
profiles, err := csprofiles.NewProfile(cfg.API.Server.Profiles)
if err != nil {
return nil, fmt.Errorf("while extracting profiles from configuration: %w", err)
}
for profileID, profile := range profiles {
loop:
for _, notif := range profile.Cfg.Notifications {
for name, pc := range pcfgs {
if notif == name {
if _, ok := ncfgs[pc.Name]; !ok {
ncfgs[pc.Name] = NotificationsCfg{
Config: pc,
Profiles: []*csconfig.ProfileCfg{profile.Cfg},
ids: []uint{uint(profileID)},
}
continue loop
}
tmp := ncfgs[pc.Name]
for _, pr := range tmp.Profiles {
var profiles []*csconfig.ProfileCfg
if pr.Name == profile.Cfg.Name {
continue
}
profiles = append(tmp.Profiles, profile.Cfg)
ids := append(tmp.ids, uint(profileID))
ncfgs[pc.Name] = NotificationsCfg{
Config: tmp.Config,
Profiles: profiles,
ids: ids,
}
}
}
pc, ok := pcfgs[notif]
if !ok {
return nil, fmt.Errorf("notification plugin '%s' does not exist", notif)
}
tmp, ok := ncfgs[pc.Name]
if !ok {
return nil, fmt.Errorf("notification plugin '%s' does not exist", pc.Name)
}
tmp.Profiles = append(tmp.Profiles, profile.Cfg)
tmp.ids = append(tmp.ids, uint(profileID))
ncfgs[pc.Name] = tmp
}
}
return ncfgs, nil
}
func NewNotificationsListCmd() *cobra.Command {
var cmdNotificationsList = &cobra.Command{
func (cli *cliNotifications) NewListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "list",
Short: "list active notifications plugins",
Long: `list active notifications plugins`,
Example: `cscli notifications list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, arg []string) error {
ncfgs, err := getProfilesConfigs()
RunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
ncfgs, err := cli.getProfilesConfigs()
if err != nil {
return fmt.Errorf("can't build profiles configuration: %w", err)
}
if csConfig.Cscli.Output == "human" {
if cfg.Cscli.Output == "human" {
notificationListTable(color.Output, ncfgs)
} else if csConfig.Cscli.Output == "json" {
} else if cfg.Cscli.Output == "json" {
x, err := json.MarshalIndent(ncfgs, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal notification configuration: %w", err)
}
fmt.Printf("%s", string(x))
} else if csConfig.Cscli.Output == "raw" {
} else if cfg.Cscli.Output == "raw" {
csvwriter := csv.NewWriter(os.Stdout)
err := csvwriter.Write([]string{"Name", "Type", "Profile name"})
if err != nil {
@ -181,90 +193,89 @@ func NewNotificationsListCmd() *cobra.Command {
}
csvwriter.Flush()
}
return nil
},
}
return cmdNotificationsList
return cmd
}
func NewNotificationsInspectCmd() *cobra.Command {
var cmdNotificationsInspect = &cobra.Command{
func (cli *cliNotifications) NewInspectCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "inspect",
Short: "Inspect active notifications plugin configuration",
Long: `Inspect active notifications plugin and show configuration`,
Example: `cscli notifications inspect <plugin_name>`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
PreRunE: func(cmd *cobra.Command, args []string) error {
if args[0] == "" {
return fmt.Errorf("please provide a plugin name to inspect")
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
ncfgs, err := getProfilesConfigs()
RunE: func(_ *cobra.Command, args []string) error {
cfg := cli.cfg()
ncfgs, err := cli.getProfilesConfigs()
if err != nil {
return fmt.Errorf("can't build profiles configuration: %w", err)
}
cfg, ok := ncfgs[args[0]]
ncfg, ok := ncfgs[args[0]]
if !ok {
return fmt.Errorf("plugin '%s' does not exist or is not active", args[0])
}
if csConfig.Cscli.Output == "human" || csConfig.Cscli.Output == "raw" {
fmt.Printf(" - %15s: %15s\n", "Type", cfg.Config.Type)
fmt.Printf(" - %15s: %15s\n", "Name", cfg.Config.Name)
fmt.Printf(" - %15s: %15s\n", "Timeout", cfg.Config.TimeOut)
fmt.Printf(" - %15s: %15s\n", "Format", cfg.Config.Format)
for k, v := range cfg.Config.Config {
if cfg.Cscli.Output == "human" || cfg.Cscli.Output == "raw" {
fmt.Printf(" - %15s: %15s\n", "Type", ncfg.Config.Type)
fmt.Printf(" - %15s: %15s\n", "Name", ncfg.Config.Name)
fmt.Printf(" - %15s: %15s\n", "Timeout", ncfg.Config.TimeOut)
fmt.Printf(" - %15s: %15s\n", "Format", ncfg.Config.Format)
for k, v := range ncfg.Config.Config {
fmt.Printf(" - %15s: %15v\n", k, v)
}
} else if csConfig.Cscli.Output == "json" {
} else if cfg.Cscli.Output == "json" {
x, err := json.MarshalIndent(cfg, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal notification configuration: %w", err)
}
fmt.Printf("%s", string(x))
}
return nil
},
}
return cmdNotificationsInspect
return cmd
}
func NewNotificationsTestCmd() *cobra.Command {
func (cli *cliNotifications) NewTestCmd() *cobra.Command {
var (
pluginBroker csplugin.PluginBroker
pluginTomb tomb.Tomb
alertOverride string
)
var cmdNotificationsTest = &cobra.Command{
cmd := &cobra.Command{
Use: "test [plugin name]",
Short: "send a generic test alert to notification plugin",
Long: `send a generic test alert to a notification plugin to test configuration even if is not active`,
Example: `cscli notifications test [plugin_name]`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
PreRunE: func(cmd *cobra.Command, args []string) error {
pconfigs, err := getPluginConfigs()
PreRunE: func(_ *cobra.Command, args []string) error {
cfg := cli.cfg()
pconfigs, err := cli.getPluginConfigs()
if err != nil {
return fmt.Errorf("can't build profiles configuration: %w", err)
}
cfg, ok := pconfigs[args[0]]
pcfg, ok := pconfigs[args[0]]
if !ok {
return fmt.Errorf("plugin name: '%s' does not exist", args[0])
}
//Create a single profile with plugin name as notification name
return pluginBroker.Init(csConfig.PluginConfig, []*csconfig.ProfileCfg{
return pluginBroker.Init(cfg.PluginConfig, []*csconfig.ProfileCfg{
{
Notifications: []string{
cfg.Name,
pcfg.Name,
},
},
}, csConfig.ConfigPaths)
}, cfg.ConfigPaths)
},
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
pluginTomb.Go(func() error {
pluginBroker.Run(&pluginTomb)
return nil
@ -303,26 +314,31 @@ func NewNotificationsTestCmd() *cobra.Command {
if err := yaml.Unmarshal([]byte(alertOverride), alert); err != nil {
return fmt.Errorf("failed to unmarshal alert override: %w", err)
}
pluginBroker.PluginChannel <- csplugin.ProfileAlert{
ProfileID: uint(0),
Alert: alert,
}
//time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
pluginTomb.Kill(fmt.Errorf("terminating"))
pluginTomb.Wait()
return nil
},
}
cmdNotificationsTest.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the generic alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
cmd.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the generic alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
return cmdNotificationsTest
return cmd
}
func NewNotificationsReinjectCmd() *cobra.Command {
var alertOverride string
var alert *models.Alert
func (cli *cliNotifications) NewReinjectCmd() *cobra.Command {
var (
alertOverride string
alert *models.Alert
)
var cmdNotificationsReinject = &cobra.Command{
cmd := &cobra.Command{
Use: "reinject",
Short: "reinject an alert into profiles to trigger notifications",
Long: `reinject an alert into profiles to be evaluated by the filter and sent to matched notifications plugins`,
@ -333,25 +349,30 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
PreRunE: func(cmd *cobra.Command, args []string) error {
PreRunE: func(_ *cobra.Command, args []string) error {
var err error
alert, err = FetchAlertFromArgString(args[0])
alert, err = cli.fetchAlertFromArgString(args[0])
if err != nil {
return err
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
var (
pluginBroker csplugin.PluginBroker
pluginTomb tomb.Tomb
)
cfg := cli.cfg()
if alertOverride != "" {
if err := json.Unmarshal([]byte(alertOverride), alert); err != nil {
return fmt.Errorf("can't unmarshal data in the alert flag: %w", err)
}
}
err := pluginBroker.Init(csConfig.PluginConfig, csConfig.API.Server.Profiles, csConfig.ConfigPaths)
err := pluginBroker.Init(cfg.PluginConfig, cfg.API.Server.Profiles, cfg.ConfigPaths)
if err != nil {
return fmt.Errorf("can't initialize plugins: %w", err)
}
@ -361,7 +382,7 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
return nil
})
profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
profiles, err := csprofiles.NewProfile(cfg.API.Server.Profiles)
if err != nil {
return fmt.Errorf("cannot extract profiles from configuration: %w", err)
}
@ -387,9 +408,9 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
default:
time.Sleep(50 * time.Millisecond)
log.Info("sleeping\n")
}
}
if profile.Cfg.OnSuccess == "break" {
log.Infof("The profile %s contains a 'on_success: break' so bailing out", profile.Cfg.Name)
break
@ -398,26 +419,31 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
//time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
pluginTomb.Kill(fmt.Errorf("terminating"))
pluginTomb.Wait()
return nil
},
}
cmdNotificationsReinject.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the reinjected alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
cmd.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the reinjected alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
return cmdNotificationsReinject
return cmd
}
func FetchAlertFromArgString(toParse string) (*models.Alert, error) {
func (cli *cliNotifications) fetchAlertFromArgString(toParse string) (*models.Alert, error) {
cfg := cli.cfg()
id, err := strconv.Atoi(toParse)
if err != nil {
return nil, fmt.Errorf("bad alert id %s", toParse)
}
apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
apiURL, err := url.Parse(cfg.API.Client.Credentials.URL)
if err != nil {
return nil, fmt.Errorf("error parsing the URL of the API: %w", err)
}
client, err := apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
Password: strfmt.Password(csConfig.API.Client.Credentials.Password),
MachineID: cfg.API.Client.Credentials.Login,
Password: strfmt.Password(cfg.API.Client.Credentials.Password),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
@ -425,9 +451,11 @@ func FetchAlertFromArgString(toParse string) (*models.Alert, error) {
if err != nil {
return nil, fmt.Errorf("error creating the client for the API: %w", err)
}
alert, _, err := client.Alerts.GetByID(context.Background(), id)
if err != nil {
return nil, fmt.Errorf("can't find alert with id %d: %w", id, err)
}
return alert, nil
}

26
cmd/crowdsec-cli/notifications_table.go
View file

@ -2,24 +2,36 @@ package main
import (
"io"
"sort"
"strings"
"github.com/aquasecurity/table"
"github.com/enescakir/emoji"
)
func notificationListTable(out io.Writer, ncfgs map[string]NotificationsCfg) {
t := newLightTable(out)
t.SetHeaders("Name", "Type", "Profile name")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
for _, b := range ncfgs {
t.SetHeaders("Active", "Name", "Type", "Profile name")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
keys := make([]string, 0, len(ncfgs))
for k := range ncfgs {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool {
return len(ncfgs[keys[i]].Profiles) > len(ncfgs[keys[j]].Profiles)
})
for _, k := range keys {
b := ncfgs[k]
profilesList := []string{}
for _, p := range b.Profiles {
profilesList = append(profilesList, p.Name)
}
t.AddRow(b.Config.Name, b.Config.Type, strings.Join(profilesList, ", "))
active := emoji.CheckMark.String()
if len(profilesList) == 0 {
active = emoji.Prohibited.String()
}
t.AddRow(active, b.Config.Name, b.Config.Type, strings.Join(profilesList, ", "))
}
t.Render()
}

87
cmd/crowdsec-cli/papi.go
View file

@ -1,6 +1,7 @@
package main
import (
"fmt"
"time"
log "github.com/sirupsen/logrus"
@ -9,67 +10,79 @@ import (
"github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiserver"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
func NewPapiCmd() *cobra.Command {
var cmdLapi = &cobra.Command{
type cliPapi struct {
cfg configGetter
}
func NewCLIPapi(cfg configGetter) *cliPapi {
return &cliPapi{
cfg: cfg,
}
}
func (cli *cliPapi) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "papi [action]",
Short: "Manage interaction with Polling API (PAPI)",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.LAPI(csConfig); err != nil {
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := require.LAPI(cfg); err != nil {
return err
}
if err := require.CAPI(csConfig); err != nil {
if err := require.CAPI(cfg); err != nil {
return err
}
if err := require.PAPI(csConfig); err != nil {
if err := require.PAPI(cfg); err != nil {
return err
}
return nil
},
}
cmdLapi.AddCommand(NewPapiStatusCmd())
cmdLapi.AddCommand(NewPapiSyncCmd())
cmd.AddCommand(cli.NewStatusCmd())
cmd.AddCommand(cli.NewSyncCmd())
return cmdLapi
return cmd
}
func NewPapiStatusCmd() *cobra.Command {
cmdCapiStatus := &cobra.Command{
func (cli *cliPapi) NewStatusCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "status",
Short: "Get status of the Polling API",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(_ *cobra.Command, _ []string) error {
var err error
dbClient, err = database.NewClient(csConfig.DbConfig)
cfg := cli.cfg()
dbClient, err = database.NewClient(cfg.DbConfig)
if err != nil {
log.Fatalf("unable to initialize database client : %s", err)
return fmt.Errorf("unable to initialize database client: %s", err)
}
apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig, csConfig.API.Server.CapiWhitelists)
apic, err := apiserver.NewAPIC(cfg.API.Server.OnlineClient, dbClient, cfg.API.Server.ConsoleConfig, cfg.API.Server.CapiWhitelists)
if err != nil {
log.Fatalf("unable to initialize API client : %s", err)
return fmt.Errorf("unable to initialize API client: %s", err)
}
papi, err := apiserver.NewPAPI(apic, dbClient, csConfig.API.Server.ConsoleConfig, log.GetLevel())
papi, err := apiserver.NewPAPI(apic, dbClient, cfg.API.Server.ConsoleConfig, log.GetLevel())
if err != nil {
log.Fatalf("unable to initialize PAPI client : %s", err)
return fmt.Errorf("unable to initialize PAPI client: %s", err)
}
perms, err := papi.GetPermissions()
if err != nil {
log.Fatalf("unable to get PAPI permissions: %s", err)
return fmt.Errorf("unable to get PAPI permissions: %s", err)
}
var lastTimestampStr *string
lastTimestampStr, err = dbClient.GetConfigItem(apiserver.PapiPullKey)
@ -84,45 +97,48 @@ func NewPapiStatusCmd() *cobra.Command {
for _, sub := range perms.Categories {
log.Infof(" - %s", sub)
}
return nil
},
}
return cmdCapiStatus
return cmd
}
func NewPapiSyncCmd() *cobra.Command {
cmdCapiSync := &cobra.Command{
func (cli *cliPapi) NewSyncCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "sync",
Short: "Sync with the Polling API, pulling all non-expired orders for the instance",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(_ *cobra.Command, _ []string) error {
var err error
cfg := cli.cfg()
t := tomb.Tomb{}
dbClient, err = database.NewClient(csConfig.DbConfig)
dbClient, err = database.NewClient(cfg.DbConfig)
if err != nil {
log.Fatalf("unable to initialize database client : %s", err)
return fmt.Errorf("unable to initialize database client: %s", err)
}
apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig, csConfig.API.Server.CapiWhitelists)
apic, err := apiserver.NewAPIC(cfg.API.Server.OnlineClient, dbClient, cfg.API.Server.ConsoleConfig, cfg.API.Server.CapiWhitelists)
if err != nil {
log.Fatalf("unable to initialize API client : %s", err)
return fmt.Errorf("unable to initialize API client: %s", err)
}
t.Go(apic.Push)
papi, err := apiserver.NewPAPI(apic, dbClient, csConfig.API.Server.ConsoleConfig, log.GetLevel())
papi, err := apiserver.NewPAPI(apic, dbClient, cfg.API.Server.ConsoleConfig, log.GetLevel())
if err != nil {
log.Fatalf("unable to initialize PAPI client : %s", err)
return fmt.Errorf("unable to initialize PAPI client: %s", err)
}
t.Go(papi.SyncDecisions)
err = papi.PullOnce(time.Time{}, true)
if err != nil {
log.Fatalf("unable to sync decisions: %s", err)
return fmt.Errorf("unable to sync decisions: %s", err)
}
log.Infof("Sending acknowledgements to CAPI")
@ -132,8 +148,9 @@ func NewPapiSyncCmd() *cobra.Command {
t.Wait()
time.Sleep(5 * time.Second) //FIXME: the push done by apic.Push is run inside a sub goroutine, sleep to make sure it's done
return nil
},
}
return cmdCapiSync
return cmd
}

4
cmd/crowdsec-cli/require/branch.go
View file

@ -56,3 +56,7 @@ func HubBranch(cfg *csconfig.Config) string {
return branch
}
func HubURLTemplate(cfg *csconfig.Config) string {
return cfg.Cscli.HubURLTemplate
}

32
cmd/crowdsec-cli/require/require.go
View file

@ -2,13 +2,16 @@ package require
import (
"fmt"
"io"
"github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func LAPI(c *csconfig.Config) error {
if err := c.LoadAPIServer(); err != nil {
if err := c.LoadAPIServer(true); err != nil {
return fmt.Errorf("failed to load Local API: %w", err)
}
@ -44,21 +47,13 @@ func CAPIRegistered(c *csconfig.Config) error {
}
func DB(c *csconfig.Config) error {
if err := c.LoadDBConfig(); err != nil {
if err := c.LoadDBConfig(true); err != nil {
return fmt.Errorf("this command requires direct database access (must be run on the local API machine): %w", err)
}
return nil
}
func Profiles(c *csconfig.Config) error {
if err := c.API.Server.LoadProfiles(); err != nil {
return fmt.Errorf("while loading profiles: %w", err)
}
return nil
}
func Notifications(c *csconfig.Config) error {
if c.ConfigPaths.NotificationDir == "" {
return fmt.Errorf("config_paths.notification_dir is not set in crowdsec config")
@ -71,10 +66,10 @@ func Notifications(c *csconfig.Config) error {
func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg {
// set branch in config, and log if necessary
branch := HubBranch(c)
remote := &cwhub.RemoteHubCfg {
Branch: branch,
URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s",
// URLTemplate: "http://localhost:8000/crowdsecurity/%s/hub/%s",
urlTemplate := HubURLTemplate(c)
remote := &cwhub.RemoteHubCfg{
Branch: branch,
URLTemplate: urlTemplate,
IndexPath: ".index.json",
}
@ -83,14 +78,19 @@ func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg {
// Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items.
// If no remote parameter is provided, the hub can only be used for local operations.
func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg) (*cwhub.Hub, error) {
func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg, logger *logrus.Logger) (*cwhub.Hub, error) {
local := c.Hub
if local == nil {
return nil, fmt.Errorf("you must configure cli before interacting with hub")
}
hub, err := cwhub.NewHub(local, remote, false)
if logger == nil {
logger = logrus.New()
logger.SetOutput(io.Discard)
}
hub, err := cwhub.NewHub(local, remote, false, logger)
if err != nil {
return nil, fmt.Errorf("failed to read Hub index: %w. Run 'sudo cscli hub update' to download the index again", err)
}

5
cmd/crowdsec-cli/setup.go
View file

@ -11,10 +11,9 @@ import (
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/setup"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
// NewSetupCmd defines the "cscli setup" command.
@ -305,7 +304,7 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error {
return fmt.Errorf("while reading file %s: %w", fromFile, err)
}
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), log.StandardLogger())
if err != nil {
return err
}

302
cmd/crowdsec-cli/simulation.go
View file

@ -3,148 +3,72 @@ package main
import (
"fmt"
"os"
"slices"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"slices"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func addToExclusion(name string) error {
csConfig.Cscli.SimulationConfig.Exclusions = append(csConfig.Cscli.SimulationConfig.Exclusions, name)
return nil
type cliSimulation struct {
cfg configGetter
}
func removeFromExclusion(name string) error {
index := slices.Index(csConfig.Cscli.SimulationConfig.Exclusions, name)
// Remove element from the slice
csConfig.Cscli.SimulationConfig.Exclusions[index] = csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1] = ""
csConfig.Cscli.SimulationConfig.Exclusions = csConfig.Cscli.SimulationConfig.Exclusions[:len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
return nil
func NewCLISimulation(cfg configGetter) *cliSimulation {
return &cliSimulation{
cfg: cfg,
}
}
func enableGlobalSimulation() error {
csConfig.Cscli.SimulationConfig.Simulation = new(bool)
*csConfig.Cscli.SimulationConfig.Simulation = true
csConfig.Cscli.SimulationConfig.Exclusions = []string{}
if err := dumpSimulationFile(); err != nil {
log.Fatalf("unable to dump simulation file: %s", err)
}
log.Printf("global simulation: enabled")
return nil
}
func dumpSimulationFile() error {
newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
if err != nil {
return fmt.Errorf("unable to marshal simulation configuration: %s", err)
}
err = os.WriteFile(csConfig.ConfigPaths.SimulationFilePath, newConfigSim, 0644)
if err != nil {
return fmt.Errorf("write simulation config in '%s' failed: %s", csConfig.ConfigPaths.SimulationFilePath, err)
}
log.Debugf("updated simulation file %s", csConfig.ConfigPaths.SimulationFilePath)
return nil
}
func disableGlobalSimulation() error {
csConfig.Cscli.SimulationConfig.Simulation = new(bool)
*csConfig.Cscli.SimulationConfig.Simulation = false
csConfig.Cscli.SimulationConfig.Exclusions = []string{}
newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
if err != nil {
return fmt.Errorf("unable to marshal new simulation configuration: %s", err)
}
err = os.WriteFile(csConfig.ConfigPaths.SimulationFilePath, newConfigSim, 0644)
if err != nil {
return fmt.Errorf("unable to write new simulation config in '%s' : %s", csConfig.ConfigPaths.SimulationFilePath, err)
}
log.Printf("global simulation: disabled")
return nil
}
func simulationStatus() error {
if csConfig.Cscli.SimulationConfig == nil {
log.Printf("global simulation: disabled (configuration file is missing)")
return nil
}
if *csConfig.Cscli.SimulationConfig.Simulation {
log.Println("global simulation: enabled")
if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
log.Println("Scenarios not in simulation mode :")
for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
log.Printf(" - %s", scenario)
}
}
} else {
log.Println("global simulation: disabled")
if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
log.Println("Scenarios in simulation mode :")
for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
log.Printf(" - %s", scenario)
}
}
}
return nil
}
func NewSimulationCmds() *cobra.Command {
var cmdSimulation = &cobra.Command{
func (cli *cliSimulation) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "simulation [command]",
Short: "Manage simulation status of scenarios",
Example: `cscli simulation status
cscli simulation enable crowdsecurity/ssh-bf
cscli simulation disable crowdsecurity/ssh-bf`,
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadSimulation(); err != nil {
log.Fatal(err)
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
if err := cli.cfg().LoadSimulation(); err != nil {
return err
}
if csConfig.Cscli.SimulationConfig == nil {
if cli.cfg().Cscli.SimulationConfig == nil {
return fmt.Errorf("no simulation configured")
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
PersistentPostRun: func(cmd *cobra.Command, _ []string) {
if cmd.Name() != "status" {
log.Infof(ReloadMessage())
}
},
}
cmdSimulation.Flags().SortFlags = false
cmdSimulation.PersistentFlags().SortFlags = false
cmd.Flags().SortFlags = false
cmd.PersistentFlags().SortFlags = false
cmdSimulation.AddCommand(NewSimulationEnableCmd())
cmdSimulation.AddCommand(NewSimulationDisableCmd())
cmdSimulation.AddCommand(NewSimulationStatusCmd())
cmd.AddCommand(cli.NewEnableCmd())
cmd.AddCommand(cli.NewDisableCmd())
cmd.AddCommand(cli.NewStatusCmd())
return cmdSimulation
return cmd
}
func NewSimulationEnableCmd() *cobra.Command {
func (cli *cliSimulation) NewEnableCmd() *cobra.Command {
var forceGlobalSimulation bool
var cmdSimulationEnable = &cobra.Command{
cmd := &cobra.Command{
Use: "enable [scenario] [-global]",
Short: "Enable the simulation, globally or on specified scenarios",
Example: `cscli simulation enable`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
hub, err := require.Hub(csConfig, nil)
RunE: func(cmd *cobra.Command, args []string) error {
hub, err := require.Hub(cli.cfg(), nil, nil)
if err != nil {
log.Fatal(err)
return err
}
if len(args) > 0 {
@ -157,64 +81,60 @@ func NewSimulationEnableCmd() *cobra.Command {
if !item.State.Installed {
log.Warningf("'%s' isn't enabled", scenario)
}
isExcluded := slices.Contains(csConfig.Cscli.SimulationConfig.Exclusions, scenario)
if *csConfig.Cscli.SimulationConfig.Simulation && !isExcluded {
isExcluded := slices.Contains(cli.cfg().Cscli.SimulationConfig.Exclusions, scenario)
if *cli.cfg().Cscli.SimulationConfig.Simulation && !isExcluded {
log.Warning("global simulation is already enabled")
continue
}
if !*csConfig.Cscli.SimulationConfig.Simulation && isExcluded {
if !*cli.cfg().Cscli.SimulationConfig.Simulation && isExcluded {
log.Warningf("simulation for '%s' already enabled", scenario)
continue
}
if *csConfig.Cscli.SimulationConfig.Simulation && isExcluded {
if err := removeFromExclusion(scenario); err != nil {
log.Fatal(err)
}
if *cli.cfg().Cscli.SimulationConfig.Simulation && isExcluded {
cli.removeFromExclusion(scenario)
log.Printf("simulation enabled for '%s'", scenario)
continue
}
if err := addToExclusion(scenario); err != nil {
log.Fatal(err)
}
cli.addToExclusion(scenario)
log.Printf("simulation mode for '%s' enabled", scenario)
}
if err := dumpSimulationFile(); err != nil {
log.Fatalf("simulation enable: %s", err)
if err := cli.dumpSimulationFile(); err != nil {
return fmt.Errorf("simulation enable: %s", err)
}
} else if forceGlobalSimulation {
if err := enableGlobalSimulation(); err != nil {
log.Fatalf("unable to enable global simulation mode : %s", err)
if err := cli.enableGlobalSimulation(); err != nil {
return fmt.Errorf("unable to enable global simulation mode: %s", err)
}
} else {
printHelp(cmd)
}
return nil
},
}
cmdSimulationEnable.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Enable global simulation (reverse mode)")
cmd.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Enable global simulation (reverse mode)")
return cmdSimulationEnable
return cmd
}
func NewSimulationDisableCmd() *cobra.Command {
func (cli *cliSimulation) NewDisableCmd() *cobra.Command {
var forceGlobalSimulation bool
var cmdSimulationDisable = &cobra.Command{
cmd := &cobra.Command{
Use: "disable [scenario]",
Short: "Disable the simulation mode. Disable only specified scenarios",
Example: `cscli simulation disable`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) > 0 {
for _, scenario := range args {
isExcluded := slices.Contains(csConfig.Cscli.SimulationConfig.Exclusions, scenario)
if !*csConfig.Cscli.SimulationConfig.Simulation && !isExcluded {
isExcluded := slices.Contains(cli.cfg().Cscli.SimulationConfig.Exclusions, scenario)
if !*cli.cfg().Cscli.SimulationConfig.Simulation && !isExcluded {
log.Warningf("%s isn't in simulation mode", scenario)
continue
}
if !*csConfig.Cscli.SimulationConfig.Simulation && isExcluded {
if err := removeFromExclusion(scenario); err != nil {
log.Fatal(err)
}
if !*cli.cfg().Cscli.SimulationConfig.Simulation && isExcluded {
cli.removeFromExclusion(scenario)
log.Printf("simulation mode for '%s' disabled", scenario)
continue
}
@ -222,42 +142,138 @@ func NewSimulationDisableCmd() *cobra.Command {
log.Warningf("simulation mode is enabled but is already disable for '%s'", scenario)
continue
}
if err := addToExclusion(scenario); err != nil {
log.Fatal(err)
}
cli.addToExclusion(scenario)
log.Printf("simulation mode for '%s' disabled", scenario)
}
if err := dumpSimulationFile(); err != nil {
log.Fatalf("simulation disable: %s", err)
if err := cli.dumpSimulationFile(); err != nil {
return fmt.Errorf("simulation disable: %s", err)
}
} else if forceGlobalSimulation {
if err := disableGlobalSimulation(); err != nil {
log.Fatalf("unable to disable global simulation mode : %s", err)
if err := cli.disableGlobalSimulation(); err != nil {
return fmt.Errorf("unable to disable global simulation mode: %s", err)
}
} else {
printHelp(cmd)
}
return nil
},
}
cmdSimulationDisable.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Disable global simulation (reverse mode)")
cmd.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Disable global simulation (reverse mode)")
return cmdSimulationDisable
return cmd
}
func NewSimulationStatusCmd() *cobra.Command {
var cmdSimulationStatus = &cobra.Command{
func (cli *cliSimulation) NewStatusCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "status",
Short: "Show simulation mode status",
Example: `cscli simulation status`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if err := simulationStatus(); err != nil {
log.Fatal(err)
}
Run: func(_ *cobra.Command, _ []string) {
cli.status()
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
},
}
return cmdSimulationStatus
return cmd
}
func (cli *cliSimulation) addToExclusion(name string) {
cfg := cli.cfg()
cfg.Cscli.SimulationConfig.Exclusions = append(cfg.Cscli.SimulationConfig.Exclusions, name)
}
func (cli *cliSimulation) removeFromExclusion(name string) {
cfg := cli.cfg()
index := slices.Index(cfg.Cscli.SimulationConfig.Exclusions, name)
// Remove element from the slice
cfg.Cscli.SimulationConfig.Exclusions[index] = cfg.Cscli.SimulationConfig.Exclusions[len(cfg.Cscli.SimulationConfig.Exclusions)-1]
cfg.Cscli.SimulationConfig.Exclusions[len(cfg.Cscli.SimulationConfig.Exclusions)-1] = ""
cfg.Cscli.SimulationConfig.Exclusions = cfg.Cscli.SimulationConfig.Exclusions[:len(cfg.Cscli.SimulationConfig.Exclusions)-1]
}
func (cli *cliSimulation) enableGlobalSimulation() error {
cfg := cli.cfg()
cfg.Cscli.SimulationConfig.Simulation = new(bool)
*cfg.Cscli.SimulationConfig.Simulation = true
cfg.Cscli.SimulationConfig.Exclusions = []string{}
if err := cli.dumpSimulationFile(); err != nil {
return fmt.Errorf("unable to dump simulation file: %s", err)
}
log.Printf("global simulation: enabled")
return nil
}
func (cli *cliSimulation) dumpSimulationFile() error {
cfg := cli.cfg()
newConfigSim, err := yaml.Marshal(cfg.Cscli.SimulationConfig)
if err != nil {
return fmt.Errorf("unable to marshal simulation configuration: %s", err)
}
err = os.WriteFile(cfg.ConfigPaths.SimulationFilePath, newConfigSim, 0o644)
if err != nil {
return fmt.Errorf("write simulation config in '%s' failed: %s", cfg.ConfigPaths.SimulationFilePath, err)
}
log.Debugf("updated simulation file %s", cfg.ConfigPaths.SimulationFilePath)
return nil
}
func (cli *cliSimulation) disableGlobalSimulation() error {
cfg := cli.cfg()
cfg.Cscli.SimulationConfig.Simulation = new(bool)
*cfg.Cscli.SimulationConfig.Simulation = false
cfg.Cscli.SimulationConfig.Exclusions = []string{}
newConfigSim, err := yaml.Marshal(cfg.Cscli.SimulationConfig)
if err != nil {
return fmt.Errorf("unable to marshal new simulation configuration: %s", err)
}
err = os.WriteFile(cfg.ConfigPaths.SimulationFilePath, newConfigSim, 0o644)
if err != nil {
return fmt.Errorf("unable to write new simulation config in '%s': %s", cfg.ConfigPaths.SimulationFilePath, err)
}
log.Printf("global simulation: disabled")
return nil
}
func (cli *cliSimulation) status() {
cfg := cli.cfg()
if cfg.Cscli.SimulationConfig == nil {
log.Printf("global simulation: disabled (configuration file is missing)")
return
}
if *cfg.Cscli.SimulationConfig.Simulation {
log.Println("global simulation: enabled")
if len(cfg.Cscli.SimulationConfig.Exclusions) > 0 {
log.Println("Scenarios not in simulation mode :")
for _, scenario := range cfg.Cscli.SimulationConfig.Exclusions {
log.Printf(" - %s", scenario)
}
}
} else {
log.Println("global simulation: disabled")
if len(cfg.Cscli.SimulationConfig.Exclusions) > 0 {
log.Println("Scenarios in simulation mode :")
for _, scenario := range cfg.Cscli.SimulationConfig.Exclusions {
log.Printf(" - %s", scenario)
}
}
}
}

87
cmd/crowdsec-cli/support.go
View file

@ -37,6 +37,7 @@ const (
SUPPORT_OS_INFO_PATH = "osinfo.txt"
SUPPORT_PARSERS_PATH = "hub/parsers.txt"
SUPPORT_SCENARIOS_PATH = "hub/scenarios.txt"
SUPPORT_CONTEXTS_PATH = "hub/scenarios.txt"
SUPPORT_COLLECTIONS_PATH = "hub/collections.txt"
SUPPORT_POSTOVERFLOWS_PATH = "hub/postoverflows.txt"
SUPPORT_BOUNCERS_PATH = "lapi/bouncers.txt"
@ -65,19 +66,25 @@ func collectMetrics() ([]byte, []byte, error) {
}
humanMetrics := bytes.NewBuffer(nil)
err := FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl, "human")
if err != nil {
return nil, nil, fmt.Errorf("could not fetch promtheus metrics: %s", err)
ms := NewMetricStore()
if err := ms.Fetch(csConfig.Cscli.PrometheusUrl); err != nil {
return nil, nil, fmt.Errorf("could not fetch prometheus metrics: %s", err)
}
if err := ms.Format(humanMetrics, nil, "human", false); err != nil {
return nil, nil, err
}
req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl, nil)
if err != nil {
return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err)
}
client := &http.Client{}
resp, err := client.Do(req)
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, nil, fmt.Errorf("could not get metrics from prometheus endpoint: %s", err)
}
@ -99,17 +106,20 @@ func collectVersion() []byte {
func collectFeatures() []byte {
log.Info("Collecting feature flags")
enabledFeatures := fflag.Crowdsec.GetEnabledFeatures()
w := bytes.NewBuffer(nil)
for _, k := range enabledFeatures {
fmt.Fprintf(w, "%s\n", k)
}
return w.Bytes()
}
func collectOSInfo() ([]byte, error) {
log.Info("Collecting OS info")
info, err := osinfo.GetOSInfo()
if err != nil {
@ -132,6 +142,7 @@ func collectHubItems(hub *cwhub.Hub, itemType string) []byte {
var err error
out := bytes.NewBuffer(nil)
log.Infof("Collecting %s list", itemType)
items := make(map[string][]*cwhub.Item)
@ -143,24 +154,33 @@ func collectHubItems(hub *cwhub.Hub, itemType string) []byte {
if err := listItems(out, []string{itemType}, items, false); err != nil {
log.Warnf("could not collect %s list: %s", itemType, err)
}
return out.Bytes()
}
func collectBouncers(dbClient *database.Client) ([]byte, error) {
out := bytes.NewBuffer(nil)
err := getBouncers(out, dbClient)
bouncers, err := dbClient.ListBouncers()
if err != nil {
return nil, err
return nil, fmt.Errorf("unable to list bouncers: %s", err)
}
getBouncersTable(out, bouncers)
return out.Bytes(), nil
}
func collectAgents(dbClient *database.Client) ([]byte, error) {
out := bytes.NewBuffer(nil)
err := getAgents(out, dbClient)
machines, err := dbClient.ListMachines()
if err != nil {
return nil, err
return nil, fmt.Errorf("unable to list machines: %s", err)
}
getAgentsTable(out, machines)
return out.Bytes(), nil
}
@ -168,12 +188,14 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil {
return []byte("No agent credentials found, are we LAPI ?")
}
pwd := strfmt.Password(password)
apiurl, err := url.Parse(endpoint)
pwd := strfmt.Password(password)
apiurl, err := url.Parse(endpoint)
if err != nil {
return []byte(fmt.Sprintf("cannot parse API URL: %s", err))
}
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return []byte(fmt.Sprintf("could not collect scenarios: %s", err))
@ -186,6 +208,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
if err != nil {
return []byte(fmt.Sprintf("could not init client: %s", err))
}
t := models.WatcherAuthRequest{
MachineID: &login,
Password: &pwd,
@ -202,6 +225,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
func collectCrowdsecConfig() []byte {
log.Info("Collecting crowdsec config")
config, err := os.ReadFile(*csConfig.FilePath)
if err != nil {
return []byte(fmt.Sprintf("could not read config file: %s", err))
@ -214,15 +238,18 @@ func collectCrowdsecConfig() []byte {
func collectCrowdsecProfile() []byte {
log.Info("Collecting crowdsec profile")
config, err := os.ReadFile(csConfig.API.Server.ProfilesPath)
if err != nil {
return []byte(fmt.Sprintf("could not read profile file: %s", err))
}
return config
}
func collectAcquisitionConfig() map[string][]byte {
log.Info("Collecting acquisition config")
ret := make(map[string][]byte)
for _, filename := range csConfig.Crowdsec.AcquisitionFiles {
@ -237,8 +264,14 @@ func collectAcquisitionConfig() map[string][]byte {
return ret
}
func NewSupportCmd() *cobra.Command {
var cmdSupport = &cobra.Command{
type cliSupport struct{}
func NewCLISupport() *cliSupport {
return &cliSupport{}
}
func (cli cliSupport) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "support [action]",
Short: "Provide commands to help during support",
Args: cobra.MinimumNArgs(1),
@ -248,9 +281,15 @@ func NewSupportCmd() *cobra.Command {
},
}
cmd.AddCommand(cli.NewDumpCmd())
return cmd
}
func (cli cliSupport) NewDumpCmd() *cobra.Command {
var outFile string
cmdDump := &cobra.Command{
cmd := &cobra.Command{
Use: "dump",
Short: "Dump all your configuration to a zip file for easier support",
Long: `Dump the following informations:
@ -260,6 +299,7 @@ func NewSupportCmd() *cobra.Command {
- Installed parsers list
- Installed scenarios list
- Installed postoverflows list
- Installed context list
- Bouncers list
- Machines list
- CAPI status
@ -271,7 +311,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
`,
Args: cobra.NoArgs,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
Run: func(_ *cobra.Command, _ []string) {
var err error
var skipHub, skipDB, skipCAPI, skipLAPI, skipAgent bool
infos := map[string][]byte{
@ -291,24 +331,25 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_AGENTS_PATH] = []byte(err.Error())
}
if err := csConfig.LoadAPIServer(); err != nil {
if err = csConfig.LoadAPIServer(true); err != nil {
log.Warnf("could not load LAPI, skipping CAPI check")
skipLAPI = true
infos[SUPPORT_CAPI_STATUS_PATH] = []byte(err.Error())
}
if err := csConfig.LoadCrowdsec(); err != nil {
if err = csConfig.LoadCrowdsec(); err != nil {
log.Warnf("could not load agent config, skipping crowdsec config check")
skipAgent = true
}
hub, err := require.Hub(csConfig, nil)
hub, err := require.Hub(csConfig, nil, nil)
if err != nil {
log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
skipHub = true
infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())
infos[SUPPORT_SCENARIOS_PATH] = []byte(err.Error())
infos[SUPPORT_POSTOVERFLOWS_PATH] = []byte(err.Error())
infos[SUPPORT_CONTEXTS_PATH] = []byte(err.Error())
infos[SUPPORT_COLLECTIONS_PATH] = []byte(err.Error())
}
@ -344,6 +385,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS)
infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS)
infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
infos[SUPPORT_CONTEXTS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS)
}
@ -381,7 +423,6 @@ cscli support dump -f /tmp/crowdsec-support.zip
}
if !skipAgent {
acquis := collectAcquisitionConfig()
for filename, content := range acquis {
@ -407,7 +448,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
log.Fatalf("could not finalize zip file: %s", err)
}
err = os.WriteFile(outFile, w.Bytes(), 0600)
err = os.WriteFile(outFile, w.Bytes(), 0o600)
if err != nil {
log.Fatalf("could not write zip file to %s: %s", outFile, err)
}
@ -415,8 +456,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
log.Infof("Written zip file to %s", outFile)
},
}
cmdDump.Flags().StringVarP(&outFile, "outFile", "f", "", "File to dump the information to")
cmdSupport.AddCommand(cmdDump)
return cmdSupport
cmd.Flags().StringVarP(&outFile, "outFile", "f", "", "File to dump the information to")
return cmd
}

16
cmd/crowdsec-cli/utils.go
View file

@ -8,7 +8,6 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
@ -26,6 +25,7 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
return fmt.Errorf("%s isn't a valid range", *ipRange)
}
}
if *ip != "" {
ipRepr := net.ParseIP(*ip)
if ipRepr == nil {
@ -33,7 +33,7 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
}
}
//avoid confusion on scope (ip vs Ip and range vs Range)
// avoid confusion on scope (ip vs Ip and range vs Range)
switch strings.ToLower(*scope) {
case "ip":
*scope = types.Ip
@ -44,18 +44,8 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
case "as":
*scope = types.AS
}
return nil
}
func getDBClient() (*database.Client, error) {
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
return nil, err
}
ret, err := database.NewClient(csConfig.DbConfig)
if err != nil {
return nil, err
}
return ret, nil
return nil
}
func removeFromSlice(val string, slice []string) []string {

13
cmd/crowdsec-cli/utils_table.go
View file

@ -25,6 +25,19 @@ func listHubItemTable(out io.Writer, title string, items []*cwhub.Item) {
t.Render()
}
func appsecMetricsTable(out io.Writer, itemName string, metrics map[string]int) {
t := newTable(out)
t.SetHeaders("Inband Hits", "Outband Hits")
t.AddRow(
strconv.Itoa(metrics["inband_hits"]),
strconv.Itoa(metrics["outband_hits"]),
)
renderTableTitle(out, fmt.Sprintf("\n - (AppSec Rule) %s:", itemName))
t.Render()
}
func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int) {
if metrics["instantiation"] == 0 {
return

27
cmd/crowdsec-cli/version.go Normal file
View file

@ -0,0 +1,27 @@
package main
import (
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
)
type cliVersion struct{}
func NewCLIVersion() *cliVersion {
return &cliVersion{}
}
func (cli cliVersion) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "version",
Short: "Display version",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
Run: func(_ *cobra.Command, _ []string) {
cwversion.Show()
},
}
return cmd
}

4
cmd/crowdsec/api.go
View file

@ -56,7 +56,8 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) {
return apiServer, nil
}
func serveAPIServer(apiServer *apiserver.APIServer, apiReady chan bool) {
func serveAPIServer(apiServer *apiserver.APIServer) {
apiReady := make(chan bool, 1)
apiTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/serveAPIServer")
go func() {
@ -80,6 +81,7 @@ func serveAPIServer(apiServer *apiserver.APIServer, apiReady chan bool) {
}
return nil
})
<-apiReady
}
func hasPlugins(profiles []*csconfig.ProfileCfg) bool {

65
cmd/crowdsec/crowdsec.go
View file

@ -1,6 +1,7 @@
package main
import (
"context"
"fmt"
"os"
"path/filepath"
@ -13,6 +14,8 @@ import (
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/pkg/acquisition"
"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
"github.com/crowdsecurity/crowdsec/pkg/appsec"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
@ -23,6 +26,10 @@ import (
func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) {
var err error
if err = alertcontext.LoadConsoleContext(cConfig, hub); err != nil {
return nil, fmt.Errorf("while loading context: %w", err)
}
// Start loading configs
csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
@ -33,9 +40,14 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, er
return nil, fmt.Errorf("while loading scenarios: %w", err)
}
if err := appsec.LoadAppsecRules(hub); err != nil {
return nil, fmt.Errorf("while loading appsec rules: %w", err)
}
if err := LoadAcquisition(cConfig); err != nil {
return nil, fmt.Errorf("while loading acquisition config: %w", err)
}
return csParsers, nil
}
@ -45,63 +57,86 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.H
//start go-routines for parsing, buckets pour and outputs.
parserWg := &sync.WaitGroup{}
parsersTomb.Go(func() error {
parserWg.Add(1)
for i := 0; i < cConfig.Crowdsec.ParserRoutinesCount; i++ {
parsersTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/runParse")
if err := runParse(inputLineChan, inputEventChan, *parsers.Ctx, parsers.Nodes); err != nil { //this error will never happen as parser.Parse is not able to return errors
log.Fatalf("starting parse error : %s", err)
return err
}
return nil
})
}
parserWg.Done()
return nil
})
parserWg.Wait()
bucketWg := &sync.WaitGroup{}
bucketsTomb.Go(func() error {
bucketWg.Add(1)
/*restore previous state as well if present*/
if cConfig.Crowdsec.BucketStateFile != "" {
log.Warningf("Restoring buckets state from %s", cConfig.Crowdsec.BucketStateFile)
if err := leaky.LoadBucketsState(cConfig.Crowdsec.BucketStateFile, buckets, holders); err != nil {
return fmt.Errorf("unable to restore buckets : %s", err)
return fmt.Errorf("unable to restore buckets: %w", err)
}
}
for i := 0; i < cConfig.Crowdsec.BucketsRoutinesCount; i++ {
bucketsTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/runPour")
if err := runPour(inputEventChan, holders, buckets, cConfig); err != nil {
log.Fatalf("starting pour error : %s", err)
return err
}
return nil
})
}
bucketWg.Done()
return nil
})
bucketWg.Wait()
apiClient, err := AuthenticatedLAPIClient(*cConfig.API.Client.Credentials, hub)
if err != nil {
return err
}
log.Debugf("Starting HeartBeat service")
apiClient.HeartBeat.StartHeartBeat(context.Background(), &outputsTomb)
outputWg := &sync.WaitGroup{}
outputsTomb.Go(func() error {
outputWg.Add(1)
for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ {
outputsTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/runOutput")
if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials, hub); err != nil {
if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, apiClient); err != nil {
log.Fatalf("starting outputs error : %s", err)
return err
}
return nil
})
}
outputWg.Done()
return nil
})
outputWg.Wait()
@ -111,16 +146,16 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.H
if cConfig.Prometheus.Level == "aggregated" {
aggregated = true
}
if err := acquisition.GetMetrics(dataSources, aggregated); err != nil {
return fmt.Errorf("while fetching prometheus metrics for datasources: %w", err)
}
}
log.Info("Starting processing data")
if err := acquisition.StartAcquisition(dataSources, inputLineChan, &acquisTomb); err != nil {
log.Fatalf("starting acquisition error : %s", err)
return err
return fmt.Errorf("starting acquisition error: %w", err)
}
return nil
@ -129,11 +164,13 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.H
func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub.Hub, agentReady chan bool) {
crowdsecTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/serveCrowdsec")
go func() {
defer trace.CatchPanic("crowdsec/runCrowdsec")
// this logs every time, even at config reload
log.Debugf("running agent after %s ms", time.Since(crowdsecT0))
agentReady <- true
if err := runCrowdsec(cConfig, parsers, hub); err != nil {
log.Fatalf("unable to start crowdsec routines: %s", err)
}
@ -145,16 +182,20 @@ func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub
*/
waitOnTomb()
log.Debugf("Shutting down crowdsec routines")
if err := ShutdownCrowdsecRoutines(); err != nil {
log.Fatalf("unable to shutdown crowdsec routines: %s", err)
}
log.Debugf("everything is dead, return crowdsecTomb")
if dumpStates {
dumpParserState()
dumpOverflowState()
dumpBucketsPour()
os.Exit(0)
}
return nil
})
}
@ -164,55 +205,65 @@ func dumpBucketsPour() {
if err != nil {
log.Fatalf("open: %s", err)
}
out, err := yaml.Marshal(leaky.BucketPourCache)
if err != nil {
log.Fatalf("marshal: %s", err)
}
b, err := fd.Write(out)
if err != nil {
log.Fatalf("write: %s", err)
}
log.Tracef("wrote %d bytes", b)
if err := fd.Close(); err != nil {
log.Fatalf(" close: %s", err)
}
}
func dumpParserState() {
fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "parser-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
if err != nil {
log.Fatalf("open: %s", err)
}
out, err := yaml.Marshal(parser.StageParseCache)
if err != nil {
log.Fatalf("marshal: %s", err)
}
b, err := fd.Write(out)
if err != nil {
log.Fatalf("write: %s", err)
}
log.Tracef("wrote %d bytes", b)
if err := fd.Close(); err != nil {
log.Fatalf(" close: %s", err)
}
}
func dumpOverflowState() {
fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "bucket-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
if err != nil {
log.Fatalf("open: %s", err)
}
out, err := yaml.Marshal(bucketOverflows)
if err != nil {
log.Fatalf("marshal: %s", err)
}
b, err := fd.Write(out)
if err != nil {
log.Fatalf("write: %s", err)
}
log.Tracef("wrote %d bytes", b)
if err := fd.Close(); err != nil {
log.Fatalf(" close: %s", err)
}

43
cmd/crowdsec/hook.go Normal file
View file

@ -0,0 +1,43 @@
package main
import (
"io"
"os"
log "github.com/sirupsen/logrus"
)
type ConditionalHook struct {
Writer io.Writer
LogLevels []log.Level
Enabled bool
}
func (hook *ConditionalHook) Fire(entry *log.Entry) error {
if hook.Enabled {
line, err := entry.String()
if err != nil {
return err
}
_, err = hook.Writer.Write([]byte(line))
return err
}
return nil
}
func (hook *ConditionalHook) Levels() []log.Level {
return hook.LogLevels
}
// The primal logging hook is set up before parsing config.yaml.
// Once config.yaml is parsed, the primal hook is disabled if the
// configured logger is writing to stderr. Otherwise it's used to
// report fatal errors and panics to stderr in addition to the log file.
var primalHook = &ConditionalHook{
Writer: os.Stderr,
LogLevels: []log.Level{log.FatalLevel, log.PanicLevel},
Enabled: true,
}

92
cmd/crowdsec/lapiclient.go Normal file
View file

@ -0,0 +1,92 @@
package main
import (
"context"
"fmt"
"net/url"
"time"
"github.com/go-openapi/strfmt"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/models"
)
func AuthenticatedLAPIClient(credentials csconfig.ApiCredentialsCfg, hub *cwhub.Hub) (*apiclient.ApiClient, error) {
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return nil, fmt.Errorf("loading list of installed hub scenarios: %w", err)
}
appsecRules, err := hub.GetInstalledItemNames(cwhub.APPSEC_RULES)
if err != nil {
return nil, fmt.Errorf("loading list of installed hub appsec rules: %w", err)
}
installedScenariosAndAppsecRules := make([]string, 0, len(scenarios)+len(appsecRules))
installedScenariosAndAppsecRules = append(installedScenariosAndAppsecRules, scenarios...)
installedScenariosAndAppsecRules = append(installedScenariosAndAppsecRules, appsecRules...)
apiURL, err := url.Parse(credentials.URL)
if err != nil {
return nil, fmt.Errorf("parsing api url ('%s'): %w", credentials.URL, err)
}
papiURL, err := url.Parse(credentials.PapiURL)
if err != nil {
return nil, fmt.Errorf("parsing polling api url ('%s'): %w", credentials.PapiURL, err)
}
password := strfmt.Password(credentials.Password)
client, err := apiclient.NewClient(&apiclient.Config{
MachineID: credentials.Login,
Password: password,
Scenarios: installedScenariosAndAppsecRules,
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
PapiURL: papiURL,
VersionPrefix: "v1",
UpdateScenario: func() ([]string, error) {
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return nil, err
}
appsecRules, err := hub.GetInstalledItemNames(cwhub.APPSEC_RULES)
if err != nil {
return nil, err
}
ret := make([]string, 0, len(scenarios)+len(appsecRules))
ret = append(ret, scenarios...)
ret = append(ret, appsecRules...)
return ret, nil
},
})
if err != nil {
return nil, fmt.Errorf("new client api: %w", err)
}
authResp, _, err := client.Auth.AuthenticateWatcher(context.Background(), models.WatcherAuthRequest{
MachineID: &credentials.Login,
Password: &password,
Scenarios: installedScenariosAndAppsecRules,
})
if err != nil {
return nil, fmt.Errorf("authenticate watcher (%s): %w", credentials.Login, err)
}
var expiration time.Time
if err := expiration.UnmarshalText([]byte(authResp.Expire)); err != nil {
return nil, fmt.Errorf("unable to parse jwt expiration: %w", err)
}
client.GetClient().Transport.(*apiclient.JWTTransport).Token = authResp.Token
client.GetClient().Transport.(*apiclient.JWTTransport).Expiration = expiration
return client, nil
}

37
cmd/crowdsec/main.go
View file

@ -6,6 +6,7 @@ import (
_ "net/http/pprof"
"os"
"runtime"
"runtime/pprof"
"strings"
"time"
@ -71,6 +72,7 @@ type Flags struct {
DisableCAPI bool
Transform string
OrderEvent bool
CpuProfile string
}
type labelsMap map[string]string
@ -80,11 +82,13 @@ func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error {
err error
files []string
)
for _, hubScenarioItem := range hub.GetItemMap(cwhub.SCENARIOS) {
if hubScenarioItem.State.Installed {
files = append(files, hubScenarioItem.State.LocalPath)
}
}
buckets = leakybucket.NewBuckets()
log.Infof("Loading %d scenario files", len(files))
@ -99,6 +103,7 @@ func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error {
holders[holderIndex].Profiling = true
}
}
return nil
}
@ -143,8 +148,10 @@ func (l labelsMap) Set(label string) error {
if len(split) != 2 {
return fmt.Errorf("invalid format for label '%s', must be key:value", pair)
}
l[split[0]] = split[1]
}
return nil
}
@ -168,10 +175,13 @@ func (f *Flags) Parse() {
flag.BoolVar(&f.DisableAPI, "no-api", false, "disable local API")
flag.BoolVar(&f.DisableCAPI, "no-capi", false, "disable communication with Central API")
flag.BoolVar(&f.OrderEvent, "order-event", false, "enforce event ordering with significant performance cost")
if runtime.GOOS == "windows" {
flag.StringVar(&f.WinSvc, "winsvc", "", "Windows service Action: Install, Remove etc..")
}
flag.StringVar(&dumpFolder, "dump-data", "", "dump parsers/buckets raw outputs")
flag.StringVar(&f.CpuProfile, "cpu-profile", "", "write cpu profile to file")
flag.Parse()
}
@ -205,6 +215,7 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
// avoid returning a new ptr to the same value
return curLevelPtr
}
return &ret
}
@ -238,6 +249,8 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
return nil, err
}
primalHook.Enabled = (cConfig.Common.LogMedia != "stdout")
if err := csconfig.LoadFeatureFlagsFile(configFile, log.StandardLogger()); err != nil {
return nil, err
}
@ -249,7 +262,7 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
}
if !cConfig.DisableAPI {
if err := cConfig.LoadAPIServer(); err != nil {
if err := cConfig.LoadAPIServer(false); err != nil {
return nil, err
}
}
@ -282,6 +295,7 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
if cConfig.DisableAPI {
cConfig.Common.Daemonize = false
}
log.Infof("single file mode : log_media=%s daemonize=%t", cConfig.Common.LogMedia, cConfig.Common.Daemonize)
}
@ -291,6 +305,7 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
if cConfig.Common.Daemonize && runtime.GOOS == "windows" {
log.Debug("Daemonization is not supported on Windows, disabling")
cConfig.Common.Daemonize = false
}
@ -308,6 +323,8 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
var crowdsecT0 time.Time
func main() {
log.AddHook(primalHook)
if err := fflag.RegisterAllFeatures(); err != nil {
log.Fatalf("failed to register features: %s", err)
}
@ -338,9 +355,25 @@ func main() {
os.Exit(0)
}
if flags.CpuProfile != "" {
f, err := os.Create(flags.CpuProfile)
if err != nil {
log.Fatalf("could not create CPU profile: %s", err)
}
log.Infof("CPU profile will be written to %s", flags.CpuProfile)
if err := pprof.StartCPUProfile(f); err != nil {
f.Close()
log.Fatalf("could not start CPU profile: %s", err)
}
defer f.Close()
defer pprof.StopCPUProfile()
}
err := StartRunSvc()
if err != nil {
log.Fatal(err)
pprof.StopCPUProfile()
log.Fatal(err) //nolint:gocritic // Disable warning for the defer pprof.StopCPUProfile() call
}
os.Exit(0)
}

21
cmd/crowdsec/metrics.go
View file

@ -114,13 +114,17 @@ func computeDynamicMetrics(next http.Handler, dbClient *database.Client) http.Ha
}
decisionsFilters := make(map[string][]string, 0)
decisions, err := dbClient.QueryDecisionCountByScenario(decisionsFilters)
if err != nil {
log.Errorf("Error querying decisions for metrics: %v", err)
next.ServeHTTP(w, r)
return
}
globalActiveDecisions.Reset()
for _, d := range decisions {
globalActiveDecisions.With(prometheus.Labels{"reason": d.Scenario, "origin": d.Origin, "action": d.Type}).Set(float64(d.Count))
}
@ -136,6 +140,7 @@ func computeDynamicMetrics(next http.Handler, dbClient *database.Client) http.Ha
if err != nil {
log.Errorf("Error querying alerts for metrics: %v", err)
next.ServeHTTP(w, r)
return
}
@ -161,7 +166,8 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow,
v1.LapiRouteHits,
leaky.BucketsCurrentCount,
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics)
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, parser.NodesWlHitsOk, parser.NodesWlHits,
)
} else {
log.Infof("Loading prometheus collectors")
prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo,
@ -169,13 +175,15 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
globalCsInfo, globalParsingHistogram, globalPourHistogram,
v1.LapiRouteHits, v1.LapiMachineHits, v1.LapiBouncerHits, v1.LapiNilDecisions, v1.LapiNonNilDecisions, v1.LapiResponseTime,
leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, leaky.BucketsCurrentCount,
globalActiveDecisions, globalAlerts,
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics)
globalActiveDecisions, globalAlerts, parser.NodesWlHitsOk, parser.NodesWlHits,
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics,
)
}
}
func servePrometheus(config *csconfig.PrometheusCfg, dbClient *database.Client, apiReady chan bool, agentReady chan bool) {
func servePrometheus(config *csconfig.PrometheusCfg, dbClient *database.Client, agentReady chan bool) {
<-agentReady
if !config.Enabled {
return
}
@ -183,9 +191,8 @@ func servePrometheus(config *csconfig.PrometheusCfg, dbClient *database.Client,
defer trace.CatchPanic("crowdsec/servePrometheus")
http.Handle("/metrics", computeDynamicMetrics(promhttp.Handler(), dbClient))
<-apiReady
<-agentReady
log.Debugf("serving metrics after %s ms", time.Since(crowdsecT0))
if err := http.ListenAndServe(fmt.Sprintf("%s:%d", config.ListenAddr, config.ListenPort), nil); err != nil {
log.Warningf("prometheus: %s", err)
}

83
cmd/crowdsec/output.go
View file

@ -3,18 +3,12 @@ package main
import (
"context"
"fmt"
"net/url"
"sync"
"time"
"github.com/go-openapi/strfmt"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/parser"
@ -22,7 +16,6 @@ import (
)
func dedupAlerts(alerts []types.RuntimeAlert) ([]*models.Alert, error) {
var dedupCache []*models.Alert
for idx, alert := range alerts {
@ -32,16 +25,21 @@ func dedupAlerts(alerts []types.RuntimeAlert) ([]*models.Alert, error) {
dedupCache = append(dedupCache, alert.Alert)
continue
}
for k, src := range alert.Sources {
refsrc := *alert.Alert //copy
log.Tracef("source[%s]", k)
refsrc.Source = &src
dedupCache = append(dedupCache, &refsrc)
}
}
if len(dedupCache) != len(alerts) {
log.Tracef("went from %d to %d alerts", len(alerts), len(dedupCache))
}
return dedupCache, nil
}
@ -52,71 +50,25 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error
if err != nil {
return fmt.Errorf("failed to transform alerts for api: %w", err)
}
_, _, err = client.Alerts.Add(ctx, alertsToPush)
if err != nil {
return fmt.Errorf("failed sending alert to LAPI: %w", err)
}
return nil
}
var bucketOverflows []types.Event
func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets,
postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node,
apiConfig csconfig.ApiCredentialsCfg, hub *cwhub.Hub) error {
func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets, postOverflowCTX parser.UnixParserCtx,
postOverflowNodes []parser.Node, client *apiclient.ApiClient) error {
var (
cache []types.RuntimeAlert
cacheMutex sync.Mutex
)
var err error
ticker := time.NewTicker(1 * time.Second)
var cache []types.RuntimeAlert
var cacheMutex sync.Mutex
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("loading list of installed hub scenarios: %w", err)
}
apiURL, err := url.Parse(apiConfig.URL)
if err != nil {
return fmt.Errorf("parsing api url ('%s'): %w", apiConfig.URL, err)
}
papiURL, err := url.Parse(apiConfig.PapiURL)
if err != nil {
return fmt.Errorf("parsing polling api url ('%s'): %w", apiConfig.PapiURL, err)
}
password := strfmt.Password(apiConfig.Password)
Client, err := apiclient.NewClient(&apiclient.Config{
MachineID: apiConfig.Login,
Password: password,
Scenarios: scenarios,
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
PapiURL: papiURL,
VersionPrefix: "v1",
UpdateScenario: func() ([]string, error) {return hub.GetInstalledItemNames(cwhub.SCENARIOS)},
})
if err != nil {
return fmt.Errorf("new client api: %w", err)
}
authResp, _, err := Client.Auth.AuthenticateWatcher(context.Background(), models.WatcherAuthRequest{
MachineID: &apiConfig.Login,
Password: &password,
Scenarios: scenarios,
})
if err != nil {
return fmt.Errorf("authenticate watcher (%s): %w", apiConfig.Login, err)
}
if err := Client.GetClient().Transport.(*apiclient.JWTTransport).Expiration.UnmarshalText([]byte(authResp.Expire)); err != nil {
return fmt.Errorf("unable to parse jwt expiration: %w", err)
}
Client.GetClient().Transport.(*apiclient.JWTTransport).Token = authResp.Token
//start the heartbeat service
log.Debugf("Starting HeartBeat service")
Client.HeartBeat.StartHeartBeat(context.Background(), &outputsTomb)
LOOP:
for {
select {
@ -127,7 +79,7 @@ LOOP:
newcache := make([]types.RuntimeAlert, 0)
cache = newcache
cacheMutex.Unlock()
if err := PushAlerts(cachecopy, Client); err != nil {
if err := PushAlerts(cachecopy, client); err != nil {
log.Errorf("while pushing to api : %s", err)
//just push back the events to the queue
cacheMutex.Lock()
@ -140,10 +92,11 @@ LOOP:
cacheMutex.Lock()
cachecopy := cache
cacheMutex.Unlock()
if err := PushAlerts(cachecopy, Client); err != nil {
if err := PushAlerts(cachecopy, client); err != nil {
log.Errorf("while pushing leftovers to api : %s", err)
}
}
break LOOP
case event := <-overflow:
/*if alert is empty and mapKey is present, the overflow is just to cleanup bucket*/
@ -154,7 +107,7 @@ LOOP:
/* process post overflow parser nodes */
event, err := parser.Parse(postOverflowCTX, event, postOverflowNodes)
if err != nil {
return fmt.Errorf("postoverflow failed : %s", err)
return fmt.Errorf("postoverflow failed: %w", err)
}
log.Printf("%s", *event.Overflow.Alert.Message)
//if the Alert is nil, it's to signal bucket is ready for GC, don't track this
@ -196,6 +149,6 @@ LOOP:
}
ticker.Stop()
return nil
return nil
}

7
cmd/crowdsec/parse.go
View file

@ -22,6 +22,13 @@ LOOP:
if !event.Process {
continue
}
/*Application security engine is going to generate 2 events:
- one that is treated as a log and can go to scenarios
- another one that will go directly to LAPI*/
if event.Type == types.APPSEC {
outputEventChan <- event
continue
}
if event.Line.Module == "" {
log.Errorf("empty event.Line.Module field, the acquisition module must set it ! : %+v", event.Line)
continue

34
cmd/crowdsec/run_in_svc.go
View file

@ -1,14 +1,12 @@
//go:build linux || freebsd || netbsd || openbsd || solaris || !windows
// +build linux freebsd netbsd openbsd solaris !windows
//go:build !windows
package main
import (
"fmt"
"os"
"runtime/pprof"
log "github.com/sirupsen/logrus"
"github.com/sirupsen/logrus/hooks/writer"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/go-cs-lib/version"
@ -25,15 +23,9 @@ func StartRunSvc() error {
defer trace.CatchPanic("crowdsec/StartRunSvc")
// Set a default logger with level=fatal on stderr,
// in addition to the one we configure afterwards
log.AddHook(&writer.Hook{
Writer: os.Stderr,
LogLevels: []log.Level{
log.PanicLevel,
log.FatalLevel,
},
})
//Always try to stop CPU profiling to avoid passing flags around
//It's a noop if profiling is not enabled
defer pprof.StopCPUProfile()
if cConfig, err = LoadConfig(flags.ConfigFile, flags.DisableAgent, flags.DisableAPI, false); err != nil {
return err
@ -41,23 +33,31 @@ func StartRunSvc() error {
log.Infof("Crowdsec %s", version.String())
apiReady := make(chan bool, 1)
agentReady := make(chan bool, 1)
// Enable profiling early
if cConfig.Prometheus != nil {
var dbClient *database.Client
var err error
if cConfig.DbConfig != nil {
dbClient, err = database.NewClient(cConfig.DbConfig)
if err != nil {
return fmt.Errorf("unable to create database client: %s", err)
return fmt.Errorf("unable to create database client: %w", err)
}
}
registerPrometheus(cConfig.Prometheus)
go servePrometheus(cConfig.Prometheus, dbClient, apiReady, agentReady)
go servePrometheus(cConfig.Prometheus, dbClient, agentReady)
} else {
// avoid leaking the channel
go func() {
<-agentReady
}()
}
return Serve(cConfig, apiReady, agentReady)
return Serve(cConfig, agentReady)
}

12
cmd/crowdsec/run_in_svc_windows.go
View file

@ -2,6 +2,7 @@ package main
import (
"fmt"
"runtime/pprof"
log "github.com/sirupsen/logrus"
"golang.org/x/sys/windows/svc"
@ -19,6 +20,10 @@ func StartRunSvc() error {
defer trace.CatchPanic("crowdsec/StartRunSvc")
//Always try to stop CPU profiling to avoid passing flags around
//It's a noop if profiling is not enabled
defer pprof.StopCPUProfile()
isRunninginService, err := svc.IsWindowsService()
if err != nil {
return fmt.Errorf("failed to determine if we are running in windows service mode: %w", err)
@ -68,7 +73,6 @@ func WindowsRun() error {
log.Infof("Crowdsec %s", version.String())
apiReady := make(chan bool, 1)
agentReady := make(chan bool, 1)
// Enable profiling early
@ -80,11 +84,11 @@ func WindowsRun() error {
dbClient, err = database.NewClient(cConfig.DbConfig)
if err != nil {
return fmt.Errorf("unable to create database client: %s", err)
return fmt.Errorf("unable to create database client: %w", err)
}
}
registerPrometheus(cConfig.Prometheus)
go servePrometheus(cConfig.Prometheus, dbClient, apiReady, agentReady)
go servePrometheus(cConfig.Prometheus, dbClient, agentReady)
}
return Serve(cConfig, apiReady, agentReady)
return Serve(cConfig, agentReady)
}

36
cmd/crowdsec/serve.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"os"
"os/signal"
"runtime/pprof"
"syscall"
"time"
@ -41,7 +42,9 @@ func debugHandler(sig os.Signal, cConfig *csconfig.Config) error {
if err := leaky.ShutdownAllBuckets(buckets); err != nil {
log.Warningf("Failed to shut down routines : %s", err)
}
log.Printf("Shutdown is finished, buckets are in %s", tmpFile)
return nil
}
@ -65,19 +68,20 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
if !cConfig.DisableAPI {
if flags.DisableCAPI {
log.Warningf("Communication with CrowdSec Central API disabled from args")
cConfig.API.Server.OnlineClient = nil
}
apiServer, err := initAPIServer(cConfig)
if err != nil {
return nil, fmt.Errorf("unable to init api server: %w", err)
}
apiReady := make(chan bool, 1)
serveAPIServer(apiServer, apiReady)
serveAPIServer(apiServer)
}
if !cConfig.DisableAgent {
hub, err := cwhub.NewHub(cConfig.Hub, nil, false)
hub, err := cwhub.NewHub(cConfig.Hub, nil, false, log.StandardLogger())
if err != nil {
return nil, fmt.Errorf("while loading hub index: %w", err)
}
@ -109,6 +113,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
log.Warningf("Failed to delete temp file (%s) : %s", tmpFile, err)
}
}
return cConfig, nil
}
@ -116,10 +121,12 @@ func ShutdownCrowdsecRoutines() error {
var reterr error
log.Debugf("Shutting down crowdsec sub-routines")
if len(dataSources) > 0 {
acquisTomb.Kill(nil)
log.Debugf("waiting for acquisition to finish")
drainChan(inputLineChan)
if err := acquisTomb.Wait(); err != nil {
log.Warningf("Acquisition returned error : %s", err)
reterr = err
@ -129,6 +136,7 @@ func ShutdownCrowdsecRoutines() error {
log.Debugf("acquisition is finished, wait for parser/bucket/ouputs.")
parsersTomb.Kill(nil)
drainChan(inputEventChan)
if err := parsersTomb.Wait(); err != nil {
log.Warningf("Parsers returned error : %s", err)
reterr = err
@ -159,6 +167,7 @@ func ShutdownCrowdsecRoutines() error {
log.Warningf("Outputs returned error : %s", err)
reterr = err
}
log.Debugf("outputs are done")
case <-time.After(3 * time.Second):
// this can happen if outputs are stuck in a http retry loop
@ -180,6 +189,7 @@ func shutdownAPI() error {
}
log.Debugf("done")
return nil
}
@ -192,6 +202,7 @@ func shutdownCrowdsec() error {
}
log.Debugf("done")
return nil
}
@ -245,6 +256,10 @@ func HandleSignals(cConfig *csconfig.Config) error {
exitChan := make(chan error)
//Always try to stop CPU profiling to avoid passing flags around
//It's a noop if profiling is not enabled
defer pprof.StopCPUProfile()
go func() {
defer trace.CatchPanic("crowdsec/HandleSignals")
Loop:
@ -287,10 +302,11 @@ func HandleSignals(cConfig *csconfig.Config) error {
if err == nil {
log.Warning("Crowdsec service shutting down")
}
return err
}
func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) error {
func Serve(cConfig *csconfig.Config, agentReady chan bool) error {
acquisTomb = tomb.Tomb{}
parsersTomb = tomb.Tomb{}
bucketsTomb = tomb.Tomb{}
@ -320,6 +336,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
if cConfig.API.CTI != nil && *cConfig.API.CTI.Enabled {
log.Infof("Crowdsec CTI helper enabled")
if err := exprhelpers.InitCrowdsecCTI(cConfig.API.CTI.Key, cConfig.API.CTI.CacheTimeout, cConfig.API.CTI.CacheSize, cConfig.API.CTI.LogLevel); err != nil {
return fmt.Errorf("failed to init crowdsec cti: %w", err)
}
@ -332,6 +349,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
if flags.DisableCAPI {
log.Warningf("Communication with CrowdSec Central API disabled from args")
cConfig.API.Server.OnlineClient = nil
}
@ -341,14 +359,12 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
}
if !flags.TestMode {
serveAPIServer(apiServer, apiReady)
serveAPIServer(apiServer)
}
} else {
apiReady <- true
}
if !cConfig.DisableAgent {
hub, err := cwhub.NewHub(cConfig.Hub, nil, false)
hub, err := cwhub.NewHub(cConfig.Hub, nil, false, log.StandardLogger())
if err != nil {
return fmt.Errorf("while loading hub index: %w", err)
}
@ -361,6 +377,8 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
// if it's just linting, we're done
if !flags.TestMode {
serveCrowdsec(csParsers, cConfig, hub, agentReady)
} else {
agentReady <- true
}
} else {
agentReady <- true
@ -390,6 +408,7 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
for _, ch := range waitChans {
<-ch
switch ch {
case apiTomb.Dead():
log.Infof("api shutdown")
@ -397,5 +416,6 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
log.Infof("crowdsec shutdown")
}
}
return nil
}

6
cmd/crowdsec/win_service.go
View file

@ -3,7 +3,6 @@
// license that can be found in the LICENSE file.
//go:build windows
// +build windows
package main
@ -24,7 +23,7 @@ type crowdsec_winservice struct {
config *csconfig.Config
}
func (m *crowdsec_winservice) Execute(args []string, r <-chan svc.ChangeRequest, changes chan<- svc.Status) (ssec bool, errno uint32) {
func (m *crowdsec_winservice) Execute(args []string, r <-chan svc.ChangeRequest, changes chan<- svc.Status) (bool, uint32) {
const cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown
changes <- svc.Status{State: svc.StartPending}
tick := time.Tick(500 * time.Millisecond)
@ -60,7 +59,8 @@ func (m *crowdsec_winservice) Execute(args []string, r <-chan svc.ChangeRequest,
if err != nil {
log.Fatal(err)
}
return
return false, 0
}
func runService(name string) error {

1
cmd/crowdsec/win_service_install.go
View file

@ -3,7 +3,6 @@
// license that can be found in the LICENSE file.
//go:build windows
// +build windows
package main

2
cmd/crowdsec/win_service_manage.go
View file

@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build windows
//go:build windows
package main

79
cmd/notification-http/main.go
View file

@ -4,6 +4,7 @@ import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"io"
"net/http"
@ -22,6 +23,10 @@ type PluginConfig struct {
SkipTLSVerification bool `yaml:"skip_tls_verification"`
Method string `yaml:"method"`
LogLevel *string `yaml:"log_level"`
Client *http.Client `yaml:"-"`
CertPath string `yaml:"cert_path"`
KeyPath string `yaml:"key_path"`
CAPath string `yaml:"ca_cert_path"`
}
type HTTPPlugin struct {
@ -35,6 +40,64 @@ var logger hclog.Logger = hclog.New(&hclog.LoggerOptions{
JSONFormat: true,
})
func getCertPool(caPath string) (*x509.CertPool, error) {
cp, err := x509.SystemCertPool()
if err != nil {
return nil, fmt.Errorf("unable to load system CA certificates: %w", err)
}
if cp == nil {
cp = x509.NewCertPool()
}
if caPath == "" {
return cp, nil
}
logger.Info(fmt.Sprintf("Using CA cert '%s'", caPath))
caCert, err := os.ReadFile(caPath)
if err != nil {
return nil, fmt.Errorf("unable to load CA certificate '%s': %w", caPath, err)
}
cp.AppendCertsFromPEM(caCert)
return cp, nil
}
func getTLSClient(tlsVerify bool, caPath, certPath, keyPath string) (*http.Client, error) {
var client *http.Client
caCertPool, err := getCertPool(caPath)
if err != nil {
return nil, err
}
tlsConfig := &tls.Config{
RootCAs: caCertPool,
InsecureSkipVerify: tlsVerify,
}
if certPath != "" && keyPath != "" {
logger.Info(fmt.Sprintf("Using client certificate '%s' and key '%s'", certPath, keyPath))
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
if err != nil {
return nil, fmt.Errorf("unable to load client certificate '%s' and key '%s': %w", certPath, keyPath, err)
}
tlsConfig.Certificates = []tls.Certificate{cert}
}
client = &http.Client{
Transport: &http.Transport{
TLSClientConfig: tlsConfig,
},
}
return client, err
}
func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notification) (*protobufs.Empty, error) {
if _, ok := s.PluginConfigByName[notification.Name]; !ok {
return nil, fmt.Errorf("invalid plugin config name %s", notification.Name)
@ -46,13 +109,6 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
}
logger.Info(fmt.Sprintf("received signal for %s config", notification.Name))
client := http.Client{}
if cfg.SkipTLSVerification {
client.Transport = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
}
request, err := http.NewRequest(cfg.Method, cfg.URL, bytes.NewReader([]byte(notification.Text)))
if err != nil {
@ -63,7 +119,7 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
request.Header.Add(headerName, headerValue)
}
logger.Debug(fmt.Sprintf("making HTTP %s call to %s with body %s", cfg.Method, cfg.URL, notification.Text))
resp, err := client.Do(request.WithContext(ctx))
resp, err := cfg.Client.Do(request.WithContext(ctx))
if err != nil {
logger.Error(fmt.Sprintf("Failed to make HTTP request : %s", err))
return nil, err
@ -88,6 +144,13 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
func (s *HTTPPlugin) Configure(ctx context.Context, config *protobufs.Config) (*protobufs.Empty, error) {
d := PluginConfig{}
err := yaml.Unmarshal(config.Config, &d)
if err != nil {
return nil, err
}
d.Client, err = getTLSClient(d.SkipTLSVerification, d.CAPath, d.CertPath, d.KeyPath)
if err != nil {
return nil, err
}
s.PluginConfigByName[d.Name] = d
logger.Debug(fmt.Sprintf("HTTP plugin '%s' use URL '%s'", d.Name, d.URL))
return &protobufs.Empty{}, err

4
config/acquis_win.yaml
View file

@ -10,7 +10,7 @@ labels:
---
##Firewall
filenames:
- C:\Windows\System32\LogFiles\Firewall\pfirewall.log
- C:\Windows\System32\LogFiles\Firewall\*.log
labels:
type: windows-firewall
---
@ -28,4 +28,4 @@ use_time_machine: true
filenames:
- C:\inetpub\logs\LogFiles\*\*.log
labels:
type: iis
type: iis

1
config/dev.yaml
View file

@ -33,6 +33,7 @@ api:
client:
credentials_path: ./config/local_api_credentials.yaml
server:
console_path: ./config/console.yaml
#insecure_skip_verify: true
listen_uri: 127.0.0.1:8081
profiles_path: ./config/profiles.yaml

15
debian/postinst vendored
View file

@ -58,10 +58,10 @@ if [ "$1" = configure ]; then
db_get crowdsec/capi
CAPI=$RET
cscli machines add -a
[ -s /etc/crowdsec/local_api_credentials.yaml ] || cscli machines add -a --force --error
if [ "$CAPI" = true ]; then
cscli capi register
cscli capi register --error
fi
else
@ -91,7 +91,7 @@ if [ "$1" = configure ]; then
systemctl --quiet is-enabled crowdsec || systemctl unmask crowdsec && systemctl enable crowdsec
API=$(cscli config show --key "Config.API.Server")
if [ "$API" = "<nil>" ] ; then
if [ "$API" = "nil" ] ; then
LAPI=false
else
PORT=$(cscli config show --key "Config.API.Server.ListenURI"|cut -d ":" -f2)
@ -102,6 +102,13 @@ if [ "$1" = configure ]; then
echo "Not attempting to start crowdsec, port ${PORT} is already used or lapi was disabled"
echo "This port is configured through /etc/crowdsec/config.yaml and /etc/crowdsec/local_api_credentials.yaml"
fi
echo "Get started with CrowdSec:"
echo " * Detailed guides are available in our documentation: https://docs.crowdsec.net"
echo " * Configuration items created by the community can be found at the Hub: https://hub.crowdsec.net"
echo " * Gain insights into your use of CrowdSec with the help of the console https://app.crowdsec.net"
fi
echo "You can always run the configuration again interactively by using '/usr/share/crowdsec/wizard.sh -c"
echo "You can always run the configuration again interactively by using '/usr/share/crowdsec/wizard.sh -c'"

2
debian/preinst vendored
View file

@ -40,4 +40,4 @@ if [ "$1" = upgrade ]; then
fi
fi
echo "You can always run the configuration again interactively by using '/usr/share/crowdsec/wizard.sh -c"
echo "You can always run the configuration again interactively by using '/usr/share/crowdsec/wizard.sh -c'"

2
debian/prerm vendored
View file

@ -1,5 +1,5 @@
if [ "$1" = "remove" ]; then
cscli dashboard remove -f -y || true
cscli dashboard remove -f -y --error || echo "Ignore the above error if you never installed the local dashboard."
systemctl stop crowdsec
systemctl disable crowdsec
fi

7
docker/README.md
View file

@ -316,14 +316,21 @@ config.yaml) each time the container is run.
| `BOUNCERS_ALLOWED_OU` | bouncer-ou | OU values allowed for bouncers, separated by comma |
| | | |
| __Hub management__ | | |
| `NO_HUB_UPGRADE` | false | Skip hub update / upgrade when the container starts |
| `COLLECTIONS` | | Collections to install, separated by space: `-e COLLECTIONS="crowdsecurity/linux crowdsecurity/apache2"` |
| `PARSERS` | | Parsers to install, separated by space |
| `SCENARIOS` | | Scenarios to install, separated by space |
| `POSTOVERFLOWS` | | Postoverflows to install, separated by space |
| `CONTEXTS` | | Context files to install, separated by space |
| `APPSEC_CONFIGS` | | Appsec configs files to install, separated by space |
| `APPSEC_RULES` | | Appsec rules files to install, separated by space |
| `DISABLE_COLLECTIONS` | | Collections to remove, separated by space: `-e DISABLE_COLLECTIONS="crowdsecurity/linux crowdsecurity/nginx"` |
| `DISABLE_PARSERS` | | Parsers to remove, separated by space |
| `DISABLE_SCENARIOS` | | Scenarios to remove, separated by space |
| `DISABLE_POSTOVERFLOWS` | | Postoverflows to remove, separated by space |
| `DISABLE_CONTEXTS` | | Context files to remove, separated by space |
| `DISABLE_APPSEC_CONFIGS`| | Appsec configs files to remove, separated by space |
| `DISABLE_APPSEC_RULES` | | Appsec rules files to remove, separated by space |
| | | |
| __Log verbosity__ | | |
| `LEVEL_INFO` | false | Force INFO level for the container log |

48
docker/docker_start.sh
View file

@ -109,6 +109,8 @@ cscli_if_clean() {
for obj in $objs; do
if cscli "$itemtype" inspect "$obj" -o json | yq -e '.tainted // false' >/dev/null 2>&1; then
echo "Object $itemtype/$obj is tainted, skipping"
elif cscli "$itemtype" inspect "$obj" -o json | yq -e '.local // false' >/dev/null 2>&1; then
echo "Object $itemtype/$obj is local, skipping"
else
# # Too verbose? Only show errors if not in debug mode
# if [ "$DEBUG" != "true" ]; then
@ -300,11 +302,14 @@ fi
conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)'
## Install collections, parsers, scenarios & postoverflows
cscli hub update
## Install hub items
cscli hub update || true
if isfalse "$NO_HUB_UPGRADE"; then
cscli hub upgrade || true
fi
cscli_if_clean collections upgrade crowdsecurity/linux
cscli_if_clean parsers upgrade crowdsecurity/whitelists
cscli_if_clean parsers install crowdsecurity/docker-logs
cscli_if_clean parsers install crowdsecurity/cri-logs
@ -328,6 +333,21 @@ if [ "$POSTOVERFLOWS" != "" ]; then
cscli_if_clean postoverflows install "$(difference "$POSTOVERFLOWS" "$DISABLE_POSTOVERFLOWS")"
fi
if [ "$CONTEXTS" != "" ]; then
# shellcheck disable=SC2086
cscli_if_clean contexts install "$(difference "$CONTEXTS" "$DISABLE_CONTEXTS")"
fi
if [ "$APPSEC_CONFIGS" != "" ]; then
# shellcheck disable=SC2086
cscli_if_clean appsec-configs install "$(difference "$APPSEC_CONFIGS" "$DISABLE_APPSEC_CONFIGS")"
fi
if [ "$APPSEC_RULES" != "" ]; then
# shellcheck disable=SC2086
cscli_if_clean appsec-rules install "$(difference "$APPSEC_RULES" "$DISABLE_APPSEC_RULES")"
fi
## Remove collections, parsers, scenarios & postoverflows
if [ "$DISABLE_COLLECTIONS" != "" ]; then
# shellcheck disable=SC2086
@ -349,6 +369,21 @@ if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" --force
fi
if [ "$DISABLE_CONTEXTS" != "" ]; then
# shellcheck disable=SC2086
cscli_if_clean contexts remove "$DISABLE_CONTEXTS" --force
fi
if [ "$DISABLE_APPSEC_CONFIGS" != "" ]; then
# shellcheck disable=SC2086
cscli_if_clean appsec-configs remove "$DISABLE_APPSEC_CONFIGS" --force
fi
if [ "$DISABLE_APPSEC_RULES" != "" ]; then
# shellcheck disable=SC2086
cscli_if_clean appsec-rules remove "$DISABLE_APPSEC_RULES" --force
fi
## Register bouncers via env
for BOUNCER in $(compgen -A variable | grep -i BOUNCER_KEY); do
KEY=$(printf '%s' "${!BOUNCER}")
@ -358,6 +393,11 @@ for BOUNCER in $(compgen -A variable | grep -i BOUNCER_KEY); do
fi
done
if [ "$ENABLE_CONSOLE_MANAGEMENT" != "" ]; then
# shellcheck disable=SC2086
cscli console enable console_management
fi
## Register bouncers via secrets (Swarm only)
shopt -s nullglob extglob
for BOUNCER in /run/secrets/@(bouncer_key|BOUNCER_KEY)* ; do

4
docker/test/Pipfile
View file

@ -1,7 +1,7 @@
[packages]
pytest-dotenv = "0.5.2"
pytest-xdist = "3.3.1"
pytest-cs = {ref = "0.7.18", git = "https://github.com/crowdsecurity/pytest-cs.git"}
pytest-xdist = "3.5.0"
pytest-cs = {ref = "0.7.19", git = "https://github.com/crowdsecurity/pytest-cs.git"}
[dev-packages]
gnureadline = "8.1.2"

531
docker/test/Pipfile.lock generated
View file

@ -1,7 +1,7 @@
{
"_meta": {
"hash": {
"sha256": "64085783c9fec3a9eda976b7700b5bad7abd2b7a0f0670fa2209c52f3647be7f"
"sha256": "b5d25a7199d15a900b285be1af97cf7b7083c6637d631ad777b454471c8319fe"
},
"pipfile-spec": 6,
"requires": {
@ -18,198 +18,211 @@
"default": {
"certifi": {
"hashes": [
"sha256:539cc1d13202e33ca466e88b2807e29f4c13049d6d87031a3c110744495cb082",
"sha256:92d6037539857d8206b8f6ae472e8b77db8058fec5937a1ef3f54304089edbb9"
"sha256:9b469f3a900bf28dc19b8cfbf8019bf47f7fdd1a65a1d4ffb98fc14166beb4d1",
"sha256:e036ab49d5b79556f99cfc2d9320b34cfbe5be05c5871b51de9329f0603b0474"
],
"markers": "python_version >= '3.6'",
"version": "==2023.7.22"
"version": "==2023.11.17"
},
"cffi": {
"hashes": [
"sha256:00a9ed42e88df81ffae7a8ab6d9356b371399b91dbdf0c3cb1e84c03a13aceb5",
"sha256:03425bdae262c76aad70202debd780501fabeaca237cdfddc008987c0e0f59ef",
"sha256:04ed324bda3cda42b9b695d51bb7d54b680b9719cfab04227cdd1e04e5de3104",
"sha256:0e2642fe3142e4cc4af0799748233ad6da94c62a8bec3a6648bf8ee68b1c7426",
"sha256:173379135477dc8cac4bc58f45db08ab45d228b3363adb7af79436135d028405",
"sha256:198caafb44239b60e252492445da556afafc7d1e3ab7a1fb3f0584ef6d742375",
"sha256:1e74c6b51a9ed6589199c787bf5f9875612ca4a8a0785fb2d4a84429badaf22a",
"sha256:2012c72d854c2d03e45d06ae57f40d78e5770d252f195b93f581acf3ba44496e",
"sha256:21157295583fe8943475029ed5abdcf71eb3911894724e360acff1d61c1d54bc",
"sha256:2470043b93ff09bf8fb1d46d1cb756ce6132c54826661a32d4e4d132e1977adf",
"sha256:285d29981935eb726a4399badae8f0ffdff4f5050eaa6d0cfc3f64b857b77185",
"sha256:30d78fbc8ebf9c92c9b7823ee18eb92f2e6ef79b45ac84db507f52fbe3ec4497",
"sha256:320dab6e7cb2eacdf0e658569d2575c4dad258c0fcc794f46215e1e39f90f2c3",
"sha256:33ab79603146aace82c2427da5ca6e58f2b3f2fb5da893ceac0c42218a40be35",
"sha256:3548db281cd7d2561c9ad9984681c95f7b0e38881201e157833a2342c30d5e8c",
"sha256:3799aecf2e17cf585d977b780ce79ff0dc9b78d799fc694221ce814c2c19db83",
"sha256:39d39875251ca8f612b6f33e6b1195af86d1b3e60086068be9cc053aa4376e21",
"sha256:3b926aa83d1edb5aa5b427b4053dc420ec295a08e40911296b9eb1b6170f6cca",
"sha256:3bcde07039e586f91b45c88f8583ea7cf7a0770df3a1649627bf598332cb6984",
"sha256:3d08afd128ddaa624a48cf2b859afef385b720bb4b43df214f85616922e6a5ac",
"sha256:3eb6971dcff08619f8d91607cfc726518b6fa2a9eba42856be181c6d0d9515fd",
"sha256:40f4774f5a9d4f5e344f31a32b5096977b5d48560c5592e2f3d2c4374bd543ee",
"sha256:4289fc34b2f5316fbb762d75362931e351941fa95fa18789191b33fc4cf9504a",
"sha256:470c103ae716238bbe698d67ad020e1db9d9dba34fa5a899b5e21577e6d52ed2",
"sha256:4f2c9f67e9821cad2e5f480bc8d83b8742896f1242dba247911072d4fa94c192",
"sha256:50a74364d85fd319352182ef59c5c790484a336f6db772c1a9231f1c3ed0cbd7",
"sha256:54a2db7b78338edd780e7ef7f9f6c442500fb0d41a5a4ea24fff1c929d5af585",
"sha256:5635bd9cb9731e6d4a1132a498dd34f764034a8ce60cef4f5319c0541159392f",
"sha256:59c0b02d0a6c384d453fece7566d1c7e6b7bae4fc5874ef2ef46d56776d61c9e",
"sha256:5d598b938678ebf3c67377cdd45e09d431369c3b1a5b331058c338e201f12b27",
"sha256:5df2768244d19ab7f60546d0c7c63ce1581f7af8b5de3eb3004b9b6fc8a9f84b",
"sha256:5ef34d190326c3b1f822a5b7a45f6c4535e2f47ed06fec77d3d799c450b2651e",
"sha256:6975a3fac6bc83c4a65c9f9fcab9e47019a11d3d2cf7f3c0d03431bf145a941e",
"sha256:6c9a799e985904922a4d207a94eae35c78ebae90e128f0c4e521ce339396be9d",
"sha256:70df4e3b545a17496c9b3f41f5115e69a4f2e77e94e1d2a8e1070bc0c38c8a3c",
"sha256:7473e861101c9e72452f9bf8acb984947aa1661a7704553a9f6e4baa5ba64415",
"sha256:8102eaf27e1e448db915d08afa8b41d6c7ca7a04b7d73af6514df10a3e74bd82",
"sha256:87c450779d0914f2861b8526e035c5e6da0a3199d8f1add1a665e1cbc6fc6d02",
"sha256:8b7ee99e510d7b66cdb6c593f21c043c248537a32e0bedf02e01e9553a172314",
"sha256:91fc98adde3d7881af9b59ed0294046f3806221863722ba7d8d120c575314325",
"sha256:94411f22c3985acaec6f83c6df553f2dbe17b698cc7f8ae751ff2237d96b9e3c",
"sha256:98d85c6a2bef81588d9227dde12db8a7f47f639f4a17c9ae08e773aa9c697bf3",
"sha256:9ad5db27f9cabae298d151c85cf2bad1d359a1b9c686a275df03385758e2f914",
"sha256:a0b71b1b8fbf2b96e41c4d990244165e2c9be83d54962a9a1d118fd8657d2045",
"sha256:a0f100c8912c114ff53e1202d0078b425bee3649ae34d7b070e9697f93c5d52d",
"sha256:a591fe9e525846e4d154205572a029f653ada1a78b93697f3b5a8f1f2bc055b9",
"sha256:a5c84c68147988265e60416b57fc83425a78058853509c1b0629c180094904a5",
"sha256:a66d3508133af6e8548451b25058d5812812ec3798c886bf38ed24a98216fab2",
"sha256:a8c4917bd7ad33e8eb21e9a5bbba979b49d9a97acb3a803092cbc1133e20343c",
"sha256:b3bbeb01c2b273cca1e1e0c5df57f12dce9a4dd331b4fa1635b8bec26350bde3",
"sha256:cba9d6b9a7d64d4bd46167096fc9d2f835e25d7e4c121fb2ddfc6528fb0413b2",
"sha256:cc4d65aeeaa04136a12677d3dd0b1c0c94dc43abac5860ab33cceb42b801c1e8",
"sha256:ce4bcc037df4fc5e3d184794f27bdaab018943698f4ca31630bc7f84a7b69c6d",
"sha256:cec7d9412a9102bdc577382c3929b337320c4c4c4849f2c5cdd14d7368c5562d",
"sha256:d400bfb9a37b1351253cb402671cea7e89bdecc294e8016a707f6d1d8ac934f9",
"sha256:d61f4695e6c866a23a21acab0509af1cdfd2c013cf256bbf5b6b5e2695827162",
"sha256:db0fbb9c62743ce59a9ff687eb5f4afbe77e5e8403d6697f7446e5f609976f76",
"sha256:dd86c085fae2efd48ac91dd7ccffcfc0571387fe1193d33b6394db7ef31fe2a4",
"sha256:e00b098126fd45523dd056d2efba6c5a63b71ffe9f2bbe1a4fe1716e1d0c331e",
"sha256:e229a521186c75c8ad9490854fd8bbdd9a0c9aa3a524326b55be83b54d4e0ad9",
"sha256:e263d77ee3dd201c3a142934a086a4450861778baaeeb45db4591ef65550b0a6",
"sha256:ed9cb427ba5504c1dc15ede7d516b84757c3e3d7868ccc85121d9310d27eed0b",
"sha256:fa6693661a4c91757f4412306191b6dc88c1703f780c8234035eac011922bc01",
"sha256:fcd131dd944808b5bdb38e6f5b53013c5aa4f334c5cad0c72742f6eba4b73db0"
"sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc",
"sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a",
"sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417",
"sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab",
"sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520",
"sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36",
"sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743",
"sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8",
"sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed",
"sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684",
"sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56",
"sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324",
"sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d",
"sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235",
"sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e",
"sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088",
"sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000",
"sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7",
"sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e",
"sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673",
"sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c",
"sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe",
"sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2",
"sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098",
"sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8",
"sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a",
"sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0",
"sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b",
"sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896",
"sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e",
"sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9",
"sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2",
"sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b",
"sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6",
"sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404",
"sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f",
"sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0",
"sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4",
"sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc",
"sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936",
"sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba",
"sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872",
"sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb",
"sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614",
"sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1",
"sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d",
"sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969",
"sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b",
"sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4",
"sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627",
"sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956",
"sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357"
],
"version": "==1.15.1"
"markers": "platform_python_implementation != 'PyPy'",
"version": "==1.16.0"
},
"charset-normalizer": {
"hashes": [
"sha256:04e57ab9fbf9607b77f7d057974694b4f6b142da9ed4a199859d9d4d5c63fe96",
"sha256:09393e1b2a9461950b1c9a45d5fd251dc7c6f228acab64da1c9c0165d9c7765c",
"sha256:0b87549028f680ca955556e3bd57013ab47474c3124dc069faa0b6545b6c9710",
"sha256:1000fba1057b92a65daec275aec30586c3de2401ccdcd41f8a5c1e2c87078706",
"sha256:1249cbbf3d3b04902ff081ffbb33ce3377fa6e4c7356f759f3cd076cc138d020",
"sha256:1920d4ff15ce893210c1f0c0e9d19bfbecb7983c76b33f046c13a8ffbd570252",
"sha256:193cbc708ea3aca45e7221ae58f0fd63f933753a9bfb498a3b474878f12caaad",
"sha256:1a100c6d595a7f316f1b6f01d20815d916e75ff98c27a01ae817439ea7726329",
"sha256:1f30b48dd7fa1474554b0b0f3fdfdd4c13b5c737a3c6284d3cdc424ec0ffff3a",
"sha256:203f0c8871d5a7987be20c72442488a0b8cfd0f43b7973771640fc593f56321f",
"sha256:246de67b99b6851627d945db38147d1b209a899311b1305dd84916f2b88526c6",
"sha256:2dee8e57f052ef5353cf608e0b4c871aee320dd1b87d351c28764fc0ca55f9f4",
"sha256:2efb1bd13885392adfda4614c33d3b68dee4921fd0ac1d3988f8cbb7d589e72a",
"sha256:2f4ac36d8e2b4cc1aa71df3dd84ff8efbe3bfb97ac41242fbcfc053c67434f46",
"sha256:3170c9399da12c9dc66366e9d14da8bf7147e1e9d9ea566067bbce7bb74bd9c2",
"sha256:3b1613dd5aee995ec6d4c69f00378bbd07614702a315a2cf6c1d21461fe17c23",
"sha256:3bb3d25a8e6c0aedd251753a79ae98a093c7e7b471faa3aa9a93a81431987ace",
"sha256:3bb7fda7260735efe66d5107fb7e6af6a7c04c7fce9b2514e04b7a74b06bf5dd",
"sha256:41b25eaa7d15909cf3ac4c96088c1f266a9a93ec44f87f1d13d4a0e86c81b982",
"sha256:45de3f87179c1823e6d9e32156fb14c1927fcc9aba21433f088fdfb555b77c10",
"sha256:46fb8c61d794b78ec7134a715a3e564aafc8f6b5e338417cb19fe9f57a5a9bf2",
"sha256:48021783bdf96e3d6de03a6e39a1171ed5bd7e8bb93fc84cc649d11490f87cea",
"sha256:4957669ef390f0e6719db3613ab3a7631e68424604a7b448f079bee145da6e09",
"sha256:5e86d77b090dbddbe78867a0275cb4df08ea195e660f1f7f13435a4649e954e5",
"sha256:6339d047dab2780cc6220f46306628e04d9750f02f983ddb37439ca47ced7149",
"sha256:681eb3d7e02e3c3655d1b16059fbfb605ac464c834a0c629048a30fad2b27489",
"sha256:6c409c0deba34f147f77efaa67b8e4bb83d2f11c8806405f76397ae5b8c0d1c9",
"sha256:7095f6fbfaa55defb6b733cfeb14efaae7a29f0b59d8cf213be4e7ca0b857b80",
"sha256:70c610f6cbe4b9fce272c407dd9d07e33e6bf7b4aa1b7ffb6f6ded8e634e3592",
"sha256:72814c01533f51d68702802d74f77ea026b5ec52793c791e2da806a3844a46c3",
"sha256:7a4826ad2bd6b07ca615c74ab91f32f6c96d08f6fcc3902ceeedaec8cdc3bcd6",
"sha256:7c70087bfee18a42b4040bb9ec1ca15a08242cf5867c58726530bdf3945672ed",
"sha256:855eafa5d5a2034b4621c74925d89c5efef61418570e5ef9b37717d9c796419c",
"sha256:8700f06d0ce6f128de3ccdbc1acaea1ee264d2caa9ca05daaf492fde7c2a7200",
"sha256:89f1b185a01fe560bc8ae5f619e924407efca2191b56ce749ec84982fc59a32a",
"sha256:8b2c760cfc7042b27ebdb4a43a4453bd829a5742503599144d54a032c5dc7e9e",
"sha256:8c2f5e83493748286002f9369f3e6607c565a6a90425a3a1fef5ae32a36d749d",
"sha256:8e098148dd37b4ce3baca71fb394c81dc5d9c7728c95df695d2dca218edf40e6",
"sha256:94aea8eff76ee6d1cdacb07dd2123a68283cb5569e0250feab1240058f53b623",
"sha256:95eb302ff792e12aba9a8b8f8474ab229a83c103d74a750ec0bd1c1eea32e669",
"sha256:9bd9b3b31adcb054116447ea22caa61a285d92e94d710aa5ec97992ff5eb7cf3",
"sha256:9e608aafdb55eb9f255034709e20d5a83b6d60c054df0802fa9c9883d0a937aa",
"sha256:a103b3a7069b62f5d4890ae1b8f0597618f628b286b03d4bc9195230b154bfa9",
"sha256:a386ebe437176aab38c041de1260cd3ea459c6ce5263594399880bbc398225b2",
"sha256:a38856a971c602f98472050165cea2cdc97709240373041b69030be15047691f",
"sha256:a401b4598e5d3f4a9a811f3daf42ee2291790c7f9d74b18d75d6e21dda98a1a1",
"sha256:a7647ebdfb9682b7bb97e2a5e7cb6ae735b1c25008a70b906aecca294ee96cf4",
"sha256:aaf63899c94de41fe3cf934601b0f7ccb6b428c6e4eeb80da72c58eab077b19a",
"sha256:b0dac0ff919ba34d4df1b6131f59ce95b08b9065233446be7e459f95554c0dc8",
"sha256:baacc6aee0b2ef6f3d308e197b5d7a81c0e70b06beae1f1fcacffdbd124fe0e3",
"sha256:bf420121d4c8dce6b889f0e8e4ec0ca34b7f40186203f06a946fa0276ba54029",
"sha256:c04a46716adde8d927adb9457bbe39cf473e1e2c2f5d0a16ceb837e5d841ad4f",
"sha256:c0b21078a4b56965e2b12f247467b234734491897e99c1d51cee628da9786959",
"sha256:c1c76a1743432b4b60ab3358c937a3fe1341c828ae6194108a94c69028247f22",
"sha256:c4983bf937209c57240cff65906b18bb35e64ae872da6a0db937d7b4af845dd7",
"sha256:c4fb39a81950ec280984b3a44f5bd12819953dc5fa3a7e6fa7a80db5ee853952",
"sha256:c57921cda3a80d0f2b8aec7e25c8aa14479ea92b5b51b6876d975d925a2ea346",
"sha256:c8063cf17b19661471ecbdb3df1c84f24ad2e389e326ccaf89e3fb2484d8dd7e",
"sha256:ccd16eb18a849fd8dcb23e23380e2f0a354e8daa0c984b8a732d9cfaba3a776d",
"sha256:cd6dbe0238f7743d0efe563ab46294f54f9bc8f4b9bcf57c3c666cc5bc9d1299",
"sha256:d62e51710986674142526ab9f78663ca2b0726066ae26b78b22e0f5e571238dd",
"sha256:db901e2ac34c931d73054d9797383d0f8009991e723dab15109740a63e7f902a",
"sha256:e03b8895a6990c9ab2cdcd0f2fe44088ca1c65ae592b8f795c3294af00a461c3",
"sha256:e1c8a2f4c69e08e89632defbfabec2feb8a8d99edc9f89ce33c4b9e36ab63037",
"sha256:e4b749b9cc6ee664a3300bb3a273c1ca8068c46be705b6c31cf5d276f8628a94",
"sha256:e6a5bf2cba5ae1bb80b154ed68a3cfa2fa00fde979a7f50d6598d3e17d9ac20c",
"sha256:e857a2232ba53ae940d3456f7533ce6ca98b81917d47adc3c7fd55dad8fab858",
"sha256:ee4006268ed33370957f55bf2e6f4d263eaf4dc3cfc473d1d90baff6ed36ce4a",
"sha256:eef9df1eefada2c09a5e7a40991b9fc6ac6ef20b1372abd48d2794a316dc0449",
"sha256:f058f6963fd82eb143c692cecdc89e075fa0828db2e5b291070485390b2f1c9c",
"sha256:f25c229a6ba38a35ae6e25ca1264621cc25d4d38dca2942a7fce0b67a4efe918",
"sha256:f2a1d0fd4242bd8643ce6f98927cf9c04540af6efa92323e9d3124f57727bfc1",
"sha256:f7560358a6811e52e9c4d142d497f1a6e10103d3a6881f18d04dbce3729c0e2c",
"sha256:f779d3ad205f108d14e99bb3859aa7dd8e9c68874617c72354d7ecaec2a054ac",
"sha256:f87f746ee241d30d6ed93969de31e5ffd09a2961a051e60ae6bddde9ec3583aa"
"sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027",
"sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087",
"sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786",
"sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8",
"sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09",
"sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185",
"sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574",
"sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e",
"sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519",
"sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898",
"sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269",
"sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3",
"sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f",
"sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6",
"sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8",
"sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a",
"sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73",
"sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc",
"sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714",
"sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2",
"sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc",
"sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce",
"sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d",
"sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e",
"sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6",
"sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269",
"sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96",
"sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d",
"sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a",
"sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4",
"sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77",
"sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d",
"sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0",
"sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed",
"sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068",
"sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac",
"sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25",
"sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8",
"sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab",
"sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26",
"sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2",
"sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db",
"sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f",
"sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5",
"sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99",
"sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c",
"sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d",
"sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811",
"sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa",
"sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a",
"sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03",
"sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b",
"sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04",
"sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c",
"sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001",
"sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458",
"sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389",
"sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99",
"sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985",
"sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537",
"sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238",
"sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f",
"sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d",
"sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796",
"sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a",
"sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143",
"sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8",
"sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c",
"sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5",
"sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5",
"sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711",
"sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4",
"sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6",
"sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c",
"sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7",
"sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4",
"sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b",
"sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae",
"sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12",
"sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c",
"sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae",
"sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8",
"sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887",
"sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b",
"sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4",
"sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f",
"sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5",
"sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33",
"sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519",
"sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"
],
"markers": "python_full_version >= '3.7.0'",
"version": "==3.2.0"
"version": "==3.3.2"
},
"cryptography": {
"hashes": [
"sha256:004b6ccc95943f6a9ad3142cfabcc769d7ee38a3f60fb0dddbfb431f818c3a67",
"sha256:047c4603aeb4bbd8db2756e38f5b8bd7e94318c047cfe4efeb5d715e08b49311",
"sha256:0d9409894f495d465fe6fda92cb70e8323e9648af912d5b9141d616df40a87b8",
"sha256:23a25c09dfd0d9f28da2352503b23e086f8e78096b9fd585d1d14eca01613e13",
"sha256:2ed09183922d66c4ec5fdaa59b4d14e105c084dd0febd27452de8f6f74704143",
"sha256:35c00f637cd0b9d5b6c6bd11b6c3359194a8eba9c46d4e875a3660e3b400005f",
"sha256:37480760ae08065437e6573d14be973112c9e6dcaf5f11d00147ee74f37a3829",
"sha256:3b224890962a2d7b57cf5eeb16ccaafba6083f7b811829f00476309bce2fe0fd",
"sha256:5a0f09cefded00e648a127048119f77bc2b2ec61e736660b5789e638f43cc397",
"sha256:5b72205a360f3b6176485a333256b9bcd48700fc755fef51c8e7e67c4b63e3ac",
"sha256:7e53db173370dea832190870e975a1e09c86a879b613948f09eb49324218c14d",
"sha256:7febc3094125fc126a7f6fb1f420d0da639f3f32cb15c8ff0dc3997c4549f51a",
"sha256:80907d3faa55dc5434a16579952ac6da800935cd98d14dbd62f6f042c7f5e839",
"sha256:86defa8d248c3fa029da68ce61fe735432b047e32179883bdb1e79ed9bb8195e",
"sha256:8ac4f9ead4bbd0bc8ab2d318f97d85147167a488be0e08814a37eb2f439d5cf6",
"sha256:93530900d14c37a46ce3d6c9e6fd35dbe5f5601bf6b3a5c325c7bffc030344d9",
"sha256:9eeb77214afae972a00dee47382d2591abe77bdae166bda672fb1e24702a3860",
"sha256:b5f4dfe950ff0479f1f00eda09c18798d4f49b98f4e2006d644b3301682ebdca",
"sha256:c3391bd8e6de35f6f1140e50aaeb3e2b3d6a9012536ca23ab0d9c35ec18c8a91",
"sha256:c880eba5175f4307129784eca96f4e70b88e57aa3f680aeba3bab0e980b0f37d",
"sha256:cecfefa17042941f94ab54f769c8ce0fe14beff2694e9ac684176a2535bf9714",
"sha256:e40211b4923ba5a6dc9769eab704bdb3fbb58d56c5b336d30996c24fcf12aadb",
"sha256:efc8ad4e6fc4f1752ebfb58aefece8b4e3c4cae940b0994d43649bdfce8d0d4f"
"sha256:087887e55e0b9c8724cf05361357875adb5c20dec27e5816b653492980d20380",
"sha256:09a77e5b2e8ca732a19a90c5bca2d124621a1edb5438c5daa2d2738bfeb02589",
"sha256:130c0f77022b2b9c99d8cebcdd834d81705f61c68e91ddd614ce74c657f8b3ea",
"sha256:141e2aa5ba100d3788c0ad7919b288f89d1fe015878b9659b307c9ef867d3a65",
"sha256:28cb2c41f131a5758d6ba6a0504150d644054fd9f3203a1e8e8d7ac3aea7f73a",
"sha256:2f9f14185962e6a04ab32d1abe34eae8a9001569ee4edb64d2304bf0d65c53f3",
"sha256:320948ab49883557a256eab46149df79435a22d2fefd6a66fe6946f1b9d9d008",
"sha256:36d4b7c4be6411f58f60d9ce555a73df8406d484ba12a63549c88bd64f7967f1",
"sha256:3b15c678f27d66d247132cbf13df2f75255627bcc9b6a570f7d2fd08e8c081d2",
"sha256:3dbd37e14ce795b4af61b89b037d4bc157f2cb23e676fa16932185a04dfbf635",
"sha256:4383b47f45b14459cab66048d384614019965ba6c1a1a141f11b5a551cace1b2",
"sha256:44c95c0e96b3cb628e8452ec060413a49002a247b2b9938989e23a2c8291fc90",
"sha256:4b063d3413f853e056161eb0c7724822a9740ad3caa24b8424d776cebf98e7ee",
"sha256:52ed9ebf8ac602385126c9a2fe951db36f2cb0c2538d22971487f89d0de4065a",
"sha256:55d1580e2d7e17f45d19d3b12098e352f3a37fe86d380bf45846ef257054b242",
"sha256:5ef9bc3d046ce83c4bbf4c25e1e0547b9c441c01d30922d812e887dc5f125c12",
"sha256:5fa82a26f92871eca593b53359c12ad7949772462f887c35edaf36f87953c0e2",
"sha256:61321672b3ac7aade25c40449ccedbc6db72c7f5f0fdf34def5e2f8b51ca530d",
"sha256:701171f825dcab90969596ce2af253143b93b08f1a716d4b2a9d2db5084ef7be",
"sha256:841ec8af7a8491ac76ec5a9522226e287187a3107e12b7d686ad354bb78facee",
"sha256:8a06641fb07d4e8f6c7dda4fc3f8871d327803ab6542e33831c7ccfdcb4d0ad6",
"sha256:8e88bb9eafbf6a4014d55fb222e7360eef53e613215085e65a13290577394529",
"sha256:a00aee5d1b6c20620161984f8ab2ab69134466c51f58c052c11b076715e72929",
"sha256:a047682d324ba56e61b7ea7c7299d51e61fd3bca7dad2ccc39b72bd0118d60a1",
"sha256:a7ef8dd0bf2e1d0a27042b231a3baac6883cdd5557036f5e8df7139255feaac6",
"sha256:ad28cff53f60d99a928dfcf1e861e0b2ceb2bc1f08a074fdd601b314e1cc9e0a",
"sha256:b9097a208875fc7bbeb1286d0125d90bdfed961f61f214d3f5be62cd4ed8a446",
"sha256:b97fe7d7991c25e6a31e5d5e795986b18fbbb3107b873d5f3ae6dc9a103278e9",
"sha256:e0ec52ba3c7f1b7d813cd52649a5b3ef1fc0d433219dc8c93827c57eab6cf888",
"sha256:ea2c3ffb662fec8bbbfce5602e2c159ff097a4631d96235fcf0fb00e59e3ece4",
"sha256:fa3dec4ba8fb6e662770b74f62f1a0c7d4e37e25b58b2bf2c1be4c95372b4a33",
"sha256:fbeb725c9dc799a574518109336acccaf1303c30d45c075c665c0793c2f79a7f"
],
"markers": "python_version >= '3.7'",
"version": "==41.0.4"
"version": "==42.0.2"
},
"docker": {
"hashes": [
"sha256:aa6d17830045ba5ef0168d5eaa34d37beeb113948c413affe1d5991fc11f9a20",
"sha256:aecd2277b8bf8e506e484f6ab7aec39abe0038e29fa4a6d3ba86c3fe01844ed9"
"sha256:12ba681f2777a0ad28ffbcc846a69c31b4dfd9752b47eb425a274ee269c5e14b",
"sha256:323736fb92cd9418fc5e7133bc953e11a9da04f4483f828b527db553f1e7e5a3"
],
"markers": "python_version >= '3.7'",
"version": "==6.1.3"
"markers": "python_version >= '3.8'",
"version": "==7.0.0"
},
"execnet": {
"hashes": [
@ -221,11 +234,11 @@
},
"idna": {
"hashes": [
"sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4",
"sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"
"sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca",
"sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f"
],
"markers": "python_version >= '3.5'",
"version": "==3.4"
"version": "==3.6"
},
"iniconfig": {
"hashes": [
@ -237,39 +250,41 @@
},
"packaging": {
"hashes": [
"sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61",
"sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"
"sha256:048fb0e9405036518eaaf48a55953c750c11e1a1b68e0dd1a9d62ed0c092cfc5",
"sha256:8c491190033a9af7e1d931d0b5dacc2ef47509b34dd0de67ed209b5203fc88c7"
],
"markers": "python_version >= '3.7'",
"version": "==23.1"
"version": "==23.2"
},
"pluggy": {
"hashes": [
"sha256:cf61ae8f126ac6f7c451172cf30e3e43d3ca77615509771b3a984a0730651e12",
"sha256:d89c696a773f8bd377d18e5ecda92b7a3793cbe66c87060a6fb58c7b6e1061f7"
"sha256:7db9f7b503d67d1c5b95f59773ebb58a8c1c288129a88665838012cfb07b8981",
"sha256:8c85c2876142a764e5b7548e7d9a0e0ddb46f5185161049a79b7e974454223be"
],
"markers": "python_version >= '3.8'",
"version": "==1.3.0"
"version": "==1.4.0"
},
"psutil": {
"hashes": [
"sha256:104a5cc0e31baa2bcf67900be36acde157756b9c44017b86b2c049f11957887d",
"sha256:3c6f686f4225553615612f6d9bc21f1c0e305f75d7d8454f9b46e901778e7217",
"sha256:4aef137f3345082a3d3232187aeb4ac4ef959ba3d7c10c33dd73763fbc063da4",
"sha256:5410638e4df39c54d957fc51ce03048acd8e6d60abc0f5107af51e5fb566eb3c",
"sha256:5b9b8cb93f507e8dbaf22af6a2fd0ccbe8244bf30b1baad6b3954e935157ae3f",
"sha256:7a7dd9997128a0d928ed4fb2c2d57e5102bb6089027939f3b722f3a210f9a8da",
"sha256:89518112647f1276b03ca97b65cc7f64ca587b1eb0278383017c2a0dcc26cbe4",
"sha256:8c5f7c5a052d1d567db4ddd231a9d27a74e8e4a9c3f44b1032762bd7b9fdcd42",
"sha256:ab8ed1a1d77c95453db1ae00a3f9c50227ebd955437bcf2a574ba8adbf6a74d5",
"sha256:acf2aef9391710afded549ff602b5887d7a2349831ae4c26be7c807c0a39fac4",
"sha256:b258c0c1c9d145a1d5ceffab1134441c4c5113b2417fafff7315a917a026c3c9",
"sha256:be8929ce4313f9f8146caad4272f6abb8bf99fc6cf59344a3167ecd74f4f203f",
"sha256:c607bb3b57dc779d55e1554846352b4e358c10fff3abf3514a7a6601beebdb30",
"sha256:ea8518d152174e1249c4f2a1c89e3e6065941df2fa13a1ab45327716a23c2b48"
"sha256:02615ed8c5ea222323408ceba16c60e99c3f91639b07da6373fb7e6539abc56d",
"sha256:05806de88103b25903dff19bb6692bd2e714ccf9e668d050d144012055cbca73",
"sha256:26bd09967ae00920df88e0352a91cff1a78f8d69b3ecabbfe733610c0af486c8",
"sha256:27cc40c3493bb10de1be4b3f07cae4c010ce715290a5be22b98493509c6299e2",
"sha256:36f435891adb138ed3c9e58c6af3e2e6ca9ac2f365efe1f9cfef2794e6c93b4e",
"sha256:50187900d73c1381ba1454cf40308c2bf6f34268518b3f36a9b663ca87e65e36",
"sha256:611052c4bc70432ec770d5d54f64206aa7203a101ec273a0cd82418c86503bb7",
"sha256:6be126e3225486dff286a8fb9a06246a5253f4c7c53b475ea5f5ac934e64194c",
"sha256:7d79560ad97af658a0f6adfef8b834b53f64746d45b403f225b85c5c2c140eee",
"sha256:8cb6403ce6d8e047495a701dc7c5bd788add903f8986d523e3e20b98b733e421",
"sha256:8db4c1b57507eef143a15a6884ca10f7c73876cdf5d51e713151c1236a0e68cf",
"sha256:aee678c8720623dc456fa20659af736241f575d79429a0e5e9cf88ae0605cc81",
"sha256:bc56c2a1b0d15aa3eaa5a60c9f3f8e3e565303b465dbf57a1b730e7a2b9844e0",
"sha256:bd1184ceb3f87651a67b2708d4c3338e9b10c5df903f2e3776b62303b26cb631",
"sha256:d06016f7f8625a1825ba3732081d77c94589dca78b7a3fc072194851e88461a4",
"sha256:d16bbddf0693323b8c6123dd804100241da461e41d6e332fb0ba6058f630f8c8"
],
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3'",
"version": "==5.9.5"
"markers": "python_version >= '2.7' and python_version not in '3.0, 3.1, 3.2, 3.3, 3.4, 3.5'",
"version": "==5.9.8"
},
"pycparser": {
"hashes": [
@ -280,23 +295,23 @@
},
"pytest": {
"hashes": [
"sha256:1d881c6124e08ff0a1bb75ba3ec0bfd8b5354a01c194ddd5a0a870a48d99b002",
"sha256:a766259cfab564a2ad52cb1aae1b881a75c3eb7e34ca3779697c23ed47c47069"
"sha256:249b1b0864530ba251b7438274c4d251c58d868edaaec8762893ad4a0d71c36c",
"sha256:50fb9cbe836c3f20f0dfa99c565201fb75dc54c8d76373cd1bde06b06657bdb6"
],
"markers": "python_version >= '3.7'",
"version": "==7.4.2"
"markers": "python_version >= '3.8'",
"version": "==8.0.0"
},
"pytest-cs": {
"git": "https://github.com/crowdsecurity/pytest-cs.git",
"ref": "df835beabc539be7f7f627b21caa0d6ad333daae"
"ref": "aea7e8549faa32f5e1d1f17755a5db3712396a2a"
},
"pytest-datadir": {
"hashes": [
"sha256:095f441782b1b907587eca7227fdbae94be43f1c96b4b2cbcc6801a4645be1af",
"sha256:9f7a3c4def6ac4cac3cc8181139ab53bd2667231052bd40cb07081748d4420f0"
"sha256:1617ed92f9afda0c877e4eac91904b5f779d24ba8f5e438752e3ae39d8d2ee3f",
"sha256:34adf361bcc7b37961bbc1dfa8d25a4829e778bab461703c38a5c50ca9c36dc8"
],
"markers": "python_version >= '3.6'",
"version": "==1.4.1"
"markers": "python_version >= '3.8'",
"version": "==1.5.0"
},
"pytest-dotenv": {
"hashes": [
@ -308,19 +323,19 @@
},
"pytest-xdist": {
"hashes": [
"sha256:d5ee0520eb1b7bcca50a60a518ab7a7707992812c578198f8b44fdfac78e8c93",
"sha256:ff9daa7793569e6a68544850fd3927cd257cc03a7ef76c95e86915355e82b5f2"
"sha256:cbb36f3d67e0c478baa57fa4edc8843887e0f6cfc42d677530a36d7472b32d8a",
"sha256:d075629c7e00b611df89f490a5063944bee7a4362a5ff11c7cc7824a03dfce24"
],
"index": "pypi",
"version": "==3.3.1"
"version": "==3.5.0"
},
"python-dotenv": {
"hashes": [
"sha256:a8df96034aae6d2d50a4ebe8216326c61c3eb64836776504fcca410e5937a3ba",
"sha256:f5971a9226b701070a4bf2c38c89e5a3f0d64de8debda981d1db98583009122a"
"sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca",
"sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"
],
"markers": "python_version >= '3.8'",
"version": "==1.0.0"
"version": "==1.0.1"
},
"pyyaml": {
"hashes": [
@ -353,6 +368,7 @@
"sha256:8d4e9c88387b0f5c7d5f281e55304de64cf7f9c0021a3525bd3b1c542da3b0e4",
"sha256:9046c58c4395dff28dd494285c82ba00b546adfc7ef001486fbf0324bc174fba",
"sha256:9eb6caa9a297fc2c2fb8862bc5370d0303ddba53ba97e71f08023b6cd73d16a8",
"sha256:a08c6f0fe150303c1c6b71ebcd7213c2858041a7e01975da3a99aed1e7a378ef",
"sha256:a0cd17c15d3bb3fa06978b4e8958dcdc6e0174ccea823003a106c7d4d7899ac5",
"sha256:afd7e57eddb1a54f0f1a974bc4391af8bcce0b444685d936840f125cf046d5bd",
"sha256:b1275ad35a5d18c62a7220633c913e1b42d44b46ee12554e5fd39c70a243d6a3",
@ -396,35 +412,20 @@
},
"urllib3": {
"hashes": [
"sha256:13abf37382ea2ce6fb744d4dad67838eec857c9f4f57009891805e0b5e123594",
"sha256:ef16afa8ba34a1f989db38e1dbbe0c302e4289a47856990d0682e374563ce35e"
],
"markers": "python_version >= '3.7'",
"version": "==2.0.5"
},
"websocket-client": {
"hashes": [
"sha256:3aad25d31284266bcfcfd1fd8a743f63282305a364b8d0948a43bd606acc652f",
"sha256:6cfc30d051ebabb73a5fa246efdcc14c8fbebbd0330f8984ac3bb6d9edd2ad03"
"sha256:051d961ad0c62a94e50ecf1af379c3aba230c66c710493493560c0c223c49f20",
"sha256:ce3711610ddce217e6d113a2732fafad960a03fd0318c91faa79481e35c11224"
],
"markers": "python_version >= '3.8'",
"version": "==1.6.3"
"version": "==2.2.0"
}
},
"develop": {
"asttokens": {
"hashes": [
"sha256:2e0171b991b2c959acc6c49318049236844a5da1d65ba2672c4880c1c894834e",
"sha256:cf8fc9e61a86461aa9fb161a14a0841a03c405fa829ac6b202670b3495d2ce69"
"sha256:051ed49c3dcae8913ea7cd08e46a606dba30b79993209636c4875bc1d637bc24",
"sha256:b03869718ba9a6eb027e134bfdf69f38a236d681c83c160d510768af11254ba0"
],
"version": "==2.4.0"
},
"backcall": {
"hashes": [
"sha256:5cbdbf27be5e7cfadb448baf0aa95508f91f2bbc6c6437cd9cd06e2a4c215e1e",
"sha256:fbbce6a29f263178a1f7915c1940bde0ec2b2a967566fe1c65c1dfb7422bd255"
],
"version": "==0.2.0"
"version": "==2.4.1"
},
"decorator": {
"hashes": [
@ -436,10 +437,11 @@
},
"executing": {
"hashes": [
"sha256:0314a69e37426e3608aada02473b4161d4caf5a4b244d1d0c48072b8fee7bacc",
"sha256:19da64c18d2d851112f09c287f8d3dbbdf725ab0e569077efb6cdcbd3497c107"
"sha256:35afe2ce3affba8ee97f2d69927fa823b08b472b7b994e36a52a964b93d16147",
"sha256:eac49ca94516ccc753f9fb5ce82603156e590b27525a8bc32cce8ae302eb61bc"
],
"version": "==1.2.0"
"markers": "python_version >= '3.5'",
"version": "==2.0.1"
},
"gnureadline": {
"hashes": [
@ -484,19 +486,19 @@
},
"ipython": {
"hashes": [
"sha256:2baeb5be6949eeebf532150f81746f8333e2ccce02de1c7eedde3f23ed5e9f1e",
"sha256:45a2c3a529296870a97b7de34eda4a31bee16bc7bf954e07d39abe49caf8f887"
"sha256:1050a3ab8473488d7eee163796b02e511d0735cf43a04ba2a8348bd0f2eaf8a5",
"sha256:48fbc236fbe0e138b88773fa0437751f14c3645fb483f1d4c5dee58b37e5ce73"
],
"markers": "python_version >= '3.11'",
"version": "==8.15.0"
"version": "==8.21.0"
},
"jedi": {
"hashes": [
"sha256:bcf9894f1753969cbac8022a8c2eaee06bfa3724e4192470aaffe7eb6272b0c4",
"sha256:cb8ce23fbccff0025e9386b5cf85e892f94c9b822378f8da49970471335ac64e"
"sha256:cf0496f3651bc65d7174ac1b7d043eff454892c708a87d1b683e57b569927ffd",
"sha256:e983c654fe5c02867aef4cdfce5a2fbb4a50adc0af145f70504238f18ef5e7e0"
],
"markers": "python_version >= '3.6'",
"version": "==0.19.0"
"version": "==0.19.1"
},
"matplotlib-inline": {
"hashes": [
@ -516,26 +518,19 @@
},
"pexpect": {
"hashes": [
"sha256:0b48a55dcb3c05f3329815901ea4fc1537514d6ba867a152b581d69ae3710937",
"sha256:fc65a43959d153d0114afe13997d439c22823a27cefceb5ff35c2178c6784c0c"
"sha256:7236d1e080e4936be2dc3e326cec0af72acf9212a7e1d060210e70a47e253523",
"sha256:ee7d41123f3c9911050ea2c2dac107568dc43b2d3b0c7557a33212c398ead30f"
],
"markers": "sys_platform != 'win32'",
"version": "==4.8.0"
},
"pickleshare": {
"hashes": [
"sha256:87683d47965c1da65cdacaf31c8441d12b8044cdec9aca500cd78fc2c683afca",
"sha256:9649af414d74d4df115d5d718f82acb59c9d418196b7b4290ed47a12ce62df56"
],
"version": "==0.7.5"
"version": "==4.9.0"
},
"prompt-toolkit": {
"hashes": [
"sha256:04505ade687dc26dc4284b1ad19a83be2f2afe83e7a828ace0c72f3a1df72aac",
"sha256:9dffbe1d8acf91e3de75f3b544e4842382fc06c6babe903ac9acb74dc6e08d88"
"sha256:3527b7af26106cbc65a040bcc84839a3566ec1b051bb0bfe953631e704b0ff7d",
"sha256:a11a29cb3bf0a28a387fe5122cdb649816a957cd9261dcedf8c9f1fef33eacf6"
],
"markers": "python_full_version >= '3.7.0'",
"version": "==3.0.39"
"version": "==3.0.43"
},
"ptyprocess": {
"hashes": [
@ -553,11 +548,11 @@
},
"pygments": {
"hashes": [
"sha256:13fc09fa63bc8d8671a6d247e1eb303c4b343eaee81d861f3404db2935653692",
"sha256:1daff0494820c69bc8941e407aa20f577374ee88364ee10a98fdbe0aece96e29"
"sha256:b27c2826c47d0f3219f29554824c30c5e8945175d888647acd804ddd04af846c",
"sha256:da46cec9fd2de5be3a8a784f434e4c4ab670b4ff54d605c4c2717e9d49c4c367"
],
"markers": "python_version >= '3.7'",
"version": "==2.16.1"
"version": "==2.17.2"
},
"six": {
"hashes": [
@ -569,25 +564,25 @@
},
"stack-data": {
"hashes": [
"sha256:32d2dd0376772d01b6cb9fc996f3c8b57a357089dec328ed4b6553d037eaf815",
"sha256:cbb2a53eb64e5785878201a97ed7c7b94883f48b87bfb0bbe8b623c74679e4a8"
"sha256:836a778de4fec4dcd1dcd89ed8abff8a221f58308462e1c4aa2a3cf30148f0b9",
"sha256:d5558e0c25a4cb0853cddad3d77da9891a08cb85dd9f9f91b9f8cd66e511e695"
],
"version": "==0.6.2"
"version": "==0.6.3"
},
"traitlets": {
"hashes": [
"sha256:417745a96681fbb358e723d5346a547521f36e9bd0d50ba7ab368fff5d67aa54",
"sha256:f584ea209240466e66e91f3c81aa7d004ba4cf794990b0c775938a1544217cd1"
"sha256:2e5a030e6eff91737c643231bfcf04a65b0132078dad75e4936700b213652e74",
"sha256:8585105b371a04b8316a43d5ce29c098575c2e477850b62b848b964f1444527e"
],
"markers": "python_version >= '3.8'",
"version": "==5.10.0"
"version": "==5.14.1"
},
"wcwidth": {
"hashes": [
"sha256:795b138f6875577cd91bba52baf9e445cd5118fd32723b460e30a0af30ea230e",
"sha256:a5220780a404dbe3353789870978e472cfe477761f06ee55077256e509b156d0"
"sha256:3da69048e4540d84af32131829ff948f1e022c1c6bdb8d6102117aac784f6859",
"sha256:72ea0c06399eb286d978fdedb6923a9eb47e1c486ce63e9b4e64fc18303972b5"
],
"version": "==0.2.6"
"version": "==0.2.13"
}
}
}

10
docker/test/tests/test_agent.py
View file

@ -13,7 +13,7 @@ def test_no_agent(crowdsec, flavor):
'DISABLE_AGENT': 'true',
}
with crowdsec(flavor=flavor, environment=env) as cs:
cs.wait_for_log("*CrowdSec Local API listening on 0.0.0.0:8080*")
cs.wait_for_log("*CrowdSec Local API listening on *:8080*")
cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
res = cs.cont.exec_run('cscli lapi status')
assert res.exit_code == 0
@ -37,7 +37,7 @@ def test_machine_register(crowdsec, flavor, tmp_path_factory):
with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs:
cs.wait_for_log([
"*Generate local agent credentials*",
"*CrowdSec Local API listening on 0.0.0.0:8080*",
"*CrowdSec Local API listening on *:8080*",
])
cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
res = cs.cont.exec_run('cscli lapi status')
@ -50,7 +50,7 @@ def test_machine_register(crowdsec, flavor, tmp_path_factory):
with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs:
cs.wait_for_log([
"*Generate local agent credentials*",
"*CrowdSec Local API listening on 0.0.0.0:8080*",
"*CrowdSec Local API listening on *:8080*",
])
cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
res = cs.cont.exec_run('cscli lapi status')
@ -65,7 +65,7 @@ def test_machine_register(crowdsec, flavor, tmp_path_factory):
with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs:
cs.wait_for_log([
"*Generate local agent credentials*",
"*CrowdSec Local API listening on 0.0.0.0:8080*",
"*CrowdSec Local API listening on *:8080*",
])
cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
res = cs.cont.exec_run('cscli lapi status')
@ -78,7 +78,7 @@ def test_machine_register(crowdsec, flavor, tmp_path_factory):
with crowdsec(flavor=flavor, environment=env, volumes=volumes) as cs:
cs.wait_for_log([
"*Local agent already registered*",
"*CrowdSec Local API listening on 0.0.0.0:8080*",
"*CrowdSec Local API listening on *:8080*",
])
cs.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
res = cs.cont.exec_run('cscli lapi status')

2
docker/test/tests/test_agent_only.py
View file

@ -29,7 +29,7 @@ def test_split_lapi_agent(crowdsec, flavor):
cs_agent = crowdsec(name=agentname, environment=agent_env, flavor=flavor)
with cs_lapi as lapi:
lapi.wait_for_log("*CrowdSec Local API listening on 0.0.0.0:8080*")
lapi.wait_for_log("*CrowdSec Local API listening on *:8080*")
lapi.wait_for_http(8080, '/health', want_status=HTTPStatus.OK)
with cs_agent as agent:
agent.wait_for_log("*Starting processing data*")

Some files were not shown because too many files have changed in this diff Show more