0ct0pu5/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'master' into handle_highAvailability

Browse source
This commit is contained in:
marco 2024-02-09 15:19:15 +01:00
commit 4bdcb33958
400 changed files with 24808 additions and 12216 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.github/governance.yml vendored
View file

@ -81,7 +81,7 @@ pull_request:
failure: Missing kind label to generate release automatically.
- prefix: area
list: [ "agent", "local-api", "cscli", "security", "configuration"]
list: [ "agent", "local-api", "cscli", "security", "configuration", "appsec"]
multiple: true
needs:
comment: |
@ -89,6 +89,7 @@ pull_request:
* `/area agent`
* `/area local-api`
* `/area cscli`
* `/area appsec`
* `/area security`
* `/area configuration`
@ -98,4 +99,4 @@ pull_request:
author_association:
collaborator: true
member: true
owner: true
owner: true

20
.github/workflows/bats-hub.yml vendored
View file

@ -1,4 +1,4 @@
name: Hub tests
name: (sub) Bats / Hub
on:
workflow_call:
@ -15,9 +15,9 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.21.1"]
test-file: ["hub-1.bats", "hub-2.bats", "hub-3.bats"]
name: "Build + tests"
name: "Functional tests"
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
@ -28,27 +28,29 @@ jobs:
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Check out CrowdSec repository"
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: true
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
go-version: "1.21.6"
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
- name: "Build crowdsec and fixture"
run: make bats-clean bats-build bats-fixture BUILD_STATIC=1
- name: "Run hub tests"
run: make bats-test-hub
run: |
./test/bin/generate-hub-tests
./test/run-tests test/dyn-bats/${{ matrix.test-file }}
- name: "Collect hub coverage"
run: ./test/bin/collect-hub-coverage >> $GITHUB_ENV

18
.github/workflows/bats-mysql.yml vendored
View file

@ -1,4 +1,4 @@
name: Functional tests (MySQL)
name: (sub) Bats / MySQL
on:
workflow_call:
@ -12,11 +12,7 @@ env:
jobs:
build:
strategy:
matrix:
go-version: ["1.21.1"]
name: "Build + tests"
name: "Functional tests"
runs-on: ubuntu-latest
timeout-minutes: 30
services:
@ -35,21 +31,21 @@ jobs:
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Check out CrowdSec repository"
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: true
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
go-version: "1.21.6"
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
- name: "Build crowdsec and fixture"
run: |

18
.github/workflows/bats-postgres.yml vendored
View file

@ -1,4 +1,4 @@
name: Functional tests (Postgres)
name: (sub) Bats / Postgres
on:
workflow_call:
@ -8,11 +8,7 @@ env:
jobs:
build:
strategy:
matrix:
go-version: ["1.21.1"]
name: "Build + tests"
name: "Functional tests"
runs-on: ubuntu-latest
timeout-minutes: 30
services:
@ -44,21 +40,21 @@ jobs:
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Check out CrowdSec repository"
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: true
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
go-version: "1.21.6"
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
- name: "Build crowdsec and fixture (DB_BACKEND: pgx)"
run: |

18
.github/workflows/bats-sqlite-coverage.yml vendored
View file

@ -1,4 +1,4 @@
name: Functional tests (sqlite)
name: (sub) Bats / sqlite + coverage
on:
workflow_call:
@ -9,11 +9,7 @@ env:
jobs:
build:
strategy:
matrix:
go-version: ["1.21.1"]
name: "Build + tests"
name: "Functional tests"
runs-on: ubuntu-latest
timeout-minutes: 20
@ -25,21 +21,21 @@ jobs:
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Check out CrowdSec repository"
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: true
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
go-version: "1.21.6"
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
- name: "Build crowdsec and fixture"
run: |

2
.github/workflows/bats.yml vendored
View file

@ -31,7 +31,7 @@ jobs:
# Jobs for Postgres (and sometimes MySQL) can have failing tests on GitHub
# CI, but they pass when run on devs' machines or in the release checks. We
# disable them here by default. Remove the if..false to enable them.
# disable them here by default. Remove if...false to enable them.
mariadb:
uses: ./.github/workflows/bats-mysql.yml

2
.github/workflows/cache-cleanup.yaml vendored
View file

@ -11,7 +11,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v3
uses: actions/checkout@v4
- name: Cleanup
run: |

12
.github/workflows/ci-windows-build-msi.yml vendored
View file

@ -21,25 +21,21 @@ on:
jobs:
build:
strategy:
matrix:
go-version: ["1.21.1"]
name: Build
runs-on: windows-2019
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: false
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
go-version: "1.21.6"
- name: Build
run: make windows_installer BUILD_RE2_WASM=1

6
.github/workflows/codeql-analysis.yml vendored
View file

@ -44,7 +44,7 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
# required to pick up tags for BUILD_VERSION
fetch-depth: 0
@ -72,9 +72,9 @@ jobs:
# uses a compiled language
- name: "Set up Go"
uses: actions/setup-go@v4
uses: actions/setup-go@v5
with:
go-version: "1.21.0"
go-version: "1.21.6"
cache-dependency-path: "**/go.sum"
- run: |

56
.github/workflows/docker-tests.yml vendored
View file

@ -15,59 +15,42 @@ on:
- 'README.md'
jobs:
test_docker_image:
test_flavor:
strategy:
# we could test all the flavors in a single pytest job,
# but let's split them (and the image build) in multiple runners for performance
matrix:
# can be slim, full or debian (no debian slim).
flavor: ["slim", "debian"]
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Check out the repo
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
uses: docker/setup-buildx-action@v3
with:
config: .github/buildkit.toml
- name: "Build flavor: slim"
uses: docker/build-push-action@v4
- name: "Build image"
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile
tags: crowdsecurity/crowdsec:test-slim
target: slim
platforms: linux/amd64
load: true
cache-from: type=gha
cache-to: type=gha,mode=min
- name: "Build flavor: full"
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile
tags: crowdsecurity/crowdsec:test
target: full
platforms: linux/amd64
load: true
cache-from: type=gha
cache-to: type=gha,mode=min
- name: "Build flavor: full (debian)"
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile.debian
tags: crowdsecurity/crowdsec:test-debian
target: full
file: ./Dockerfile${{ matrix.flavor == 'debian' && '.debian' || '' }}
tags: crowdsecurity/crowdsec:test${{ matrix.flavor == 'full' && '' || '-' }}${{ matrix.flavor == 'full' && '' || matrix.flavor }}
target: ${{ matrix.flavor == 'debian' && 'full' || matrix.flavor }}
platforms: linux/amd64
load: true
cache-from: type=gha
cache-to: type=gha,mode=min
- name: "Setup Python"
uses: actions/setup-python@v4
uses: actions/setup-python@v5
with:
python-version: "3.x"
@ -78,7 +61,7 @@ jobs:
- name: "Cache virtualenvs"
id: cache-pipenv
uses: actions/cache@v3
uses: actions/cache@v4
with:
path: ~/.local/share/virtualenvs
key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
@ -95,9 +78,10 @@ jobs:
- name: "Run tests"
env:
CROWDSEC_TEST_VERSION: test
CROWDSEC_TEST_FLAVORS: slim,debian
CROWDSEC_TEST_FLAVORS: ${{ matrix.flavor }}
CROWDSEC_TEST_NETWORK: net-test
CROWDSEC_TEST_TIMEOUT: 90
# running serially to reduce test flakiness
run: |
cd docker/test
pipenv run pytest -n 2 --durations=0 --color=yes
pipenv run pytest -n 1 --durations=0 --color=yes

14
.github/workflows/go-tests-windows.yml vendored
View file

@ -20,25 +20,21 @@ env:
jobs:
build:
strategy:
matrix:
go-version: ["1.21.1"]
name: "Build + tests"
runs-on: windows-2022
steps:
- name: Check out CrowdSec repository
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: false
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
go-version: "1.21.6"
- name: Build
run: |
@ -60,7 +56,7 @@ jobs:
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
version: v1.54
version: v1.55
args: --issues-exit-code=1 --timeout 10m
only-new-issues: false
# the cache is already managed above, enabling it here

47
.github/workflows/go-tests.yml vendored
View file

@ -24,23 +24,18 @@ env:
RICHGO_FORCE_COLOR: 1
AWS_HOST: localstack
# these are to mimic aws config
AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_REGION: us-east-1
KINESIS_INITIALIZE_STREAMS: "stream-1-shard:1,stream-2-shards:2"
CROWDSEC_FEATURE_DISABLE_HTTP_RETRY_BACKOFF: true
jobs:
build:
strategy:
matrix:
go-version: ["1.21.1"]
name: "Build + tests"
runs-on: ubuntu-latest
services:
localstack:
image: localstack/localstack:1.3.0
image: localstack/localstack:3.0
ports:
- 4566:4566 # Localstack exposes all services on the same port
env:
@ -49,7 +44,7 @@ jobs:
KINESIS_ERROR_PROBABILITY: ""
DOCKER_HOST: unix:///var/run/docker.sock
KINESIS_INITIALIZE_STREAMS: ${{ env.KINESIS_INITIALIZE_STREAMS }}
HOSTNAME_EXTERNAL: ${{ env.AWS_HOST }} # Required so that resource urls are provided properly
LOCALSTACK_HOST: ${{ env.AWS_HOST }} # Required so that resource urls are provided properly
# e.g sqs url will get localhost if we don't set this env to map our service
options: >-
--name=localstack
@ -58,7 +53,7 @@ jobs:
--health-timeout=5s
--health-retries=3
zoo1:
image: confluentinc/cp-zookeeper:7.3.0
image: confluentinc/cp-zookeeper:7.4.3
ports:
- "2181:2181"
env:
@ -108,18 +103,35 @@ jobs:
--health-timeout 10s
--health-retries 5
loki:
image: grafana/loki:2.9.1
ports:
- "3100:3100"
options: >-
--name=loki1
--health-cmd "wget -q -O - http://localhost:3100/ready | grep 'ready'"
--health-interval 30s
--health-timeout 10s
--health-retries 5
--health-start-period 30s
steps:
- name: Check out CrowdSec repository
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: false
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
go-version: "1.21.6"
- name: Create localstack streams
run: |
aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-1-shard --shard-count 1
aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-2-shards --shard-count 2
- name: Build and run tests, static
run: |
@ -128,12 +140,13 @@ jobs:
go install github.com/kyoh86/richgo@v0.3.10
set -o pipefail
make build BUILD_STATIC=1
make go-acc | richgo testfilter
make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
- name: Run tests again, dynamic
run: |
make clean build
make go-acc | richgo testfilter
set -o pipefail
make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
- name: Upload unit coverage to Codecov
uses: codecov/codecov-action@v3
@ -144,7 +157,7 @@ jobs:
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
version: v1.54
version: v1.55
args: --issues-exit-code=1 --timeout 10m
only-new-issues: false
# the cache is already managed above, enabling it here

47
.github/workflows/publish-docker-master.yml vendored Normal file
View file

@ -0,0 +1,47 @@
name: (push-master) Publish latest Docker images
on:
push:
branches: [ master ]
paths:
- 'pkg/**'
- 'cmd/**'
- 'mk/**'
- 'docker/docker_start.sh'
- 'docker/config.yaml'
- '.github/workflows/publish-docker-master.yml'
- '.github/workflows/publish-docker.yml'
- 'Dockerfile'
- 'Dockerfile.debian'
- 'go.mod'
- 'go.sum'
- 'Makefile'
jobs:
dev-alpine:
uses: ./.github/workflows/publish-docker.yml
with:
platform: linux/amd64
crowdsec_version: ""
image_version: dev
latest: false
push: true
slim: false
debian: false
secrets:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
dev-debian:
uses: ./.github/workflows/publish-docker.yml
with:
platform: linux/amd64
crowdsec_version: ""
image_version: dev
latest: false
push: true
slim: false
debian: true
secrets:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}

48
.github/workflows/publish-docker-release.yml vendored Normal file
View file

@ -0,0 +1,48 @@
name: (manual) Publish Docker images
on:
workflow_dispatch:
inputs:
image_version:
description: Docker Image version (base tag, i.e. v1.6.0-2)
required: true
crowdsec_version:
description: Crowdsec version (BUILD_VERSION)
required: true
latest:
description: Overwrite latest (and slim) tags?
default: false
required: true
push:
description: Really push?
default: false
required: true
jobs:
alpine:
uses: ./.github/workflows/publish-docker.yml
secrets:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
with:
image_version: ${{ github.event.inputs.image_version }}
crowdsec_version: ${{ github.event.inputs.crowdsec_version }}
latest: ${{ github.event.inputs.latest == 'true' }}
push: ${{ github.event.inputs.push == 'true' }}
slim: true
debian: false
platform: "linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6"
debian:
uses: ./.github/workflows/publish-docker.yml
secrets:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
with:
image_version: ${{ github.event.inputs.image_version }}
crowdsec_version: ${{ github.event.inputs.crowdsec_version }}
latest: ${{ github.event.inputs.latest == 'true' }}
push: ${{ github.event.inputs.push == 'true' }}
slim: false
debian: true
platform: "linux/amd64,linux/386,linux/arm64"

125
.github/workflows/publish-docker.yml vendored Normal file
View file

@ -0,0 +1,125 @@
name: (sub) Publish Docker images
on:
workflow_call:
secrets:
DOCKER_USERNAME:
required: true
DOCKER_PASSWORD:
required: true
inputs:
platform:
required: true
type: string
image_version:
required: true
type: string
crowdsec_version:
required: true
type: string
latest:
required: true
type: boolean
push:
required: true
type: boolean
slim:
required: true
type: boolean
debian:
required: true
type: boolean
jobs:
push_to_registry:
name: Push Docker image to registries
runs-on: ubuntu-latest
steps:
- name: Check out the repo
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
config: .github/buildkit.toml
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Prepare (slim)
if: ${{ inputs.slim }}
id: slim
run: |
DOCKERHUB_IMAGE=${{ secrets.DOCKER_USERNAME }}/crowdsec
GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
VERSION=${{ inputs.image_version }}
DEBIAN=${{ inputs.debian && '-debian' || '' }}
TAGS="${DOCKERHUB_IMAGE}:${VERSION}-slim${DEBIAN},${GHCR_IMAGE}:${VERSION}-slim${DEBIAN}"
if [[ ${{ inputs.latest }} == true ]]; then
TAGS=$TAGS,${DOCKERHUB_IMAGE}:slim${DEBIAN},${GHCR_IMAGE}:slim${DEBIAN}
fi
echo "tags=${TAGS}" >> $GITHUB_OUTPUT
echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
- name: Prepare (full)
id: full
run: |
DOCKERHUB_IMAGE=${{ secrets.DOCKER_USERNAME }}/crowdsec
GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
VERSION=${{ inputs.image_version }}
DEBIAN=${{ inputs.debian && '-debian' || '' }}
TAGS="${DOCKERHUB_IMAGE}:${VERSION}${DEBIAN},${GHCR_IMAGE}:${VERSION}${DEBIAN}"
if [[ ${{ inputs.latest }} == true ]]; then
TAGS=$TAGS,${DOCKERHUB_IMAGE}:latest${DEBIAN},${GHCR_IMAGE}:latest${DEBIAN}
fi
echo "tags=${TAGS}" >> $GITHUB_OUTPUT
echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
- name: Build and push image (slim)
if: ${{ inputs.slim }}
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile${{ inputs.debian && '.debian' || '' }}
push: ${{ inputs.push }}
tags: ${{ steps.slim.outputs.tags }}
target: slim
platforms: ${{ inputs.platform }}
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.slim.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}
build-args: |
BUILD_VERSION=${{ inputs.crowdsec_version }}
- name: Build and push image (full)
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile${{ inputs.debian && '.debian' || '' }}
push: ${{ inputs.push }}
tags: ${{ steps.full.outputs.tags }}
target: full
platforms: ${{ inputs.platform }}
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.full.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}
build-args: |
BUILD_VERSION=${{ inputs.crowdsec_version }}

16
.github/workflows/release_publish-package.yml → .github/workflows/publish-tarball-release.yml vendored
View file

@ -1,5 +1,5 @@
# .github/workflows/build-docker-image.yml
name: build
name: Release
on:
release:
@ -12,24 +12,20 @@ permissions:
jobs:
build:
strategy:
matrix:
go-version: ["1.21.1"]
name: Build and upload binary package
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v3
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: false
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v4
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: ${{ matrix.go-version }}
go-version: "1.21.6"
- name: Build the binaries
run: |
@ -41,4 +37,4 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
tag_name="${GITHUB_REF##*/}"
hub release edit -a crowdsec-release.tgz -a vendor.tgz -a *-vendor.tar.xz -m "" "$tag_name"
gh release upload "$tag_name" crowdsec-release.tgz vendor.tgz *-vendor.tar.xz

70
.github/workflows/publish_docker-image_on_master-debian.yml vendored
View file

@ -1,70 +0,0 @@
name: Publish Debian Docker image on Push to Master
on:
push:
branches: [ master ]
paths:
- 'pkg/**'
- 'cmd/**'
- 'plugins/**'
- 'docker/docker_start.sh'
- 'docker/config.yaml'
- '.github/workflows/publish_docker-image_on_master-debian.yml'
- 'Dockerfile.debian'
- 'go.mod'
- 'go.sum'
- 'Makefile'
jobs:
push_to_registry:
name: Push Debian Docker image to Docker Hub
runs-on: ubuntu-latest
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Prepare
id: prep
run: |
DOCKER_IMAGE=crowdsecurity/crowdsec
GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
VERSION=dev-debian
TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}"
echo "tags=${TAGS}" >> $GITHUB_OUTPUT
echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
config: .github/buildkit.toml
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push full image
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile.debian
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.prep.outputs.tags }}
platforms: linux/amd64
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.prep.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}
cache-from: type=gha
cache-to: type=gha,mode=min

70
.github/workflows/publish_docker-image_on_master.yml vendored
View file

@ -1,70 +0,0 @@
name: Publish Docker image on Push to Master
on:
push:
branches: [ master ]
paths:
- 'pkg/**'
- 'cmd/**'
- 'plugins/**'
- 'docker/docker_start.sh'
- 'docker/config.yaml'
- '.github/workflows/publish_docker-image_on_master.yml'
- 'Dockerfile'
- 'go.mod'
- 'go.sum'
- 'Makefile'
jobs:
push_to_registry:
name: Push Docker image to Docker Hub
runs-on: ubuntu-latest
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Prepare
id: prep
run: |
DOCKER_IMAGE=crowdsecurity/crowdsec
GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
VERSION=dev
TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}"
echo "tags=${TAGS}" >> $GITHUB_OUTPUT
echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
config: .github/buildkit.toml
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push full image
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.prep.outputs.tags }}
platforms: linux/amd64
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.prep.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}
cache-from: type=gha
cache-to: type=gha,mode=min

61
.github/workflows/release_publish_docker-image-debian.yml vendored
View file

@ -1,61 +0,0 @@
name: Publish Docker Debian image
on:
release:
types:
- released
- prereleased
workflow_dispatch:
jobs:
push_to_registry:
name: Push Docker debian image to Docker Hub
runs-on: ubuntu-latest
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Prepare
id: prep
run: |
DOCKER_IMAGE=crowdsecurity/crowdsec
VERSION=bullseye
if [[ $GITHUB_REF == refs/tags/* ]]; then
VERSION=${GITHUB_REF#refs/tags/}
elif [[ $GITHUB_REF == refs/heads/* ]]; then
VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -E 's#/+#-#g')
elif [[ $GITHUB_REF == refs/pull/* ]]; then
VERSION=pr-${{ github.event.number }}
fi
TAGS="${DOCKER_IMAGE}:${VERSION}-debian"
if [[ "${{ github.event.action }}" == "released" ]]; then
TAGS=$TAGS,${DOCKER_IMAGE}:latest-debian
fi
echo "version=${VERSION}" >> $GITHUB_OUTPUT
echo "tags=${TAGS}" >> $GITHUB_OUTPUT
echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
config: .github/buildkit.toml
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile.debian
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.prep.outputs.tags }}
platforms: linux/amd64,linux/arm64,linux/386
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.prep.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}

86
.github/workflows/release_publish_docker-image.yml vendored
View file

@ -1,86 +0,0 @@
name: Publish Docker image
on:
release:
types:
- released
- prereleased
jobs:
push_to_registry:
name: Push Docker image to Docker Hub
runs-on: ubuntu-latest
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Prepare
id: prep
run: |
DOCKER_IMAGE=crowdsecurity/crowdsec
GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
VERSION=edge
if [[ $GITHUB_REF == refs/tags/* ]]; then
VERSION=${GITHUB_REF#refs/tags/}
elif [[ $GITHUB_REF == refs/heads/* ]]; then
VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -E 's#/+#-#g')
elif [[ $GITHUB_REF == refs/pull/* ]]; then
VERSION=pr-${{ github.event.number }}
fi
TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}"
TAGS_SLIM="${DOCKER_IMAGE}:${VERSION}-slim"
if [[ ${{ github.event.action }} == released ]]; then
TAGS=$TAGS,${DOCKER_IMAGE}:latest,${GHCR_IMAGE}:latest
TAGS_SLIM=$TAGS_SLIM,${DOCKER_IMAGE}:slim
fi
echo "version=${VERSION}" >> $GITHUB_OUTPUT
echo "tags=${TAGS}" >> $GITHUB_OUTPUT
echo "tags_slim=${TAGS_SLIM}" >> $GITHUB_OUTPUT
echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
config: .github/buildkit.toml
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push slim image
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.prep.outputs.tags_slim }}
target: slim
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6,linux/386
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.prep.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}
- name: Build and push full image
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.prep.outputs.tags }}
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6,linux/386
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.prep.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}

4
.github/workflows/update_docker_hub_doc.yml vendored
View file

@ -1,4 +1,4 @@
name: Update Docker Hub README
name: (push-master) Update Docker Hub README
on:
push:
@ -13,7 +13,7 @@ jobs:
steps:
-
name: Check out the repo
uses: actions/checkout@v3
uses: actions/checkout@v4
if: ${{ github.repository_owner == 'crowdsecurity' }}
-
name: Update docker hub README

149
.golangci.yml
View file

@ -9,8 +9,24 @@ run:
- pkg/yamlpatch/merge_test.go
linters-settings:
cyclop:
# lower this after refactoring
max-complexity: 70
gci:
sections:
- standard
- default
- prefix(github.com/crowdsecurity)
- prefix(github.com/crowdsecurity/crowdsec)
gocognit:
# lower this after refactoring
min-complexity: 145
gocyclo:
min-complexity: 30
# lower this after refactoring
min-complexity: 70
funlen:
# Checks the number of lines in a function.
@ -28,11 +44,21 @@ linters-settings:
lll:
line-length: 140
maintidx:
# raise this after refactoring
under: 9
misspell:
locale: US
nestif:
# lower this after refactoring
min-complexity: 28
nlreturn:
block-size: 4
nolintlint:
allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
allow-unused: false # report any unused nolint directives
require-explanation: false # don't require an explanation for nolint directives
require-specific: false # don't require nolint directives to be specific about which linter is being skipped
@ -40,6 +66,13 @@ linters-settings:
interfacebloat:
max: 12
depguard:
rules:
main:
deny:
- pkg: "github.com/pkg/errors"
desc: "errors.Wrap() is deprecated in favor of fmt.Errorf()"
linters:
enable-all: true
disable:
@ -64,15 +97,21 @@ linters:
# - asasalint # check for pass []any as any in variadic func(...any)
# - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers
# - bidichk # Checks for dangerous unicode character sequences
# - bodyclose # checks whether HTTP response body is closed successfully
# - cyclop # checks function and package cyclomatic complexity
# - decorder # check declaration order and count of types, constants, variables and functions
# - depguard # Go linter that checks if package imports are in a list of acceptable packages
# - dupword # checks for duplicate words in the source code
# - durationcheck # check for two durations multiplied together
# - errcheck # Errcheck is a program for checking for unchecked errors in go programs. These unchecked errors can be critical bugs in some cases
# - errorlint # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13.
# - exportloopref # checks for pointers to enclosing loop variables
# - funlen # Tool for detection of long functions
# - ginkgolinter # enforces standards of using ginkgo and gomega
# - gochecknoinits # Checks that no init functions are present in Go code
# - gocognit # Computes and checks the cognitive complexity of functions
# - gocritic # Provides diagnostics that check for bugs, performance and style issues.
# - gocyclo # Computes and checks the cyclomatic complexity of functions
# - goheader # Checks is file header matches to pattern
# - gomoddirectives # Manage the use of 'replace', 'retract', and 'excludes' directives in go.mod.
# - gomodguard # Allow and block list linter for direct Go module dependencies. This is different from depguard where there are different block types for example version constraints and module recommendations.
@ -84,10 +123,15 @@ linters:
# - ineffassign # Detects when assignments to existing variables are not used
# - interfacebloat # A linter that checks the number of methods inside an interface.
# - logrlint # Check logr arguments.
# - maintidx # maintidx measures the maintainability index of each function.
# - makezero # Finds slice declarations with non-zero initial length
# - misspell # Finds commonly misspelled English words in comments
# - nakedret # Finds naked returns in functions greater than a specified function length
# - nestif # Reports deeply nested if statements
# - nilerr # Finds the code that returns nil even if it checks that the error is not nil.
# - nolintlint # Reports ill-formed or insufficient nolint directives
# - nonamedreturns # Reports all named returns
# - nosprintfhostport # Checks for misuse of Sprintf to construct a host with port in a URL.
# - predeclared # find code that shadows one of Go's predeclared identifiers
# - reassign # Checks that package variables are not reassigned
# - rowserrcheck # checks whether Err of rows is checked successfully
@ -100,38 +144,34 @@ linters:
# - unconvert # Remove unnecessary type conversions
# - unused # (megacheck): Checks Go code for unused constants, variables, functions and types
# - usestdlibvars # A linter that detect the possibility to use variables/constants from the Go standard library.
# - wastedassign # wastedassign finds wasted assignment statements.
#
# Recommended? (easy)
#
- depguard # Go linter that checks if package imports are in a list of acceptable packages
- dogsled # Checks assignments with too many blank identifiers (e.g. x, _, _, _, := f())
- errchkjson # Checks types passed to the json encoding functions. Reports unsupported types and optionally reports occations, where the check for the returned error can be omitted.
- errorlint # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13.
- exhaustive # check exhaustiveness of enum switch statements
- gci # Gci control golang package import order and make it always deterministic.
- godot # Check if comments end in a period
- gofmt # Gofmt checks whether code was gofmt-ed. By default this tool runs with -s option to check for code simplification
- goimports # In addition to fixing imports, goimports also formats your code in the same style as gofmt.
- gosec # (gas): Inspects source code for security problems
- inamedparam # reports interfaces with unnamed method parameters
- lll # Reports long lines
- musttag # enforce field tags in (un)marshaled structs
- nakedret # Finds naked returns in functions greater than a specified function length
- nonamedreturns # Reports all named returns
- nosprintfhostport # Checks for misuse of Sprintf to construct a host with port in a URL.
- promlinter # Check Prometheus metrics naming via promlint
- protogetter # Reports direct reads from proto message fields when getters should be used
- revive # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint.
- tagalign # check that struct tags are well aligned [fast: true, auto-fix: true]
- tagalign # check that struct tags are well aligned
- thelper # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers
- wastedassign # wastedassign finds wasted assignment statements.
- wrapcheck # Checks that errors returned from external packages are wrapped
#
# Recommended? (requires some work)
#
- bodyclose # checks whether HTTP response body is closed successfully
- containedctx # containedctx is a linter that detects struct contained context.Context field
- contextcheck # check the function whether use a non-inherited context
- errname # Checks that sentinel errors are prefixed with the `Err` and error types are suffixed with the `Error`.
@ -153,15 +193,10 @@ linters:
#
# Well intended, but not ready for this
#
- cyclop # checks function and package cyclomatic complexity
- dupl # Tool for code clone detection
- forcetypeassert # finds forced type assertions
- gocognit # Computes and checks the cognitive complexity of functions
- gocyclo # Computes and checks the cyclomatic complexity of functions
- godox # Tool for detection of FIXME, TODO and other comment keywords
- goerr113 # Golang linter to check the errors handling expressions
- maintidx # maintidx measures the maintainability index of each function.
- nestif # Reports deeply nested if statements
- paralleltest # paralleltest detects missing usage of t.Parallel() method in your Go test
- testpackage # linter that makes you use a separate _test package
@ -189,8 +224,11 @@ issues:
# break ‘em.” ― Terry Pratchett
max-issues-per-linter: 0
max-same-issues: 10
max-same-issues: 0
exclude-rules:
# Won't fix:
- path: go.mod
text: "replacement are not allowed: golang.org/x/time/rate"
@ -199,30 +237,10 @@ issues:
- govet
text: "shadow: declaration of \"err\" shadows declaration"
#
# typecheck
#
- linters:
- typecheck
text: "undefined: min"
- linters:
- typecheck
text: "undefined: max"
#
# errcheck
#
- linters:
- errcheck
text: "Error return value of `.*` is not checked"
#
# gocritic
#
- linters:
- gocritic
text: "ifElseChain: rewrite if-else to switch statement"
@ -239,6 +257,61 @@ issues:
- gocritic
text: "commentFormatting: put a space between `//` and comment text"
# Will fix, trivial - just beware of merge conflicts
- linters:
- staticcheck
text: "x509.ParseCRL has been deprecated since Go 1.19: Use ParseRevocationList instead"
- perfsprint
text: "fmt.Sprintf can be replaced .*"
#
# Will fix, easy but some neurons required
#
- linters:
- errorlint
text: "non-wrapping format verb for fmt.Errorf. Use `%w` to format errors"
- linters:
- errorlint
text: "type assertion on error will fail on wrapped errors. Use errors.As to check for specific errors"
- linters:
- errorlint
text: "type switch on error will fail on wrapped errors. Use errors.As to check for specific errors"
- linters:
- errorlint
text: "type assertion on error will fail on wrapped errors. Use errors.Is to check for specific errors"
- linters:
- errorlint
text: "comparing with .* will fail on wrapped errors. Use errors.Is to check for a specific error"
- linters:
- errorlint
text: "switch on an error will fail on wrapped errors. Use errors.Is to check for specific errors"
- linters:
- nosprintfhostport
text: "host:port in url should be constructed with net.JoinHostPort and not directly with fmt.Sprintf"
# https://github.com/timakin/bodyclose
- linters:
- bodyclose
text: "response body must be closed"
# named/naked returns are evil, with a single exception
# https://go.dev/wiki/CodeReviewComments#named-result-parameters
- linters:
- nonamedreturns
text: "named return .* with type .* found"
#
# Will fix, might be trickier
#
# https://github.com/pkg/errors/issues/245
- linters:
- depguard
text: "import 'github.com/pkg/errors' is not allowed .*"

28
Dockerfile
View file

@ -1,12 +1,13 @@
# vim: set ft=dockerfile:
ARG GOVERSION=1.21.1
FROM golang:1.21.6-alpine3.18 AS build
FROM golang:${GOVERSION}-alpine AS build
ARG BUILD_VERSION
WORKDIR /go/src/crowdsec
# We like to choose the release of re2 to use, and Alpine does not ship a static version anyway.
ENV RE2_VERSION=2023-03-01
ENV BUILD_VERSION=${BUILD_VERSION}
# wizard.sh requires GNU coreutils
RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils pkgconfig && \
@ -15,7 +16,7 @@ RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold core
cd re2-${RE2_VERSION} && \
make install && \
echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
go install github.com/mikefarah/yq/v4@v4.34.1
go install github.com/mikefarah/yq/v4@v4.40.4
COPY . .
@ -32,31 +33,32 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
FROM alpine:latest as slim
RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata bash && \
RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata bash rsync && \
mkdir -p /staging/etc/crowdsec && \
mkdir -p /staging/etc/crowdsec/acquis.d && \
mkdir -p /staging/var/lib/crowdsec && \
mkdir -p /var/lib/crowdsec/data
COPY --from=build /go/bin/yq /usr/local/bin/yq
COPY --from=build /go/bin/yq /usr/local/bin/crowdsec /usr/local/bin/cscli /usr/local/bin/
COPY --from=build /etc/crowdsec /staging/etc/crowdsec
COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml
ENTRYPOINT /bin/bash docker_start.sh
ENTRYPOINT /bin/bash /docker_start.sh
FROM slim as plugins
# Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
# The files are here for reference, as users will need to mount a new version to be actually able to use notifications
COPY --from=build /go/src/crowdsec/cmd/notification-email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml /staging/etc/crowdsec/notifications/sentinel.yaml
COPY --from=build \
/go/src/crowdsec/cmd/notification-email/email.yaml \
/go/src/crowdsec/cmd/notification-http/http.yaml \
/go/src/crowdsec/cmd/notification-slack/slack.yaml \
/go/src/crowdsec/cmd/notification-splunk/splunk.yaml \
/go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml \
/staging/etc/crowdsec/notifications/
COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
FROM slim as geoip

27
Dockerfile.debian
View file

@ -1,7 +1,7 @@
# vim: set ft=dockerfile:
ARG GOVERSION=1.21.1
FROM golang:1.21.6-bookworm AS build
FROM golang:${GOVERSION}-bookworm AS build
ARG BUILD_VERSION
WORKDIR /go/src/crowdsec
@ -10,6 +10,7 @@ ENV DEBCONF_NOWARNINGS="yes"
# We like to choose the release of re2 to use, the debian version is usually older.
ENV RE2_VERSION=2023-03-01
ENV BUILD_VERSION=${BUILD_VERSION}
# wizard.sh requires GNU coreutils
RUN apt-get update && \
@ -20,7 +21,7 @@ RUN apt-get update && \
make && \
make install && \
echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
go install github.com/mikefarah/yq/v4@v4.34.1
go install github.com/mikefarah/yq/v4@v4.40.4
COPY . .
@ -47,16 +48,15 @@ RUN apt-get update && \
iproute2 \
ca-certificates \
bash \
tzdata && \
tzdata \
rsync && \
mkdir -p /staging/etc/crowdsec && \
mkdir -p /staging/etc/crowdsec/acquis.d && \
mkdir -p /staging/var/lib/crowdsec && \
mkdir -p /var/lib/crowdsec/data
COPY --from=build /go/bin/yq /usr/local/bin/yq
COPY --from=build /go/bin/yq /usr/local/bin/crowdsec /usr/local/bin/cscli /usr/local/bin/
COPY --from=build /etc/crowdsec /staging/etc/crowdsec
COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml && \
@ -68,11 +68,14 @@ FROM slim as plugins
# Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
# The files are here for reference, as users will need to mount a new version to be actually able to use notifications
COPY --from=build /go/src/crowdsec/cmd/notification-email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml /staging/etc/crowdsec/notifications/sentinel.yaml
COPY --from=build \
/go/src/crowdsec/cmd/notification-email/email.yaml \
/go/src/crowdsec/cmd/notification-http/http.yaml \
/go/src/crowdsec/cmd/notification-slack/slack.yaml \
/go/src/crowdsec/cmd/notification-splunk/splunk.yaml \
/go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml \
/staging/etc/crowdsec/notifications/
COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
FROM slim as geoip

55
Makefile
View file

@ -128,11 +128,10 @@ endif
#--------------------------------------
.PHONY: build
build: pre-build goversion crowdsec cscli plugins
build: pre-build goversion crowdsec cscli plugins ## Build crowdsec, cscli and plugins
# Sanity checks and build information
.PHONY: pre-build
pre-build:
pre-build: ## Sanity checks and build information
$(info Building $(BUILD_VERSION) ($(BUILD_TAG)) $(BUILD_TYPE) for $(GOOS)/$(GOARCH))
ifneq (,$(RE2_FAIL))
@ -153,14 +152,14 @@ ifeq ($(call bool,$(TEST_COVERAGE)),1)
$(info Test coverage collection enabled)
endif
# intentional, empty line
$(info )
.PHONY: all
all: clean test build
all: clean test build ## Clean, test and build (requires localstack)
.PHONY: plugins
plugins:
plugins: ## Build notification plugins
@$(foreach plugin,$(PLUGINS), \
$(MAKE) -C $(PLUGINS_DIR_PREFIX)$(plugin) build $(MAKE_FLAGS); \
)
@ -184,7 +183,7 @@ clean-rpm:
@$(RM) -r rpm/SRPMS
.PHONY: clean
clean: clean-debian clean-rpm testclean
clean: clean-debian clean-rpm testclean ## Remove build artifacts
@$(MAKE) -C $(CROWDSEC_FOLDER) clean $(MAKE_FLAGS)
@$(MAKE) -C $(CSCLI_FOLDER) clean $(MAKE_FLAGS)
@$(RM) $(CROWDSEC_BIN) $(WIN_IGNORE_ERR)
@ -196,21 +195,16 @@ clean: clean-debian clean-rpm testclean
)
.PHONY: cscli
cscli: goversion
cscli: goversion ## Build cscli
@$(MAKE) -C $(CSCLI_FOLDER) build $(MAKE_FLAGS)
.PHONY: crowdsec
crowdsec: goversion
crowdsec: goversion ## Build crowdsec
@$(MAKE) -C $(CROWDSEC_FOLDER) build $(MAKE_FLAGS)
.PHONY: notification-email
notification-email: goversion
@$(MAKE) -C cmd/notification-email build $(MAKE_FLAGS)
.PHONY: testclean
testclean: bats-clean
testclean: bats-clean ## Remove test artifacts
@$(RM) pkg/apiserver/ent $(WIN_IGNORE_ERR)
@$(RM) pkg/cwhub/hubdir $(WIN_IGNORE_ERR)
@$(RM) pkg/cwhub/install $(WIN_IGNORE_ERR)
@ -218,42 +212,39 @@ testclean: bats-clean
# for the tests with localstack
export AWS_ENDPOINT_FORCE=http://localhost:4566
export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
export AWS_ACCESS_KEY_ID=test
export AWS_SECRET_ACCESS_KEY=test
testenv:
@echo 'NOTE: You need Docker, docker-compose and run "make localstack" in a separate shell ("make localstack-stop" to terminate it)'
# run the tests with localstack
.PHONY: test
test: testenv goversion
test: testenv goversion ## Run unit tests with localstack
$(GOTEST) $(LD_OPTS) ./...
# run the tests with localstack and coverage
.PHONY: go-acc
go-acc: testenv goversion
go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS) | \
sed 's/ *coverage:.*of statements in.*//'
go-acc: testenv goversion ## Run unit tests with localstack + coverage
go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS)
# mock AWS services
.PHONY: localstack
localstack:
localstack: ## Run localstack containers (required for unit testing)
docker-compose -f test/localstack/docker-compose.yml up
.PHONY: localstack-stop
localstack-stop:
localstack-stop: ## Stop localstack containers
docker-compose -f test/localstack/docker-compose.yml down
# build vendor.tgz to be distributed with the release
.PHONY: vendor
vendor: vendor-remove
vendor: vendor-remove ## CI only - vendor dependencies and archive them for packaging
$(GO) mod vendor
tar czf vendor.tgz vendor
tar --create --auto-compress --file=$(RELDIR)-vendor.tar.xz vendor
# remove vendor directories and vendor.tgz
.PHONY: vendor-remove
vendor-remove:
vendor-remove: ## Remove vendor dependencies and archives
$(RM) vendor vendor.tgz *-vendor.tar.xz
.PHONY: package
@ -285,18 +276,15 @@ else
@if (Test-Path -Path $(RELDIR)) { echo "$(RELDIR) already exists, abort" ; exit 1 ; }
endif
# build a release tarball
.PHONY: release
release: check_release build package
release: check_release build package ## Build a release tarball
# build the windows installer
.PHONY: windows_installer
windows_installer: build
windows_installer: build ## Windows - build the installer
@.\make_installer.ps1 -version $(BUILD_VERSION)
# build the chocolatey package
.PHONY: chocolatey
chocolatey: windows_installer
chocolatey: windows_installer ## Windows - build the chocolatey package
@.\make_chocolatey.ps1 -version $(BUILD_VERSION)
# Include test/bats.mk only if it exists
@ -309,3 +297,4 @@ include test/bats.mk
endif
include mk/goversion.mk
include mk/help.mk

4
azure-pipelines.yml
View file

@ -25,9 +25,9 @@ stages:
custom: 'tool'
arguments: 'install --global SignClient --version 1.3.155'
- task: GoTool@0
displayName: "Install Go 1.20"
displayName: "Install Go"
inputs:
version: '1.21.1'
version: '1.21.6'
- pwsh: |
choco install -y make

256
cmd/crowdsec-cli/alerts.go
View file

@ -11,7 +11,6 @@ import (
"strconv"
"strings"
"text/template"
"time"
"github.com/fatih/color"
"github.com/go-openapi/strfmt"
@ -21,12 +20,11 @@ import (
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
func DecisionsFromAlert(alert *models.Alert) string {
@ -49,53 +47,9 @@ func DecisionsFromAlert(alert *models.Alert) string {
return ret
}
func DateFromAlert(alert *models.Alert) string {
ts, err := time.Parse(time.RFC3339, alert.CreatedAt)
if err != nil {
log.Infof("while parsing %s with %s : %s", alert.CreatedAt, time.RFC3339, err)
return alert.CreatedAt
}
return ts.Format(time.RFC822)
}
func SourceFromAlert(alert *models.Alert) string {
//more than one item, just number and scope
if len(alert.Decisions) > 1 {
return fmt.Sprintf("%d %ss (%s)", len(alert.Decisions), *alert.Decisions[0].Scope, *alert.Decisions[0].Origin)
}
//fallback on single decision information
if len(alert.Decisions) == 1 {
return fmt.Sprintf("%s:%s", *alert.Decisions[0].Scope, *alert.Decisions[0].Value)
}
//try to compose a human friendly version
if *alert.Source.Value != "" && *alert.Source.Scope != "" {
scope := ""
scope = fmt.Sprintf("%s:%s", *alert.Source.Scope, *alert.Source.Value)
extra := ""
if alert.Source.Cn != "" {
extra = alert.Source.Cn
}
if alert.Source.AsNumber != "" {
extra += fmt.Sprintf("/%s", alert.Source.AsNumber)
}
if alert.Source.AsName != "" {
extra += fmt.Sprintf("/%s", alert.Source.AsName)
}
if extra != "" {
scope += " (" + extra + ")"
}
return scope
}
return ""
}
func AlertsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
if csConfig.Cscli.Output == "raw" {
func alertsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
switch csConfig.Cscli.Output {
case "raw":
csvwriter := csv.NewWriter(os.Stdout)
header := []string{"id", "scope", "value", "reason", "country", "as", "decisions", "created_at"}
if printMachine {
@ -125,7 +79,7 @@ func AlertsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
}
}
csvwriter.Flush()
} else if csConfig.Cscli.Output == "json" {
case "json":
if *alerts == nil {
// avoid returning "null" in json
// could be cleaner if we used slice of alerts directly
@ -133,8 +87,8 @@ func AlertsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
return nil
}
x, _ := json.MarshalIndent(alerts, "", " ")
fmt.Printf("%s", string(x))
} else if csConfig.Cscli.Output == "human" {
fmt.Print(string(x))
case "human":
if len(*alerts) == 0 {
fmt.Println("No active alerts")
return nil
@ -162,54 +116,61 @@ var alertTemplate = `
`
func DisplayOneAlert(alert *models.Alert, withDetail bool) error {
if csConfig.Cscli.Output == "human" {
tmpl, err := template.New("alert").Parse(alertTemplate)
if err != nil {
return err
}
err = tmpl.Execute(os.Stdout, alert)
if err != nil {
return err
}
func displayOneAlert(alert *models.Alert, withDetail bool) error {
tmpl, err := template.New("alert").Parse(alertTemplate)
if err != nil {
return err
}
err = tmpl.Execute(os.Stdout, alert)
if err != nil {
return err
}
alertDecisionsTable(color.Output, alert)
alertDecisionsTable(color.Output, alert)
if len(alert.Meta) > 0 {
fmt.Printf("\n - Context :\n")
sort.Slice(alert.Meta, func(i, j int) bool {
return alert.Meta[i].Key < alert.Meta[j].Key
})
table := newTable(color.Output)
table.SetRowLines(false)
table.SetHeaders("Key", "Value")
for _, meta := range alert.Meta {
var valSlice []string
if err := json.Unmarshal([]byte(meta.Value), &valSlice); err != nil {
return fmt.Errorf("unknown context value type '%s' : %s", meta.Value, err)
}
for _, value := range valSlice {
table.AddRow(
meta.Key,
value,
)
}
if len(alert.Meta) > 0 {
fmt.Printf("\n - Context :\n")
sort.Slice(alert.Meta, func(i, j int) bool {
return alert.Meta[i].Key < alert.Meta[j].Key
})
table := newTable(color.Output)
table.SetRowLines(false)
table.SetHeaders("Key", "Value")
for _, meta := range alert.Meta {
var valSlice []string
if err := json.Unmarshal([]byte(meta.Value), &valSlice); err != nil {
return fmt.Errorf("unknown context value type '%s' : %s", meta.Value, err)
}
table.Render()
}
if withDetail {
fmt.Printf("\n - Events :\n")
for _, event := range alert.Events {
alertEventTable(color.Output, event)
for _, value := range valSlice {
table.AddRow(
meta.Key,
value,
)
}
}
table.Render()
}
if withDetail {
fmt.Printf("\n - Events :\n")
for _, event := range alert.Events {
alertEventTable(color.Output, event)
}
}
return nil
}
func NewAlertsCmd() *cobra.Command {
var cmdAlerts = &cobra.Command{
type cliAlerts struct{
client *apiclient.ApiClient
}
func NewCLIAlerts() *cliAlerts {
return &cliAlerts{}
}
func (cli *cliAlerts) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "alerts [action]",
Short: "Manage alerts",
Args: cobra.MinimumNArgs(1),
@ -224,7 +185,7 @@ func NewAlertsCmd() *cobra.Command {
if err != nil {
return fmt.Errorf("parsing api url %s: %w", apiURL, err)
}
Client, err = apiclient.NewClient(&apiclient.Config{
cli.client, err = apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
Password: strfmt.Password(csConfig.API.Client.Credentials.Password),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
@ -239,15 +200,15 @@ func NewAlertsCmd() *cobra.Command {
},
}
cmdAlerts.AddCommand(NewAlertsListCmd())
cmdAlerts.AddCommand(NewAlertsInspectCmd())
cmdAlerts.AddCommand(NewAlertsFlushCmd())
cmdAlerts.AddCommand(NewAlertsDeleteCmd())
cmd.AddCommand(cli.NewListCmd())
cmd.AddCommand(cli.NewInspectCmd())
cmd.AddCommand(cli.NewFlushCmd())
cmd.AddCommand(cli.NewDeleteCmd())
return cmdAlerts
return cmd
}
func NewAlertsListCmd() *cobra.Command {
func (cli *cliAlerts) NewListCmd() *cobra.Command {
var alertListFilter = apiclient.AlertsListOpts{
ScopeEquals: new(string),
ValueEquals: new(string),
@ -260,10 +221,11 @@ func NewAlertsListCmd() *cobra.Command {
IncludeCAPI: new(bool),
OriginEquals: new(string),
}
var limit = new(int)
limit := new(int)
contained := new(bool)
var printMachine bool
var cmdAlertsList = &cobra.Command{
cmd := &cobra.Command{
Use: "list [filters]",
Short: "List alerts",
Example: `cscli alerts list
@ -340,12 +302,12 @@ cscli alerts list --type ban`,
alertListFilter.Contains = new(bool)
}
alerts, _, err := Client.Alerts.List(context.Background(), alertListFilter)
alerts, _, err := cli.client.Alerts.List(context.Background(), alertListFilter)
if err != nil {
return fmt.Errorf("unable to list alerts: %v", err)
}
err = AlertsToTable(alerts, printMachine)
err = alertsToTable(alerts, printMachine)
if err != nil {
return fmt.Errorf("unable to list alerts: %v", err)
}
@ -353,25 +315,25 @@ cscli alerts list --type ban`,
return nil
},
}
cmdAlertsList.Flags().SortFlags = false
cmdAlertsList.Flags().BoolVarP(alertListFilter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
cmdAlertsList.Flags().StringVar(alertListFilter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
cmdAlertsList.Flags().StringVar(alertListFilter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
cmdAlertsList.Flags().StringVarP(alertListFilter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
cmdAlertsList.Flags().StringVarP(alertListFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
cmdAlertsList.Flags().StringVarP(alertListFilter.RangeEquals, "range", "r", "", "restrict to alerts from this range (shorthand for --scope range --value <RANGE/X>)")
cmdAlertsList.Flags().StringVar(alertListFilter.TypeEquals, "type", "", "restrict to alerts with given decision type (ie. ban, captcha)")
cmdAlertsList.Flags().StringVar(alertListFilter.ScopeEquals, "scope", "", "restrict to alerts of this scope (ie. ip,range)")
cmdAlertsList.Flags().StringVarP(alertListFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmdAlertsList.Flags().StringVar(alertListFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmdAlertsList.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
cmdAlertsList.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that sent alerts")
cmdAlertsList.Flags().IntVarP(limit, "limit", "l", 50, "limit size of alerts list table (0 to view all alerts)")
cmd.Flags().SortFlags = false
cmd.Flags().BoolVarP(alertListFilter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
cmd.Flags().StringVar(alertListFilter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
cmd.Flags().StringVar(alertListFilter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
cmd.Flags().StringVarP(alertListFilter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
cmd.Flags().StringVarP(alertListFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
cmd.Flags().StringVarP(alertListFilter.RangeEquals, "range", "r", "", "restrict to alerts from this range (shorthand for --scope range --value <RANGE/X>)")
cmd.Flags().StringVar(alertListFilter.TypeEquals, "type", "", "restrict to alerts with given decision type (ie. ban, captcha)")
cmd.Flags().StringVar(alertListFilter.ScopeEquals, "scope", "", "restrict to alerts of this scope (ie. ip,range)")
cmd.Flags().StringVarP(alertListFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmd.Flags().StringVar(alertListFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmd.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
cmd.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that sent alerts")
cmd.Flags().IntVarP(limit, "limit", "l", 50, "limit size of alerts list table (0 to view all alerts)")
return cmdAlertsList
return cmd
}
func NewAlertsDeleteCmd() *cobra.Command {
func (cli *cliAlerts) NewDeleteCmd() *cobra.Command {
var ActiveDecision *bool
var AlertDeleteAll bool
var delAlertByID string
@ -383,7 +345,7 @@ func NewAlertsDeleteCmd() *cobra.Command {
IPEquals: new(string),
RangeEquals: new(string),
}
var cmdAlertsDelete = &cobra.Command{
cmd := &cobra.Command{
Use: "delete [filters] [--all]",
Short: `Delete alerts
/!\ This command can be use only on the same machine than the local API.`,
@ -446,12 +408,12 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`,
var alerts *models.DeleteAlertsResponse
if delAlertByID == "" {
alerts, _, err = Client.Alerts.Delete(context.Background(), alertDeleteFilter)
alerts, _, err = cli.client.Alerts.Delete(context.Background(), alertDeleteFilter)
if err != nil {
return fmt.Errorf("unable to delete alerts : %v", err)
}
} else {
alerts, _, err = Client.Alerts.DeleteOne(context.Background(), delAlertByID)
alerts, _, err = cli.client.Alerts.DeleteOne(context.Background(), delAlertByID)
if err != nil {
return fmt.Errorf("unable to delete alert: %v", err)
}
@ -461,21 +423,21 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`,
return nil
},
}
cmdAlertsDelete.Flags().SortFlags = false
cmdAlertsDelete.Flags().StringVar(alertDeleteFilter.ScopeEquals, "scope", "", "the scope (ie. ip,range)")
cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmdAlertsDelete.Flags().StringVar(&delAlertByID, "id", "", "alert ID")
cmdAlertsDelete.Flags().BoolVarP(&AlertDeleteAll, "all", "a", false, "delete all alerts")
cmdAlertsDelete.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
return cmdAlertsDelete
cmd.Flags().SortFlags = false
cmd.Flags().StringVar(alertDeleteFilter.ScopeEquals, "scope", "", "the scope (ie. ip,range)")
cmd.Flags().StringVarP(alertDeleteFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmd.Flags().StringVarP(alertDeleteFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
cmd.Flags().StringVarP(alertDeleteFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmd.Flags().StringVarP(alertDeleteFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmd.Flags().StringVar(&delAlertByID, "id", "", "alert ID")
cmd.Flags().BoolVarP(&AlertDeleteAll, "all", "a", false, "delete all alerts")
cmd.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
return cmd
}
func NewAlertsInspectCmd() *cobra.Command {
func (cli *cliAlerts) NewInspectCmd() *cobra.Command {
var details bool
var cmdAlertsInspect = &cobra.Command{
cmd := &cobra.Command{
Use: `inspect "alert_id"`,
Short: `Show info about an alert`,
Example: `cscli alerts inspect 123`,
@ -490,13 +452,13 @@ func NewAlertsInspectCmd() *cobra.Command {
if err != nil {
return fmt.Errorf("bad alert id %s", alertID)
}
alert, _, err := Client.Alerts.GetByID(context.Background(), id)
alert, _, err := cli.client.Alerts.GetByID(context.Background(), id)
if err != nil {
return fmt.Errorf("can't find alert with id %s: %s", alertID, err)
}
switch csConfig.Cscli.Output {
case "human":
if err := DisplayOneAlert(alert, details); err != nil {
if err := displayOneAlert(alert, details); err != nil {
continue
}
case "json":
@ -517,16 +479,16 @@ func NewAlertsInspectCmd() *cobra.Command {
return nil
},
}
cmdAlertsInspect.Flags().SortFlags = false
cmdAlertsInspect.Flags().BoolVarP(&details, "details", "d", false, "show alerts with events")
cmd.Flags().SortFlags = false
cmd.Flags().BoolVarP(&details, "details", "d", false, "show alerts with events")
return cmdAlertsInspect
return cmd
}
func NewAlertsFlushCmd() *cobra.Command {
func (cli *cliAlerts) NewFlushCmd() *cobra.Command {
var maxItems int
var maxAge string
var cmdAlertsFlush = &cobra.Command{
cmd := &cobra.Command{
Use: `flush`,
Short: `Flush alerts
/!\ This command can be used only on the same machine than the local API`,
@ -537,12 +499,12 @@ func NewAlertsFlushCmd() *cobra.Command {
if err := require.LAPI(csConfig); err != nil {
return err
}
dbClient, err = database.NewClient(csConfig.DbConfig)
db, err := database.NewClient(csConfig.DbConfig)
if err != nil {
return fmt.Errorf("unable to create new database client: %s", err)
}
log.Info("Flushing alerts. !! This may take a long time !!")
err = dbClient.FlushAlerts(maxAge, maxItems)
err = db.FlushAlerts(maxAge, maxItems)
if err != nil {
return fmt.Errorf("unable to flush alerts: %s", err)
}
@ -552,9 +514,9 @@ func NewAlertsFlushCmd() *cobra.Command {
},
}
cmdAlertsFlush.Flags().SortFlags = false
cmdAlertsFlush.Flags().IntVar(&maxItems, "max-items", 5000, "Maximum number of alert items to keep in the database")
cmdAlertsFlush.Flags().StringVar(&maxAge, "max-age", "7d", "Maximum age of alert items to keep in the database")
cmd.Flags().SortFlags = false
cmd.Flags().IntVar(&maxItems, "max-items", 5000, "Maximum number of alert items to keep in the database")
cmd.Flags().StringVar(&maxAge, "max-age", "7d", "Maximum age of alert items to keep in the database")
return cmdAlertsFlush
return cmd
}

510
cmd/crowdsec-cli/bouncers.go
View file

@ -4,7 +4,7 @@ import (
"encoding/csv"
"encoding/json"
"fmt"
"io"
"os"
"slices"
"strings"
"time"
@ -14,253 +14,40 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
func getBouncers(out io.Writer, dbClient *database.Client) error {
bouncers, err := dbClient.ListBouncers()
if err != nil {
return fmt.Errorf("unable to list bouncers: %s", err)
func askYesNo(message string, defaultAnswer bool) (bool, error) {
var answer bool
prompt := &survey.Confirm{
Message: message,
Default: defaultAnswer,
}
if csConfig.Cscli.Output == "human" {
getBouncersTable(out, bouncers)
} else if csConfig.Cscli.Output == "json" {
enc := json.NewEncoder(out)
enc.SetIndent("", " ")
if err := enc.Encode(bouncers); err != nil {
return fmt.Errorf("failed to unmarshal: %w", err)
}
return nil
} else if csConfig.Cscli.Output == "raw" {
csvwriter := csv.NewWriter(out)
err := csvwriter.Write([]string{"name", "ip", "revoked", "last_pull", "type", "version", "auth_type"})
if err != nil {
return fmt.Errorf("failed to write raw header: %w", err)
}
for _, b := range bouncers {
var revoked string
if !b.Revoked {
revoked = "validated"
} else {
revoked = "pending"
}
err := csvwriter.Write([]string{b.Name, b.IPAddress, revoked, b.LastPull.Format(time.RFC3339), b.Type, b.Version, b.AuthType})
if err != nil {
return fmt.Errorf("failed to write raw: %w", err)
}
}
csvwriter.Flush()
if err := survey.AskOne(prompt, &answer); err != nil {
return defaultAnswer, err
}
return nil
return answer, nil
}
func NewBouncersListCmd() *cobra.Command {
cmdBouncersList := &cobra.Command{
Use: "list",
Short: "list all bouncers within the database",
Example: `cscli bouncers list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, arg []string) error {
err := getBouncers(color.Output, dbClient)
if err != nil {
return fmt.Errorf("unable to list bouncers: %s", err)
}
return nil
},
}
return cmdBouncersList
type cliBouncers struct {
db *database.Client
cfg configGetter
}
func runBouncersAdd(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
keyLength, err := flags.GetInt("length")
if err != nil {
return err
func NewCLIBouncers(cfg configGetter) *cliBouncers {
return &cliBouncers{
cfg: cfg,
}
key, err := flags.GetString("key")
if err != nil {
return err
}
keyName := args[0]
var apiKey string
if keyName == "" {
return fmt.Errorf("please provide a name for the api key")
}
apiKey = key
if key == "" {
apiKey, err = middlewares.GenerateAPIKey(keyLength)
}
if err != nil {
return fmt.Errorf("unable to generate api key: %s", err)
}
_, err = dbClient.CreateBouncer(keyName, "", middlewares.HashSHA512(apiKey), types.ApiKeyAuthType)
if err != nil {
return fmt.Errorf("unable to create bouncer: %s", err)
}
if csConfig.Cscli.Output == "human" {
fmt.Printf("API key for '%s':\n\n", keyName)
fmt.Printf(" %s\n\n", apiKey)
fmt.Print("Please keep this key since you will not be able to retrieve it!\n")
} else if csConfig.Cscli.Output == "raw" {
fmt.Printf("%s", apiKey)
} else if csConfig.Cscli.Output == "json" {
j, err := json.Marshal(apiKey)
if err != nil {
return fmt.Errorf("unable to marshal api key")
}
fmt.Printf("%s", string(j))
}
return nil
}
func NewBouncersAddCmd() *cobra.Command {
cmdBouncersAdd := &cobra.Command{
Use: "add MyBouncerName [--length 16]",
Short: "add a single bouncer to the database",
Example: `cscli bouncers add MyBouncerName
cscli bouncers add MyBouncerName -l 24
cscli bouncers add MyBouncerName -k <random-key>`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: runBouncersAdd,
}
flags := cmdBouncersAdd.Flags()
flags.IntP("length", "l", 16, "length of the api key")
flags.StringP("key", "k", "", "api key for the bouncer")
return cmdBouncersAdd
}
func runBouncersDelete(cmd *cobra.Command, args []string) error {
for _, bouncerID := range args {
err := dbClient.DeleteBouncer(bouncerID)
if err != nil {
return fmt.Errorf("unable to delete bouncer '%s': %s", bouncerID, err)
}
log.Infof("bouncer '%s' deleted successfully", bouncerID)
}
return nil
}
func NewBouncersDeleteCmd() *cobra.Command {
cmdBouncersDelete := &cobra.Command{
Use: "delete MyBouncerName",
Short: "delete bouncer(s) from the database",
Args: cobra.MinimumNArgs(1),
Aliases: []string{"remove"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
var err error
dbClient, err = getDBClient()
if err != nil {
cobra.CompError("unable to create new database client: " + err.Error())
return nil, cobra.ShellCompDirectiveNoFileComp
}
bouncers, err := dbClient.ListBouncers()
if err != nil {
cobra.CompError("unable to list bouncers " + err.Error())
}
ret := make([]string, 0)
for _, bouncer := range bouncers {
if strings.Contains(bouncer.Name, toComplete) && !slices.Contains(args, bouncer.Name) {
ret = append(ret, bouncer.Name)
}
}
return ret, cobra.ShellCompDirectiveNoFileComp
},
RunE: runBouncersDelete,
}
return cmdBouncersDelete
}
func NewBouncersPruneCmd() *cobra.Command {
var parsedDuration time.Duration
cmdBouncersPrune := &cobra.Command{
Use: "prune",
Short: "prune multiple bouncers from the database",
Args: cobra.NoArgs,
DisableAutoGenTag: true,
Example: `cscli bouncers prune -d 60m
cscli bouncers prune -d 60m --force`,
PreRunE: func(cmd *cobra.Command, args []string) error {
dur, _ := cmd.Flags().GetString("duration")
var err error
parsedDuration, err = time.ParseDuration(fmt.Sprintf("-%s", dur))
if err != nil {
return fmt.Errorf("unable to parse duration '%s': %s", dur, err)
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
force, _ := cmd.Flags().GetBool("force")
if parsedDuration >= 0-2*time.Minute {
var answer bool
prompt := &survey.Confirm{
Message: "The duration you provided is less than or equal 2 minutes this may remove active bouncers continue ?",
Default: false,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask about prune check: %s", err)
}
if !answer {
fmt.Println("user aborted prune no changes were made")
return nil
}
}
bouncers, err := dbClient.QueryBouncersLastPulltimeLT(time.Now().UTC().Add(parsedDuration))
if err != nil {
return fmt.Errorf("unable to query bouncers: %s", err)
}
if len(bouncers) == 0 {
fmt.Println("no bouncers to prune")
return nil
}
getBouncersTable(color.Output, bouncers)
if !force {
var answer bool
prompt := &survey.Confirm{
Message: "You are about to PERMANENTLY remove the above bouncers from the database these will NOT be recoverable, continue ?",
Default: false,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask about prune check: %s", err)
}
if !answer {
fmt.Println("user aborted prune no changes were made")
return nil
}
}
nbDeleted, err := dbClient.BulkDeleteBouncers(bouncers)
if err != nil {
return fmt.Errorf("unable to prune bouncers: %s", err)
}
fmt.Printf("successfully delete %d bouncers\n", nbDeleted)
return nil
},
}
cmdBouncersPrune.Flags().StringP("duration", "d", "60m", "duration of time since last pull")
cmdBouncersPrune.Flags().Bool("force", false, "force prune without asking for confirmation")
return cmdBouncersPrune
}
func NewBouncersCmd() *cobra.Command {
var cmdBouncers = &cobra.Command{
func (cli *cliBouncers) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "bouncers [action]",
Short: "Manage bouncers [requires local API]",
Long: `To list/add/delete/prune bouncers.
@ -269,24 +56,261 @@ Note: This command requires database direct access, so is intended to be run on
Args: cobra.MinimumNArgs(1),
Aliases: []string{"bouncer"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
var err error
if err = require.LAPI(csConfig); err != nil {
if err = require.LAPI(cli.cfg()); err != nil {
return err
}
dbClient, err = database.NewClient(csConfig.DbConfig)
cli.db, err = database.NewClient(cli.cfg().DbConfig)
if err != nil {
return fmt.Errorf("unable to create new database client: %s", err)
return fmt.Errorf("can't connect to the database: %s", err)
}
return nil
},
}
cmdBouncers.AddCommand(NewBouncersListCmd())
cmdBouncers.AddCommand(NewBouncersAddCmd())
cmdBouncers.AddCommand(NewBouncersDeleteCmd())
cmdBouncers.AddCommand(NewBouncersPruneCmd())
cmd.AddCommand(cli.newListCmd())
cmd.AddCommand(cli.newAddCmd())
cmd.AddCommand(cli.newDeleteCmd())
cmd.AddCommand(cli.newPruneCmd())
return cmdBouncers
return cmd
}
func (cli *cliBouncers) list() error {
out := color.Output
bouncers, err := cli.db.ListBouncers()
if err != nil {
return fmt.Errorf("unable to list bouncers: %s", err)
}
switch cli.cfg().Cscli.Output {
case "human":
getBouncersTable(out, bouncers)
case "json":
enc := json.NewEncoder(out)
enc.SetIndent("", " ")
if err := enc.Encode(bouncers); err != nil {
return fmt.Errorf("failed to marshal: %w", err)
}
return nil
case "raw":
csvwriter := csv.NewWriter(out)
if err := csvwriter.Write([]string{"name", "ip", "revoked", "last_pull", "type", "version", "auth_type"}); err != nil {
return fmt.Errorf("failed to write raw header: %w", err)
}
for _, b := range bouncers {
valid := "validated"
if b.Revoked {
valid = "pending"
}
if err := csvwriter.Write([]string{b.Name, b.IPAddress, valid, b.LastPull.Format(time.RFC3339), b.Type, b.Version, b.AuthType}); err != nil {
return fmt.Errorf("failed to write raw: %w", err)
}
}
csvwriter.Flush()
}
return nil
}
func (cli *cliBouncers) newListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "list",
Short: "list all bouncers within the database",
Example: `cscli bouncers list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.list()
},
}
return cmd
}
func (cli *cliBouncers) add(bouncerName string, key string) error {
var err error
keyLength := 32
if key == "" {
key, err = middlewares.GenerateAPIKey(keyLength)
if err != nil {
return fmt.Errorf("unable to generate api key: %s", err)
}
}
_, err = cli.db.CreateBouncer(bouncerName, "", middlewares.HashSHA512(key), types.ApiKeyAuthType)
if err != nil {
return fmt.Errorf("unable to create bouncer: %s", err)
}
switch cli.cfg().Cscli.Output {
case "human":
fmt.Printf("API key for '%s':\n\n", bouncerName)
fmt.Printf(" %s\n\n", key)
fmt.Print("Please keep this key since you will not be able to retrieve it!\n")
case "raw":
fmt.Print(key)
case "json":
j, err := json.Marshal(key)
if err != nil {
return fmt.Errorf("unable to marshal api key")
}
fmt.Print(string(j))
}
return nil
}
func (cli *cliBouncers) newAddCmd() *cobra.Command {
var key string
cmd := &cobra.Command{
Use: "add MyBouncerName",
Short: "add a single bouncer to the database",
Example: `cscli bouncers add MyBouncerName
cscli bouncers add MyBouncerName --key <random-key>`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
return cli.add(args[0], key)
},
}
flags := cmd.Flags()
flags.StringP("length", "l", "", "length of the api key")
flags.MarkDeprecated("length", "use --key instead")
flags.StringVarP(&key, "key", "k", "", "api key for the bouncer")
return cmd
}
func (cli *cliBouncers) deleteValid(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
bouncers, err := cli.db.ListBouncers()
if err != nil {
cobra.CompError("unable to list bouncers " + err.Error())
}
ret := []string{}
for _, bouncer := range bouncers {
if strings.Contains(bouncer.Name, toComplete) && !slices.Contains(args, bouncer.Name) {
ret = append(ret, bouncer.Name)
}
}
return ret, cobra.ShellCompDirectiveNoFileComp
}
func (cli *cliBouncers) delete(bouncers []string) error {
for _, bouncerID := range bouncers {
err := cli.db.DeleteBouncer(bouncerID)
if err != nil {
return fmt.Errorf("unable to delete bouncer '%s': %s", bouncerID, err)
}
log.Infof("bouncer '%s' deleted successfully", bouncerID)
}
return nil
}
func (cli *cliBouncers) newDeleteCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "delete MyBouncerName",
Short: "delete bouncer(s) from the database",
Args: cobra.MinimumNArgs(1),
Aliases: []string{"remove"},
DisableAutoGenTag: true,
ValidArgsFunction: cli.deleteValid,
RunE: func(_ *cobra.Command, args []string) error {
return cli.delete(args)
},
}
return cmd
}
func (cli *cliBouncers) prune(duration time.Duration, force bool) error {
if duration < 2*time.Minute {
if yes, err := askYesNo(
"The duration you provided is less than 2 minutes. " +
"This may remove active bouncers. Continue?", false); err != nil {
return err
} else if !yes {
fmt.Println("User aborted prune. No changes were made.")
return nil
}
}
bouncers, err := cli.db.QueryBouncersLastPulltimeLT(time.Now().UTC().Add(duration))
if err != nil {
return fmt.Errorf("unable to query bouncers: %w", err)
}
if len(bouncers) == 0 {
fmt.Println("No bouncers to prune.")
return nil
}
getBouncersTable(color.Output, bouncers)
if !force {
if yes, err := askYesNo(
"You are about to PERMANENTLY remove the above bouncers from the database. " +
"These will NOT be recoverable. Continue?", false); err != nil {
return err
} else if !yes {
fmt.Println("User aborted prune. No changes were made.")
return nil
}
}
deleted, err := cli.db.BulkDeleteBouncers(bouncers)
if err != nil {
return fmt.Errorf("unable to prune bouncers: %s", err)
}
fmt.Fprintf(os.Stderr, "Successfully deleted %d bouncers\n", deleted)
return nil
}
func (cli *cliBouncers) newPruneCmd() *cobra.Command {
var (
duration time.Duration
force bool
)
const defaultDuration = 60 * time.Minute
cmd := &cobra.Command{
Use: "prune",
Short: "prune multiple bouncers from the database",
Args: cobra.NoArgs,
DisableAutoGenTag: true,
Example: `cscli bouncers prune -d 45m
cscli bouncers prune -d 45m --force`,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.prune(duration, force)
},
}
flags := cmd.Flags()
flags.DurationVarP(&duration, "duration", "d", defaultDuration, "duration of time since last pull")
flags.BoolVar(&force, "force", false, "force prune without asking for confirmation")
return cmd
}

119
cmd/crowdsec-cli/capi.go
View file

@ -13,26 +13,32 @@ import (
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
const CAPIBaseURL string = "https://api.crowdsec.net/"
const CAPIURLPrefix = "v3"
const (
CAPIBaseURL = "https://api.crowdsec.net/"
CAPIURLPrefix = "v3"
)
func NewCapiCmd() *cobra.Command {
var cmdCapi = &cobra.Command{
type cliCapi struct{}
func NewCLICapi() *cliCapi {
return &cliCapi{}
}
func (cli cliCapi) NewCommand() *cobra.Command {
var cmd = &cobra.Command{
Use: "capi [action]",
Short: "Manage interaction with Central API (CAPI)",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
if err := require.LAPI(csConfig); err != nil {
return err
}
@ -45,31 +51,33 @@ func NewCapiCmd() *cobra.Command {
},
}
cmdCapi.AddCommand(NewCapiRegisterCmd())
cmdCapi.AddCommand(NewCapiStatusCmd())
cmd.AddCommand(cli.NewRegisterCmd())
cmd.AddCommand(cli.NewStatusCmd())
return cmdCapi
return cmd
}
func NewCapiRegisterCmd() *cobra.Command {
var capiUserPrefix string
var outputFile string
func (cli cliCapi) NewRegisterCmd() *cobra.Command {
var (
capiUserPrefix string
outputFile string
)
var cmdCapiRegister = &cobra.Command{
var cmd = &cobra.Command{
Use: "register",
Short: "Register to Central API (CAPI)",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(_ *cobra.Command, _ []string) error {
var err error
capiUser, err := generateID(capiUserPrefix)
if err != nil {
log.Fatalf("unable to generate machine id: %s", err)
return fmt.Errorf("unable to generate machine id: %s", err)
}
password := strfmt.Password(generatePassword(passwordLength))
apiurl, err := url.Parse(types.CAPIBaseURL)
if err != nil {
log.Fatalf("unable to parse api url %s : %s", types.CAPIBaseURL, err)
return fmt.Errorf("unable to parse api url %s: %w", types.CAPIBaseURL, err)
}
_, err = apiclient.RegisterClient(&apiclient.Config{
MachineID: capiUser,
@ -80,7 +88,7 @@ func NewCapiRegisterCmd() *cobra.Command {
}, nil)
if err != nil {
log.Fatalf("api client register ('%s'): %s", types.CAPIBaseURL, err)
return fmt.Errorf("api client register ('%s'): %w", types.CAPIBaseURL, err)
}
log.Printf("Successfully registered to Central API (CAPI)")
@ -98,90 +106,91 @@ func NewCapiRegisterCmd() *cobra.Command {
Password: password.String(),
URL: types.CAPIBaseURL,
}
if fflag.PapiClient.IsEnabled() {
apiCfg.PapiURL = types.PAPIBaseURL
}
apiConfigDump, err := yaml.Marshal(apiCfg)
if err != nil {
log.Fatalf("unable to marshal api credentials: %s", err)
return fmt.Errorf("unable to marshal api credentials: %w", err)
}
if dumpFile != "" {
err = os.WriteFile(dumpFile, apiConfigDump, 0600)
err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
if err != nil {
log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
return fmt.Errorf("write api credentials in '%s' failed: %w", dumpFile, err)
}
log.Printf("Central API credentials dumped to '%s'", dumpFile)
log.Printf("Central API credentials written to '%s'", dumpFile)
} else {
fmt.Printf("%s\n", string(apiConfigDump))
fmt.Println(string(apiConfigDump))
}
log.Warning(ReloadMessage())
return nil
},
}
cmdCapiRegister.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
cmdCapiRegister.Flags().StringVar(&capiUserPrefix, "schmilblick", "", "set a schmilblick (use in tests only)")
if err := cmdCapiRegister.Flags().MarkHidden("schmilblick"); err != nil {
cmd.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
cmd.Flags().StringVar(&capiUserPrefix, "schmilblick", "", "set a schmilblick (use in tests only)")
if err := cmd.Flags().MarkHidden("schmilblick"); err != nil {
log.Fatalf("failed to hide flag: %s", err)
}
return cmdCapiRegister
return cmd
}
func NewCapiStatusCmd() *cobra.Command {
var cmdCapiStatus = &cobra.Command{
func (cli cliCapi) NewStatusCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "status",
Short: "Check status with the Central API (CAPI)",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if csConfig.API.Server.OnlineClient == nil {
log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
}
if csConfig.API.Server.OnlineClient.Credentials == nil {
log.Fatalf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
RunE: func(_ *cobra.Command, _ []string) error {
if err := require.CAPIRegistered(csConfig); err != nil {
return err
}
password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
apiurl, err := url.Parse(csConfig.API.Server.OnlineClient.Credentials.URL)
if err != nil {
log.Fatalf("parsing api url ('%s'): %s", csConfig.API.Server.OnlineClient.Credentials.URL, err)
return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err)
}
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to load hub index : %s", err)
}
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
hub, err := require.Hub(csConfig, nil, nil)
if err != nil {
log.Fatalf("failed to get scenarios : %s", err)
return err
}
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("failed to get scenarios: %w", err)
}
if len(scenarios) == 0 {
log.Fatalf("no scenarios installed, abort")
return fmt.Errorf("no scenarios installed, abort")
}
Client, err = apiclient.NewDefaultClient(apiurl, CAPIURLPrefix, fmt.Sprintf("crowdsec/%s", version.String()), nil)
if err != nil {
log.Fatalf("init default client: %s", err)
return fmt.Errorf("init default client: %w", err)
}
t := models.WatcherAuthRequest{
MachineID: &csConfig.API.Server.OnlineClient.Credentials.Login,
Password: &password,
Scenarios: scenarios,
}
log.Infof("Loaded credentials from %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
log.Infof("Trying to authenticate with username %s on %s", csConfig.API.Server.OnlineClient.Credentials.Login, apiurl)
_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
if err != nil {
log.Fatalf("Failed to authenticate to Central API (CAPI) : %s", err)
return fmt.Errorf("failed to authenticate to Central API (CAPI): %w", err)
}
log.Infof("You can successfully interact with Central API (CAPI)")
return nil
},
}
return cmdCapiStatus
return cmd
}

176
cmd/crowdsec-cli/collections.go
View file

@ -1,176 +0,0 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCollectionsCmd() *cobra.Command {
var cmdCollections = &cobra.Command{
Use: "collections [action]",
Short: "Manage collections from hub",
Long: `Install/Remove/Upgrade/Inspect collections from the CrowdSec Hub.`,
/*TBD fix help*/
Args: cobra.MinimumNArgs(1),
Aliases: []string{"collection"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
var ignoreError bool
var cmdCollectionsInstall = &cobra.Command{
Use: "install collection",
Short: "Install given collection(s)",
Long: `Fetch and install given collection(s) from hub`,
Example: `cscli collections install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.COLLECTIONS, args, toComplete)
},
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
for _, name := range args {
t := cwhub.GetItem(cwhub.COLLECTIONS, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.COLLECTIONS, name)
Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
}
}
return nil
},
}
cmdCollectionsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdCollectionsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdCollectionsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple collections")
cmdCollections.AddCommand(cmdCollectionsInstall)
var cmdCollectionsRemove = &cobra.Command{
Use: "remove collection",
Short: "Remove given collection(s)",
Long: `Remove given collection(s) from hub`,
Example: `cscli collections remove crowdsec/xxx crowdsec/xyz`,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, forceAction)
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one collection to remove or '--all'")
}
for _, name := range args {
if !forceAction {
item := cwhub.GetItem(cwhub.COLLECTIONS, name)
if item == nil {
return fmt.Errorf("unable to retrieve: %s", name)
}
if len(item.BelongsToCollections) > 0 {
log.Warningf("%s belongs to other collections :\n%s\n", name, item.BelongsToCollections)
log.Printf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection\n", name)
continue
}
}
cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, forceAction)
}
return nil
},
}
cmdCollectionsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdCollectionsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdCollectionsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the collections")
cmdCollections.AddCommand(cmdCollectionsRemove)
var cmdCollectionsUpgrade = &cobra.Command{
Use: "upgrade collection",
Short: "Upgrade given collection(s)",
Long: `Fetch and upgrade given collection(s) from hub`,
Example: `cscli collections upgrade crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
} else {
if len(args) == 0 {
return fmt.Errorf("specify at least one collection to upgrade or '--all'")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, forceAction)
}
}
return nil
},
}
cmdCollectionsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the collections")
cmdCollectionsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
cmdCollections.AddCommand(cmdCollectionsUpgrade)
var cmdCollectionsInspect = &cobra.Command{
Use: "inspect collection",
Short: "Inspect given collection",
Long: `Inspect given collection`,
Example: `cscli collections inspect crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
for _, name := range args {
InspectItem(name, cwhub.COLLECTIONS)
}
},
}
cmdCollectionsInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
cmdCollections.AddCommand(cmdCollectionsInspect)
var cmdCollectionsList = &cobra.Command{
Use: "list collection [-a]",
Short: "List all collections",
Long: `List all collections`,
Example: `cscli collections list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all)
},
}
cmdCollectionsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
cmdCollections.AddCommand(cmdCollectionsList)
return cmdCollections
}

4
cmd/crowdsec-cli/completion.go
View file

@ -7,8 +7,7 @@ import (
)
func NewCompletionCmd() *cobra.Command {
var completionCmd = &cobra.Command{
completionCmd := &cobra.Command{
Use: "completion [bash|zsh|powershell|fish]",
Short: "Generate completion script",
Long: `To load completions:
@ -82,5 +81,6 @@ func NewCompletionCmd() *cobra.Command {
}
},
}
return completionCmd
}

88
cmd/crowdsec-cli/config_backup.go
View file

@ -1,6 +1,7 @@
package main
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
@ -9,8 +10,87 @@ import (
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func backupHub(dirPath string) error {
hub, err := require.Hub(csConfig, nil, nil)
if err != nil {
return err
}
for _, itemType := range cwhub.ItemTypes {
clog := log.WithFields(log.Fields{
"type": itemType,
})
itemMap := hub.GetItemMap(itemType)
if itemMap == nil {
clog.Infof("No %s to backup.", itemType)
continue
}
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itemType)
if err = os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
}
upstreamParsers := []string{}
for k, v := range itemMap {
clog = clog.WithFields(log.Fields{
"file": v.Name,
})
if !v.State.Installed { //only backup installed ones
clog.Debugf("[%s] : not installed", k)
continue
}
//for the local/tainted ones, we back up the full file
if v.State.Tainted || v.State.IsLocal() || !v.State.UpToDate {
//we need to backup stages for parsers
if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS {
fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
if err = os.MkdirAll(fstagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
}
}
clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.State.IsLocal(), v.State.UpToDate)
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
if err = CopyFile(v.State.LocalPath, tfile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.State.LocalPath, tfile, err)
}
clog.Infof("local/tainted saved %s to %s", v.State.LocalPath, tfile)
continue
}
clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.State.UpToDate)
clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.State.UpToDate)
upstreamParsers = append(upstreamParsers, v.Name)
}
//write the upstream items
upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
if err != nil {
return fmt.Errorf("failed marshaling upstream parsers : %s", err)
}
err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0o644)
if err != nil {
return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
}
clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
}
return nil
}
/*
Backup crowdsec configurations to directory <dirPath>:
@ -33,7 +113,7 @@ func backupConfigToDirectory(dirPath string) error {
/*if parent directory doesn't exist, bail out. create final dir with Mkdir*/
parentDir := filepath.Dir(dirPath)
if _, err := os.Stat(parentDir); err != nil {
if _, err = os.Stat(parentDir); err != nil {
return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err)
}
@ -122,7 +202,7 @@ func backupConfigToDirectory(dirPath string) error {
log.Infof("Saved profiles to %s", backupProfiles)
}
if err = BackupHub(dirPath); err != nil {
if err = backupHub(dirPath); err != nil {
return fmt.Errorf("failed to backup hub config: %s", err)
}
@ -130,10 +210,6 @@ func backupConfigToDirectory(dirPath string) error {
}
func runConfigBackup(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
if err := backupConfigToDirectory(args[0]); err != nil {
return fmt.Errorf("failed to backup config: %w", err)
}

14
cmd/crowdsec-cli/config_feature_flags.go
View file

@ -2,10 +2,12 @@ package main
import (
"fmt"
"path/filepath"
"github.com/fatih/color"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
)
@ -42,6 +44,7 @@ func runConfigFeatureFlags(cmd *cobra.Command, args []string) error {
if feat.State == fflag.RetiredState {
fmt.Printf("\n %s %s", magenta("RETIRED"), feat.DeprecationMsg)
}
fmt.Println()
}
@ -56,10 +59,12 @@ func runConfigFeatureFlags(cmd *cobra.Command, args []string) error {
retired = append(retired, feat)
continue
}
if feat.IsEnabled() {
enabled = append(enabled, feat)
continue
}
disabled = append(disabled, feat)
}
@ -87,7 +92,14 @@ func runConfigFeatureFlags(cmd *cobra.Command, args []string) error {
fmt.Println("To enable a feature you can: ")
fmt.Println(" - set the environment variable CROWDSEC_FEATURE_<uppercase_feature_name> to true")
fmt.Printf(" - add the line '- <feature_name>' to the file %s/feature.yaml\n", csConfig.ConfigPaths.ConfigDir)
featurePath, err := filepath.Abs(csconfig.GetFeatureFilePath(ConfigFilePath))
if err != nil {
// we already read the file, shouldn't happen
return err
}
fmt.Printf(" - add the line '- <feature_name>' to the file %s\n", featurePath)
fmt.Println()
if len(enabled) == 0 && len(disabled) == 0 {

116
cmd/crowdsec-cli/config_restore.go
View file

@ -13,6 +13,7 @@ import (
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
type OldAPICfg struct {
@ -20,6 +21,104 @@ type OldAPICfg struct {
Password string `json:"password"`
}
func restoreHub(dirPath string) error {
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), nil)
if err != nil {
return err
}
for _, itype := range cwhub.ItemTypes {
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
if _, err = os.Stat(itemDirectory); err != nil {
log.Infof("no %s in backup", itype)
continue
}
/*restore the upstream items*/
upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
file, err := os.ReadFile(upstreamListFN)
if err != nil {
return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
}
var upstreamList []string
err = json.Unmarshal(file, &upstreamList)
if err != nil {
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
}
for _, toinstall := range upstreamList {
item := hub.GetItem(itype, toinstall)
if item == nil {
log.Errorf("Item %s/%s not found in hub", itype, toinstall)
continue
}
err := item.Install(false, false)
if err != nil {
log.Errorf("Error while installing %s : %s", toinstall, err)
}
}
/*restore the local and tainted items*/
files, err := os.ReadDir(itemDirectory)
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
}
for _, file := range files {
//this was the upstream data
if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
continue
}
if itype == cwhub.PARSERS || itype == cwhub.POSTOVERFLOWS {
//we expect a stage here
if !file.IsDir() {
continue
}
stage := file.Name()
stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
}
// find items
ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
}
//finally copy item
for _, tfile := range ifiles {
log.Infof("Going to restore local/tainted [%s]", tfile.Name())
sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
} else {
log.Infof("Going to restore local/tainted [%s]", file.Name())
sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
}
}
return nil
}
/*
Restore crowdsec configurations to directory <dirPath>:
@ -47,7 +146,12 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
// Now we have config.yaml, we should regenerate config struct to have rights paths etc
ConfigFilePath = fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)
initConfig()
log.Debug("Reloading configuration")
csConfig, _, err = loadConfigFor("config")
if err != nil {
return fmt.Errorf("failed to reload configuration: %s", err)
}
backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath)
if _, err = os.Stat(backupCAPICreds); err == nil {
@ -96,7 +200,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
if csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" {
apiConfigDumpFile = csConfig.API.Server.OnlineClient.CredentialsFilePath
}
err = os.WriteFile(apiConfigDumpFile, apiConfigDump, 0o644)
err = os.WriteFile(apiConfigDumpFile, apiConfigDump, 0o600)
if err != nil {
return fmt.Errorf("write api credentials in '%s' failed: %s", apiConfigDumpFile, err)
}
@ -128,7 +232,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
}
}
// if there is files in the acquis backup dir, restore them
// if there are files in the acquis backup dir, restore them
acquisBackupDir := filepath.Join(dirPath, "acquis", "*.yaml")
if acquisFiles, err := filepath.Glob(acquisBackupDir); err == nil {
for _, acquisFile := range acquisFiles {
@ -168,7 +272,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
}
}
if err = RestoreHub(dirPath); err != nil {
if err = restoreHub(dirPath); err != nil {
return fmt.Errorf("failed to restore hub config : %s", err)
}
@ -183,10 +287,6 @@ func runConfigRestore(cmd *cobra.Command, args []string) error {
return err
}
if err := require.Hub(csConfig); err != nil {
return err
}
if err := restoreConfigFromDirectory(args[0], oldBackup); err != nil {
return fmt.Errorf("failed to restore config from %s: %w", args[0], err)
}

14
cmd/crowdsec-cli/config_show.go
View file

@ -7,6 +7,7 @@ import (
"text/template"
"github.com/antonmedv/expr"
"github.com/sanity-io/litter"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
@ -23,6 +24,7 @@ func showConfigKey(key string) error {
opts := []expr.Option{}
opts = append(opts, exprhelpers.GetExprOptions(map[string]interface{}{})...)
opts = append(opts, expr.Env(Env{}))
program, err := expr.Compile(key, opts...)
if err != nil {
return err
@ -35,13 +37,13 @@ func showConfigKey(key string) error {
switch csConfig.Cscli.Output {
case "human", "raw":
// Don't use litter for strings, it adds quotes
// that we didn't have before
switch output.(type) {
case string:
fmt.Printf("%s\n", output)
case int:
fmt.Printf("%d\n", output)
fmt.Println(output)
default:
fmt.Printf("%v\n", output)
litter.Dump(output)
}
case "json":
data, err := json.MarshalIndent(output, "", " ")
@ -51,6 +53,7 @@ func showConfigKey(key string) error {
fmt.Printf("%s\n", string(data))
}
return nil
}
@ -82,7 +85,6 @@ Crowdsec{{if and .Crowdsec.Enable (not (ValueBool .Crowdsec.Enable))}} (disabled
cscli:
- Output : {{.Cscli.Output}}
- Hub Branch : {{.Cscli.HubBranch}}
- Hub Folder : {{.Cscli.HubDir}}
{{- end }}
{{- if .API }}
@ -211,6 +213,7 @@ func runConfigShow(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
err = tmp.Execute(os.Stdout, csConfig)
if err != nil {
return err
@ -230,6 +233,7 @@ func runConfigShow(cmd *cobra.Command, args []string) error {
fmt.Printf("%s\n", string(data))
}
return nil
}

92
cmd/crowdsec-cli/console.go
View file

@ -7,6 +7,7 @@ import (
"fmt"
"net/url"
"os"
"strings"
"github.com/fatih/color"
"github.com/go-openapi/strfmt"
@ -17,13 +18,11 @@ import (
"github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
func NewConsoleCmd() *cobra.Command {
@ -49,6 +48,7 @@ func NewConsoleCmd() *cobra.Command {
name := ""
overwrite := false
tags := []string{}
opts := []string{}
cmdEnroll := &cobra.Command{
Use: "enroll [enroll-key]",
@ -58,10 +58,12 @@ Enroll this instance to https://app.crowdsec.net
You can get your enrollment key by creating an account on https://app.crowdsec.net.
After running this command your will need to validate the enrollment in the webapp.`,
Example: `cscli console enroll YOUR-ENROLL-KEY
Example: fmt.Sprintf(`cscli console enroll YOUR-ENROLL-KEY
cscli console enroll --name [instance_name] YOUR-ENROLL-KEY
cscli console enroll --name [instance_name] --tags [tag_1] --tags [tag_2] YOUR-ENROLL-KEY
`,
cscli console enroll --enable context,manual YOUR-ENROLL-KEY
valid options are : %s,all (see 'cscli console status' for details)`, strings.Join(csconfig.CONSOLE_CONFIGS, ",")),
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
@ -71,11 +73,12 @@ After running this command your will need to validate the enrollment in the weba
return fmt.Errorf("could not parse CAPI URL: %s", err)
}
if err := require.Hub(csConfig); err != nil {
hub, err := require.Hub(csConfig, nil, nil)
if err != nil {
return err
}
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("failed to get installed scenarios: %s", err)
}
@ -84,6 +87,37 @@ After running this command your will need to validate the enrollment in the weba
scenarios = make([]string, 0)
}
enable_opts := []string{csconfig.SEND_MANUAL_SCENARIOS, csconfig.SEND_TAINTED_SCENARIOS}
if len(opts) != 0 {
for _, opt := range opts {
valid := false
if opt == "all" {
enable_opts = csconfig.CONSOLE_CONFIGS
break
}
for _, available_opt := range csconfig.CONSOLE_CONFIGS {
if opt == available_opt {
valid = true
enable := true
for _, enabled_opt := range enable_opts {
if opt == enabled_opt {
enable = false
continue
}
}
if enable {
enable_opts = append(enable_opts, opt)
}
break
}
}
if !valid {
return fmt.Errorf("option %s doesn't exist", opt)
}
}
}
c, _ := apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Server.OnlineClient.Credentials.Login,
Password: password,
@ -101,11 +135,13 @@ After running this command your will need to validate the enrollment in the weba
return nil
}
if err := SetConsoleOpts([]string{csconfig.SEND_MANUAL_SCENARIOS, csconfig.SEND_TAINTED_SCENARIOS}, true); err != nil {
if err := SetConsoleOpts(enable_opts, true); err != nil {
return err
}
log.Info("Enabled tainted&manual alerts sharing, see 'cscli console status'.")
for _, opt := range enable_opts {
log.Infof("Enabled %s : %s", opt, csconfig.CONSOLE_CONFIGS_HELP[opt])
}
log.Info("Watcher successfully enrolled. Visit https://app.crowdsec.net to accept it.")
log.Info("Please restart crowdsec after accepting the enrollment.")
return nil
@ -114,6 +150,7 @@ After running this command your will need to validate the enrollment in the weba
cmdEnroll.Flags().StringVarP(&name, "name", "n", "", "Name to display in the console")
cmdEnroll.Flags().BoolVarP(&overwrite, "overwrite", "", false, "Force enroll the instance")
cmdEnroll.Flags().StringSliceVarP(&tags, "tags", "t", tags, "Tags to display in the console")
cmdEnroll.Flags().StringSliceVarP(&opts, "enable", "e", opts, "Enable console options")
cmdConsole.AddCommand(cmdEnroll)
var enableAll, disableAll bool
@ -188,11 +225,11 @@ Disable given information push to the central API.`,
case "json":
c := csConfig.API.Server.ConsoleConfig
out := map[string](*bool){
csconfig.SEND_MANUAL_SCENARIOS: c.ShareManualDecisions,
csconfig.SEND_CUSTOM_SCENARIOS: c.ShareCustomScenarios,
csconfig.SEND_MANUAL_SCENARIOS: c.ShareManualDecisions,
csconfig.SEND_CUSTOM_SCENARIOS: c.ShareCustomScenarios,
csconfig.SEND_TAINTED_SCENARIOS: c.ShareTaintedScenarios,
csconfig.SEND_CONTEXT: c.ShareContext,
csconfig.CONSOLE_MANAGEMENT: c.ConsoleManagement,
csconfig.SEND_CONTEXT: c.ShareContext,
csconfig.CONSOLE_MANAGEMENT: c.ConsoleManagement,
}
data, err := json.MarshalIndent(out, "", " ")
if err != nil {
@ -229,13 +266,28 @@ Disable given information push to the central API.`,
return cmdConsole
}
func dumpConsoleConfig(c *csconfig.LocalApiServerCfg) error {
out, err := yaml.Marshal(c.ConsoleConfig)
if err != nil {
return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", c.ConsoleConfigPath, err)
}
if c.ConsoleConfigPath == "" {
c.ConsoleConfigPath = csconfig.DefaultConsoleConfigFilePath
log.Debugf("Empty console_path, defaulting to %s", c.ConsoleConfigPath)
}
if err := os.WriteFile(c.ConsoleConfigPath, out, 0o600); err != nil {
return fmt.Errorf("while dumping console config to %s: %w", c.ConsoleConfigPath, err)
}
return nil
}
func SetConsoleOpts(args []string, wanted bool) error {
for _, arg := range args {
switch arg {
case csconfig.CONSOLE_MANAGEMENT:
if !fflag.PapiClient.IsEnabled() {
continue
}
/*for each flag check if it's already set before setting it*/
if csConfig.API.Server.ConsoleConfig.ConsoleManagement != nil {
if *csConfig.API.Server.ConsoleConfig.ConsoleManagement == wanted {
@ -248,6 +300,7 @@ func SetConsoleOpts(args []string, wanted bool) error {
log.Infof("%s set to %t", csconfig.CONSOLE_MANAGEMENT, wanted)
csConfig.API.Server.ConsoleConfig.ConsoleManagement = ptr.Of(wanted)
}
if csConfig.API.Server.OnlineClient.Credentials != nil {
changed := false
if wanted && csConfig.API.Server.OnlineClient.Credentials.PapiURL == "" {
@ -257,13 +310,16 @@ func SetConsoleOpts(args []string, wanted bool) error {
changed = true
csConfig.API.Server.OnlineClient.Credentials.PapiURL = ""
}
if changed {
fileContent, err := yaml.Marshal(csConfig.API.Server.OnlineClient.Credentials)
if err != nil {
return fmt.Errorf("cannot marshal credentials: %s", err)
}
log.Infof("Updating credentials file: %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
err = os.WriteFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, fileContent, 0600)
err = os.WriteFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, fileContent, 0o600)
if err != nil {
return fmt.Errorf("cannot write credentials file: %s", err)
}
@ -326,7 +382,7 @@ func SetConsoleOpts(args []string, wanted bool) error {
}
}
if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
if err := dumpConsoleConfig(csConfig.API.Server); err != nil {
return fmt.Errorf("failed writing console config: %s", err)
}

17
cmd/crowdsec-cli/console_table.go
View file

@ -17,43 +17,30 @@ func cmdConsoleStatusTable(out io.Writer, csConfig csconfig.Config) {
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
for _, option := range csconfig.CONSOLE_CONFIGS {
activated := string(emoji.CrossMark)
switch option {
case csconfig.SEND_CUSTOM_SCENARIOS:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareCustomScenarios {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send alerts from custom scenarios to the console")
case csconfig.SEND_MANUAL_SCENARIOS:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareManualDecisions {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send manual decisions to the console")
case csconfig.SEND_TAINTED_SCENARIOS:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send alerts from tainted scenarios to the console")
case csconfig.SEND_CONTEXT:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareContext {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send context with alerts to the console")
case csconfig.CONSOLE_MANAGEMENT:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ConsoleManagement {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Receive decisions from console")
}
t.AddRow(option, activated, csconfig.CONSOLE_CONFIGS_HELP[option])
}
t.Render()

20
cmd/crowdsec-cli/copyfile.go
View file

@ -18,56 +18,66 @@ func copyFileContents(src, dst string) (err error) {
return
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
return
}
defer func() {
cerr := out.Close()
if err == nil {
err = cerr
}
}()
if _, err = io.Copy(out, in); err != nil {
return
}
err = out.Sync()
return
}
/*copy the file, ioutile doesn't offer the feature*/
func CopyFile(sourceSymLink, destinationFile string) (err error) {
func CopyFile(sourceSymLink, destinationFile string) error {
sourceFile, err := filepath.EvalSymlinks(sourceSymLink)
if err != nil {
log.Infof("Not a symlink : %s", err)
sourceFile = sourceSymLink
}
sourceFileStat, err := os.Stat(sourceFile)
if err != nil {
return
return err
}
if !sourceFileStat.Mode().IsRegular() {
// cannot copy non-regular files (e.g., directories,
// symlinks, devices, etc.)
return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String())
}
destinationFileStat, err := os.Stat(destinationFile)
if err != nil {
if !os.IsNotExist(err) {
return
return err
}
} else {
if !(destinationFileStat.Mode().IsRegular()) {
return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String())
}
if os.SameFile(sourceFileStat, destinationFileStat) {
return
return err
}
}
if err = os.Link(sourceFile, destinationFile); err != nil {
err = copyFileContents(sourceFile, destinationFile)
}
return
return err
}

233
cmd/crowdsec-cli/dashboard.go
View file

@ -1,3 +1,5 @@
//go:build linux
package main
import (
@ -9,6 +11,7 @@ import (
"path/filepath"
"strconv"
"strings"
"syscall"
"unicode"
"github.com/AlecAivazis/survey/v2"
@ -16,15 +19,14 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/metabase"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/metabase"
)
var (
metabaseUser = "crowdsec@crowdsec.net"
metabasePassword string
metabaseDbPath string
metabaseDBPath string
metabaseConfigPath string
metabaseConfigFolder = "metabase/"
metabaseConfigFile = "metabase.yaml"
@ -37,12 +39,21 @@ var (
forceYes bool
/*informations needed to setup a random password on user's behalf*/
// information needed to set up a random password on user's behalf
)
func NewDashboardCmd() *cobra.Command {
/* ---- UPDATE COMMAND */
var cmdDashboard = &cobra.Command{
type cliDashboard struct {
cfg configGetter
}
func NewCLIDashboard(cfg configGetter) *cliDashboard {
return &cliDashboard{
cfg: cfg,
}
}
func (cli *cliDashboard) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "dashboard [command]",
Short: "Manage your metabase dashboard container [requires local API]",
Long: `Install/Start/Stop/Remove a metabase container exposing dashboard and metrics.
@ -56,8 +67,9 @@ cscli dashboard start
cscli dashboard stop
cscli dashboard remove
`,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.LAPI(csConfig); err != nil {
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := require.LAPI(cfg); err != nil {
return err
}
@ -65,13 +77,13 @@ cscli dashboard remove
return err
}
metabaseConfigFolderPath := filepath.Join(csConfig.ConfigPaths.ConfigDir, metabaseConfigFolder)
metabaseConfigFolderPath := filepath.Join(cfg.ConfigPaths.ConfigDir, metabaseConfigFolder)
metabaseConfigPath = filepath.Join(metabaseConfigFolderPath, metabaseConfigFile)
if err := os.MkdirAll(metabaseConfigFolderPath, os.ModePerm); err != nil {
return err
}
if err := require.DB(csConfig); err != nil {
if err := require.DB(cfg); err != nil {
return err
}
@ -86,23 +98,24 @@ cscli dashboard remove
metabaseContainerID = oldContainerID
}
}
return nil
},
}
cmdDashboard.AddCommand(NewDashboardSetupCmd())
cmdDashboard.AddCommand(NewDashboardStartCmd())
cmdDashboard.AddCommand(NewDashboardStopCmd())
cmdDashboard.AddCommand(NewDashboardShowPasswordCmd())
cmdDashboard.AddCommand(NewDashboardRemoveCmd())
cmd.AddCommand(cli.newSetupCmd())
cmd.AddCommand(cli.newStartCmd())
cmd.AddCommand(cli.newStopCmd())
cmd.AddCommand(cli.newShowPasswordCmd())
cmd.AddCommand(cli.newRemoveCmd())
return cmdDashboard
return cmd
}
func NewDashboardSetupCmd() *cobra.Command {
func (cli *cliDashboard) newSetupCmd() *cobra.Command {
var force bool
var cmdDashSetup = &cobra.Command{
cmd := &cobra.Command{
Use: "setup",
Short: "Setup a metabase container.",
Long: `Perform a metabase docker setup, download standard dashboards, create a fresh user and start the container`,
@ -113,9 +126,9 @@ cscli dashboard setup
cscli dashboard setup --listen 0.0.0.0
cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
`,
RunE: func(cmd *cobra.Command, args []string) error {
if metabaseDbPath == "" {
metabaseDbPath = csConfig.ConfigPaths.DataDir
RunE: func(_ *cobra.Command, _ []string) error {
if metabaseDBPath == "" {
metabaseDBPath = cli.cfg().ConfigPaths.DataDir
}
if metabasePassword == "" {
@ -136,7 +149,10 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
if err != nil {
return err
}
mb, err := metabase.SetupMetabase(csConfig.API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDbPath, dockerGroup.Gid, metabaseContainerID, metabaseImage)
if err = cli.chownDatabase(dockerGroup.Gid); err != nil {
return err
}
mb, err := metabase.SetupMetabase(cli.cfg().API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDBPath, dockerGroup.Gid, metabaseContainerID, metabaseImage)
if err != nil {
return err
}
@ -149,29 +165,32 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
fmt.Printf("\tURL : '%s'\n", mb.Config.ListenURL)
fmt.Printf("\tusername : '%s'\n", mb.Config.Username)
fmt.Printf("\tpassword : '%s'\n", mb.Config.Password)
return nil
},
}
cmdDashSetup.Flags().BoolVarP(&force, "force", "f", false, "Force setup : override existing files")
cmdDashSetup.Flags().StringVarP(&metabaseDbPath, "dir", "d", "", "Shared directory with metabase container")
cmdDashSetup.Flags().StringVarP(&metabaseListenAddress, "listen", "l", metabaseListenAddress, "Listen address of container")
cmdDashSetup.Flags().StringVar(&metabaseImage, "metabase-image", metabaseImage, "Metabase image to use")
cmdDashSetup.Flags().StringVarP(&metabaseListenPort, "port", "p", metabaseListenPort, "Listen port of container")
cmdDashSetup.Flags().BoolVarP(&forceYes, "yes", "y", false, "force yes")
//cmdDashSetup.Flags().StringVarP(&metabaseUser, "user", "u", "crowdsec@crowdsec.net", "metabase user")
cmdDashSetup.Flags().StringVar(&metabasePassword, "password", "", "metabase password")
return cmdDashSetup
flags := cmd.Flags()
flags.BoolVarP(&force, "force", "f", false, "Force setup : override existing files")
flags.StringVarP(&metabaseDBPath, "dir", "d", "", "Shared directory with metabase container")
flags.StringVarP(&metabaseListenAddress, "listen", "l", metabaseListenAddress, "Listen address of container")
flags.StringVar(&metabaseImage, "metabase-image", metabaseImage, "Metabase image to use")
flags.StringVarP(&metabaseListenPort, "port", "p", metabaseListenPort, "Listen port of container")
flags.BoolVarP(&forceYes, "yes", "y", false, "force yes")
// flags.StringVarP(&metabaseUser, "user", "u", "crowdsec@crowdsec.net", "metabase user")
flags.StringVar(&metabasePassword, "password", "", "metabase password")
return cmd
}
func NewDashboardStartCmd() *cobra.Command {
var cmdDashStart = &cobra.Command{
func (cli *cliDashboard) newStartCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "start",
Short: "Start the metabase container.",
Long: `Stats the metabase container using docker.`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
mb, err := metabase.NewMetabase(metabaseConfigPath, metabaseContainerID)
if err != nil {
return err
@ -185,51 +204,57 @@ func NewDashboardStartCmd() *cobra.Command {
}
log.Infof("Started metabase")
log.Infof("url : http://%s:%s", mb.Config.ListenAddr, mb.Config.ListenPort)
return nil
},
}
cmdDashStart.Flags().BoolVarP(&forceYes, "yes", "y", false, "force yes")
return cmdDashStart
cmd.Flags().BoolVarP(&forceYes, "yes", "y", false, "force yes")
return cmd
}
func NewDashboardStopCmd() *cobra.Command {
var cmdDashStop = &cobra.Command{
func (cli *cliDashboard) newStopCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "stop",
Short: "Stops the metabase container.",
Long: `Stops the metabase container using docker.`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
if err := metabase.StopContainer(metabaseContainerID); err != nil {
return fmt.Errorf("unable to stop container '%s': %s", metabaseContainerID, err)
}
return nil
},
}
return cmdDashStop
return cmd
}
func NewDashboardShowPasswordCmd() *cobra.Command {
var cmdDashShowPassword = &cobra.Command{Use: "show-password",
func (cli *cliDashboard) newShowPasswordCmd() *cobra.Command {
cmd := &cobra.Command{Use: "show-password",
Short: "displays password of metabase.",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
m := metabase.Metabase{}
if err := m.LoadConfig(metabaseConfigPath); err != nil {
return err
}
log.Printf("'%s'", m.Config.Password)
return nil
},
}
return cmdDashShowPassword
return cmd
}
func NewDashboardRemoveCmd() *cobra.Command {
func (cli *cliDashboard) newRemoveCmd() *cobra.Command {
var force bool
var cmdDashRemove = &cobra.Command{
cmd := &cobra.Command{
Use: "remove",
Short: "removes the metabase container.",
Long: `removes the metabase container using docker.`,
@ -239,7 +264,7 @@ func NewDashboardRemoveCmd() *cobra.Command {
cscli dashboard remove
cscli dashboard remove --force
`,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
if !forceYes {
var answer bool
prompt := &survey.Confirm{
@ -276,8 +301,8 @@ cscli dashboard remove --force
}
log.Infof("container %s stopped & removed", metabaseContainerID)
}
log.Debugf("Removing metabase db %s", csConfig.ConfigPaths.DataDir)
if err := metabase.RemoveDatabase(csConfig.ConfigPaths.DataDir); err != nil {
log.Debugf("Removing metabase db %s", cli.cfg().ConfigPaths.DataDir)
if err := metabase.RemoveDatabase(cli.cfg().ConfigPaths.DataDir); err != nil {
log.Warnf("failed to remove metabase internal db : %s", err)
}
if force {
@ -291,20 +316,25 @@ cscli dashboard remove --force
}
}
}
return nil
},
}
cmdDashRemove.Flags().BoolVarP(&force, "force", "f", false, "Remove also the metabase image")
cmdDashRemove.Flags().BoolVarP(&forceYes, "yes", "y", false, "force yes")
return cmdDashRemove
flags := cmd.Flags()
flags.BoolVarP(&force, "force", "f", false, "Remove also the metabase image")
flags.BoolVarP(&forceYes, "yes", "y", false, "force yes")
return cmd
}
func passwordIsValid(password string) bool {
hasDigit := false
for _, j := range password {
if unicode.IsDigit(j) {
hasDigit = true
break
}
}
@ -312,8 +342,8 @@ func passwordIsValid(password string) bool {
if !hasDigit || len(password) < 6 {
return false
}
return true
return true
}
func checkSystemMemory(forceYes *bool) error {
@ -321,8 +351,10 @@ func checkSystemMemory(forceYes *bool) error {
if totMem >= uint64(math.Pow(2, 30)) {
return nil
}
if !*forceYes {
var answer bool
prompt := &survey.Confirm{
Message: "Metabase requires 1-2GB of RAM, your system is below this requirement continue ?",
Default: true,
@ -330,12 +362,16 @@ func checkSystemMemory(forceYes *bool) error {
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask about RAM check: %s", err)
}
if !answer {
return fmt.Errorf("user stated no to continue")
}
return nil
}
log.Warn("Metabase requires 1-2GB of RAM, your system is below this requirement")
return nil
}
@ -343,68 +379,97 @@ func warnIfNotLoopback(addr string) {
if addr == "127.0.0.1" || addr == "::1" {
return
}
log.Warnf("You are potentially exposing your metabase port to the internet (addr: %s), please consider using a reverse proxy", addr)
}
func disclaimer(forceYes *bool) error {
if !*forceYes {
var answer bool
prompt := &survey.Confirm{
Message: "CrowdSec takes no responsibility for the security of your metabase instance. Do you accept these responsibilities ?",
Default: true,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask to question: %s", err)
}
if !answer {
return fmt.Errorf("user stated no to responsibilities")
}
return nil
}
log.Warn("CrowdSec takes no responsibility for the security of your metabase instance. You used force yes, so you accept this disclaimer")
return nil
}
func checkGroups(forceYes *bool) (*user.Group, error) {
groupExist := false
dockerGroup, err := user.LookupGroup(crowdsecGroup)
if err == nil {
groupExist = true
return dockerGroup, nil
}
if !groupExist {
if !*forceYes {
var answer bool
prompt := &survey.Confirm{
Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup),
Default: true,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return dockerGroup, fmt.Errorf("unable to ask to question: %s", err)
}
if !answer {
return dockerGroup, fmt.Errorf("unable to continue without creating '%s' group", crowdsecGroup)
}
}
groupAddCmd, err := exec.LookPath("groupadd")
if err != nil {
return dockerGroup, fmt.Errorf("unable to find 'groupadd' command, can't continue")
if !*forceYes {
var answer bool
prompt := &survey.Confirm{
Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup),
Default: true,
}
groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}}
if err := groupAdd.Run(); err != nil {
return dockerGroup, fmt.Errorf("unable to add group '%s': %s", dockerGroup, err)
if err := survey.AskOne(prompt, &answer); err != nil {
return dockerGroup, fmt.Errorf("unable to ask to question: %s", err)
}
dockerGroup, err = user.LookupGroup(crowdsecGroup)
if err != nil {
return dockerGroup, fmt.Errorf("unable to lookup '%s' group: %+v", dockerGroup, err)
if !answer {
return dockerGroup, fmt.Errorf("unable to continue without creating '%s' group", crowdsecGroup)
}
}
intID, err := strconv.Atoi(dockerGroup.Gid)
groupAddCmd, err := exec.LookPath("groupadd")
if err != nil {
return dockerGroup, fmt.Errorf("unable to convert group ID to int: %s", err)
return dockerGroup, fmt.Errorf("unable to find 'groupadd' command, can't continue")
}
if err := os.Chown(csConfig.DbConfig.DbPath, 0, intID); err != nil {
return dockerGroup, fmt.Errorf("unable to chown sqlite db file '%s': %s", csConfig.DbConfig.DbPath, err)
groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}}
if err := groupAdd.Run(); err != nil {
return dockerGroup, fmt.Errorf("unable to add group '%s': %s", dockerGroup, err)
}
return dockerGroup, nil
return user.LookupGroup(crowdsecGroup)
}
func (cli *cliDashboard) chownDatabase(gid string) error {
cfg := cli.cfg()
intID, err := strconv.Atoi(gid)
if err != nil {
return fmt.Errorf("unable to convert group ID to int: %s", err)
}
if stat, err := os.Stat(cfg.DbConfig.DbPath); !os.IsNotExist(err) {
info := stat.Sys()
if err := os.Chown(cfg.DbConfig.DbPath, int(info.(*syscall.Stat_t).Uid), intID); err != nil {
return fmt.Errorf("unable to chown sqlite db file '%s': %s", cfg.DbConfig.DbPath, err)
}
}
if cfg.DbConfig.Type == "sqlite" && cfg.DbConfig.UseWal != nil && *cfg.DbConfig.UseWal {
for _, ext := range []string{"-wal", "-shm"} {
file := cfg.DbConfig.DbPath + ext
if stat, err := os.Stat(file); !os.IsNotExist(err) {
info := stat.Sys()
if err := os.Chown(file, int(info.(*syscall.Stat_t).Uid), intID); err != nil {
return fmt.Errorf("unable to chown sqlite db file '%s': %s", file, err)
}
}
}
}
return nil
}

32
cmd/crowdsec-cli/dashboard_unsupported.go Normal file
View file

@ -0,0 +1,32 @@
//go:build !linux
package main
import (
"runtime"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
type cliDashboard struct{
cfg configGetter
}
func NewCLIDashboard(cfg configGetter) *cliDashboard {
return &cliDashboard{
cfg: cfg,
}
}
func (cli cliDashboard) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "dashboard",
DisableAutoGenTag: true,
Run: func(_ *cobra.Command, _ []string) {
log.Infof("Dashboard command is disabled on %s", runtime.GOOS)
},
}
return cmd
}

185
cmd/crowdsec-cli/decisions.go
View file

@ -25,7 +25,7 @@ import (
var Client *apiclient.ApiClient
func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
func (cli *cliDecisions) decisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
/*here we cheat a bit : to make it more readable for the user, we dedup some entries*/
spamLimit := make(map[string]bool)
skipped := 0
@ -33,27 +33,36 @@ func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error
for aIdx := 0; aIdx < len(*alerts); aIdx++ {
alertItem := (*alerts)[aIdx]
newDecisions := make([]*models.Decision, 0)
for _, decisionItem := range alertItem.Decisions {
spamKey := fmt.Sprintf("%t:%s:%s:%s", *decisionItem.Simulated, *decisionItem.Type, *decisionItem.Scope, *decisionItem.Value)
if _, ok := spamLimit[spamKey]; ok {
skipped++
continue
}
spamLimit[spamKey] = true
newDecisions = append(newDecisions, decisionItem)
}
alertItem.Decisions = newDecisions
}
if csConfig.Cscli.Output == "raw" {
switch cli.cfg().Cscli.Output {
case "raw":
csvwriter := csv.NewWriter(os.Stdout)
header := []string{"id", "source", "ip", "reason", "action", "country", "as", "events_count", "expiration", "simulated", "alert_id"}
if printMachine {
header = append(header, "machine")
}
err := csvwriter.Write(header)
if err != nil {
return err
}
for _, alertItem := range *alerts {
for _, decisionItem := range alertItem.Decisions {
raw := []string{
@ -79,31 +88,46 @@ func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error
}
}
}
csvwriter.Flush()
} else if csConfig.Cscli.Output == "json" {
case "json":
if *alerts == nil {
// avoid returning "null" in `json"
// could be cleaner if we used slice of alerts directly
fmt.Println("[]")
return nil
}
x, _ := json.MarshalIndent(alerts, "", " ")
fmt.Printf("%s", string(x))
} else if csConfig.Cscli.Output == "human" {
case "human":
if len(*alerts) == 0 {
fmt.Println("No active decisions")
return nil
}
decisionsTable(color.Output, alerts, printMachine)
cli.decisionsTable(color.Output, alerts, printMachine)
if skipped > 0 {
fmt.Printf("%d duplicated entries skipped\n", skipped)
}
}
return nil
}
func NewDecisionsCmd() *cobra.Command {
var cmdDecisions = &cobra.Command{
type cliDecisions struct {
cfg configGetter
}
func NewCLIDecisions(cfg configGetter) *cliDecisions {
return &cliDecisions{
cfg: cfg,
}
}
func (cli *cliDecisions) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "decisions [action]",
Short: "Manage decisions",
Long: `Add/List/Delete/Import decisions from LAPI`,
@ -112,17 +136,18 @@ func NewDecisionsCmd() *cobra.Command {
/*TBD example*/
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIClient(); err != nil {
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := cfg.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
}
password := strfmt.Password(csConfig.API.Client.Credentials.Password)
apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL)
password := strfmt.Password(cfg.API.Client.Credentials.Password)
apiurl, err := url.Parse(cfg.API.Client.Credentials.URL)
if err != nil {
return fmt.Errorf("parsing api url %s: %w", csConfig.API.Client.Credentials.URL, err)
return fmt.Errorf("parsing api url %s: %w", cfg.API.Client.Credentials.URL, err)
}
Client, err = apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
MachineID: cfg.API.Client.Credentials.Login,
Password: password,
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiurl,
@ -131,19 +156,20 @@ func NewDecisionsCmd() *cobra.Command {
if err != nil {
return fmt.Errorf("creating api client: %w", err)
}
return nil
},
}
cmdDecisions.AddCommand(NewDecisionsListCmd())
cmdDecisions.AddCommand(NewDecisionsAddCmd())
cmdDecisions.AddCommand(NewDecisionsDeleteCmd())
cmdDecisions.AddCommand(NewDecisionsImportCmd())
cmd.AddCommand(cli.newListCmd())
cmd.AddCommand(cli.newAddCmd())
cmd.AddCommand(cli.newDeleteCmd())
cmd.AddCommand(cli.newImportCmd())
return cmdDecisions
return cmd
}
func NewDecisionsListCmd() *cobra.Command {
func (cli *cliDecisions) newListCmd() *cobra.Command {
var filter = apiclient.AlertsListOpts{
ValueEquals: new(string),
ScopeEquals: new(string),
@ -157,11 +183,13 @@ func NewDecisionsListCmd() *cobra.Command {
IncludeCAPI: new(bool),
Limit: new(int),
}
NoSimu := new(bool)
contained := new(bool)
var printMachine bool
var cmdDecisionsList = &cobra.Command{
cmd := &cobra.Command{
Use: "list [options]",
Short: "List decisions from LAPI",
Example: `cscli decisions list -i 1.2.3.4
@ -171,7 +199,7 @@ cscli decisions list -t ban
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(cmd *cobra.Command, _ []string) error {
var err error
/*take care of shorthand options*/
if err = manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil {
@ -243,7 +271,7 @@ cscli decisions list -t ban
return fmt.Errorf("unable to retrieve decisions: %w", err)
}
err = DecisionsToTable(alerts, printMachine)
err = cli.decisionsToTable(alerts, printMachine)
if err != nil {
return fmt.Errorf("unable to print decisions: %w", err)
}
@ -251,26 +279,26 @@ cscli decisions list -t ban
return nil
},
}
cmdDecisionsList.Flags().SortFlags = false
cmdDecisionsList.Flags().BoolVarP(filter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
cmdDecisionsList.Flags().StringVar(filter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
cmdDecisionsList.Flags().StringVar(filter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
cmdDecisionsList.Flags().StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)")
cmdDecisionsList.Flags().StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)")
cmdDecisionsList.Flags().StringVar(filter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmdDecisionsList.Flags().StringVarP(filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)")
cmdDecisionsList.Flags().StringVarP(filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)")
cmdDecisionsList.Flags().StringVarP(filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
cmdDecisionsList.Flags().StringVarP(filter.RangeEquals, "range", "r", "", "restrict to alerts from this source range (shorthand for --scope range --value <RANGE>)")
cmdDecisionsList.Flags().IntVarP(filter.Limit, "limit", "l", 100, "number of alerts to get (use 0 to remove the limit)")
cmdDecisionsList.Flags().BoolVar(NoSimu, "no-simu", false, "exclude decisions in simulation mode")
cmdDecisionsList.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that triggered decisions")
cmdDecisionsList.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
cmd.Flags().SortFlags = false
cmd.Flags().BoolVarP(filter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
cmd.Flags().StringVar(filter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
cmd.Flags().StringVar(filter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
cmd.Flags().StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)")
cmd.Flags().StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)")
cmd.Flags().StringVar(filter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmd.Flags().StringVarP(filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)")
cmd.Flags().StringVarP(filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)")
cmd.Flags().StringVarP(filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
cmd.Flags().StringVarP(filter.RangeEquals, "range", "r", "", "restrict to alerts from this source range (shorthand for --scope range --value <RANGE>)")
cmd.Flags().IntVarP(filter.Limit, "limit", "l", 100, "number of alerts to get (use 0 to remove the limit)")
cmd.Flags().BoolVar(NoSimu, "no-simu", false, "exclude decisions in simulation mode")
cmd.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that triggered decisions")
cmd.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
return cmdDecisionsList
return cmd
}
func NewDecisionsAddCmd() *cobra.Command {
func (cli *cliDecisions) newAddCmd() *cobra.Command {
var (
addIP string
addRange string
@ -281,7 +309,7 @@ func NewDecisionsAddCmd() *cobra.Command {
addType string
)
var cmdDecisionsAdd = &cobra.Command{
cmd := &cobra.Command{
Use: "add [options]",
Short: "Add decision to LAPI",
Example: `cscli decisions add --ip 1.2.3.4
@ -292,7 +320,7 @@ cscli decisions add --scope username --value foobar
/*TBD : fix long and example*/
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(cmd *cobra.Command, _ []string) error {
var err error
alerts := models.AddAlertsRequest{}
origin := types.CscliOrigin
@ -306,7 +334,7 @@ cscli decisions add --scope username --value foobar
createdAt := time.Now().UTC().Format(time.RFC3339)
/*take care of shorthand options*/
if err := manageCliDecisionAlerts(&addIP, &addRange, &addScope, &addValue); err != nil {
if err = manageCliDecisionAlerts(&addIP, &addRange, &addScope, &addValue); err != nil {
return err
}
@ -318,11 +346,11 @@ cscli decisions add --scope username --value foobar
addScope = types.Range
} else if addValue == "" {
printHelp(cmd)
return fmt.Errorf("Missing arguments, a value is required (--ip, --range or --scope and --value)")
return fmt.Errorf("missing arguments, a value is required (--ip, --range or --scope and --value)")
}
if addReason == "" {
addReason = fmt.Sprintf("manual '%s' from '%s'", addType, csConfig.API.Client.Credentials.Login)
addReason = fmt.Sprintf("manual '%s' from '%s'", addType, cli.cfg().API.Client.Credentials.Login)
}
decision := models.Decision{
Duration: &addDuration,
@ -365,23 +393,24 @@ cscli decisions add --scope username --value foobar
}
log.Info("Decision successfully added")
return nil
},
}
cmdDecisionsAdd.Flags().SortFlags = false
cmdDecisionsAdd.Flags().StringVarP(&addIP, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmdDecisionsAdd.Flags().StringVarP(&addRange, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmdDecisionsAdd.Flags().StringVarP(&addDuration, "duration", "d", "4h", "Decision duration (ie. 1h,4h,30m)")
cmdDecisionsAdd.Flags().StringVarP(&addValue, "value", "v", "", "The value (ie. --scope username --value foobar)")
cmdDecisionsAdd.Flags().StringVar(&addScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)")
cmdDecisionsAdd.Flags().StringVarP(&addReason, "reason", "R", "", "Decision reason (ie. scenario-name)")
cmdDecisionsAdd.Flags().StringVarP(&addType, "type", "t", "ban", "Decision type (ie. ban,captcha,throttle)")
cmd.Flags().SortFlags = false
cmd.Flags().StringVarP(&addIP, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmd.Flags().StringVarP(&addRange, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmd.Flags().StringVarP(&addDuration, "duration", "d", "4h", "Decision duration (ie. 1h,4h,30m)")
cmd.Flags().StringVarP(&addValue, "value", "v", "", "The value (ie. --scope username --value foobar)")
cmd.Flags().StringVar(&addScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)")
cmd.Flags().StringVarP(&addReason, "reason", "R", "", "Decision reason (ie. scenario-name)")
cmd.Flags().StringVarP(&addType, "type", "t", "ban", "Decision type (ie. ban,captcha,throttle)")
return cmdDecisionsAdd
return cmd
}
func NewDecisionsDeleteCmd() *cobra.Command {
func (cli *cliDecisions) newDeleteCmd() *cobra.Command {
var delFilter = apiclient.DecisionsDeleteOpts{
ScopeEquals: new(string),
ValueEquals: new(string),
@ -391,11 +420,14 @@ func NewDecisionsDeleteCmd() *cobra.Command {
ScenarioEquals: new(string),
OriginEquals: new(string),
}
var delDecisionId string
var delDecisionID string
var delDecisionAll bool
contained := new(bool)
var cmdDecisionsDelete = &cobra.Command{
cmd := &cobra.Command{
Use: "delete [options]",
Short: "Delete decisions",
DisableAutoGenTag: true,
@ -406,21 +438,21 @@ cscli decisions delete --id 42
cscli decisions delete --type captcha
`,
/*TBD : refaire le Long/Example*/
PreRunE: func(cmd *cobra.Command, args []string) error {
PreRunE: func(cmd *cobra.Command, _ []string) error {
if delDecisionAll {
return nil
}
if *delFilter.ScopeEquals == "" && *delFilter.ValueEquals == "" &&
*delFilter.TypeEquals == "" && *delFilter.IPEquals == "" &&
*delFilter.RangeEquals == "" && *delFilter.ScenarioEquals == "" &&
*delFilter.OriginEquals == "" && delDecisionId == "" {
*delFilter.OriginEquals == "" && delDecisionID == "" {
cmd.Usage()
return fmt.Errorf("at least one filter or --all must be specified")
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
var err error
var decisions *models.DeleteDecisionResponse
@ -453,36 +485,37 @@ cscli decisions delete --type captcha
delFilter.Contains = new(bool)
}
if delDecisionId == "" {
if delDecisionID == "" {
decisions, _, err = Client.Decisions.Delete(context.Background(), delFilter)
if err != nil {
return fmt.Errorf("Unable to delete decisions: %v", err)
return fmt.Errorf("unable to delete decisions: %v", err)
}
} else {
if _, err = strconv.Atoi(delDecisionId); err != nil {
return fmt.Errorf("id '%s' is not an integer: %v", delDecisionId, err)
if _, err = strconv.Atoi(delDecisionID); err != nil {
return fmt.Errorf("id '%s' is not an integer: %v", delDecisionID, err)
}
decisions, _, err = Client.Decisions.DeleteOne(context.Background(), delDecisionId)
decisions, _, err = Client.Decisions.DeleteOne(context.Background(), delDecisionID)
if err != nil {
return fmt.Errorf("Unable to delete decision: %v", err)
return fmt.Errorf("unable to delete decision: %v", err)
}
}
log.Infof("%s decision(s) deleted", decisions.NbDeleted)
return nil
},
}
cmdDecisionsDelete.Flags().SortFlags = false
cmdDecisionsDelete.Flags().StringVarP(delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmdDecisionsDelete.Flags().StringVarP(delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmdDecisionsDelete.Flags().StringVarP(delFilter.TypeEquals, "type", "t", "", "the decision type (ie. ban,captcha)")
cmdDecisionsDelete.Flags().StringVarP(delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmdDecisionsDelete.Flags().StringVarP(delFilter.ScenarioEquals, "scenario", "s", "", "the scenario name (ie. crowdsecurity/ssh-bf)")
cmdDecisionsDelete.Flags().StringVar(delFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmd.Flags().SortFlags = false
cmd.Flags().StringVarP(delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmd.Flags().StringVarP(delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmd.Flags().StringVarP(delFilter.TypeEquals, "type", "t", "", "the decision type (ie. ban,captcha)")
cmd.Flags().StringVarP(delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmd.Flags().StringVarP(delFilter.ScenarioEquals, "scenario", "s", "", "the scenario name (ie. crowdsecurity/ssh-bf)")
cmd.Flags().StringVar(delFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmdDecisionsDelete.Flags().StringVar(&delDecisionId, "id", "", "decision id")
cmdDecisionsDelete.Flags().BoolVar(&delDecisionAll, "all", false, "delete all decisions")
cmdDecisionsDelete.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
cmd.Flags().StringVar(&delDecisionID, "id", "", "decision id")
cmd.Flags().BoolVar(&delDecisionAll, "all", false, "delete all decisions")
cmd.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
return cmdDecisionsDelete
return cmd
}

27
cmd/crowdsec-cli/decisions_import.go
View file

@ -37,21 +37,25 @@ func parseDecisionList(content []byte, format string) ([]decisionRaw, error) {
switch format {
case "values":
log.Infof("Parsing values")
scanner := bufio.NewScanner(bytes.NewReader(content))
for scanner.Scan() {
value := strings.TrimSpace(scanner.Text())
ret = append(ret, decisionRaw{Value: value})
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("unable to parse values: '%s'", err)
}
case "json":
log.Infof("Parsing json")
if err := json.Unmarshal(content, &ret); err != nil {
return nil, err
}
case "csv":
log.Infof("Parsing csv")
if err := csvutil.Unmarshal(content, &ret); err != nil {
return nil, fmt.Errorf("unable to parse csv: '%s'", err)
}
@ -63,7 +67,7 @@ func parseDecisionList(content []byte, format string) ([]decisionRaw, error) {
}
func runDecisionsImport(cmd *cobra.Command, args []string) error {
func (cli *cliDecisions) runImport(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
input, err := flags.GetString("input")
@ -75,6 +79,7 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
if defaultDuration == "" {
return fmt.Errorf("--duration cannot be empty")
}
@ -83,6 +88,7 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
if defaultScope == "" {
return fmt.Errorf("--scope cannot be empty")
}
@ -91,6 +97,7 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
if defaultReason == "" {
return fmt.Errorf("--reason cannot be empty")
}
@ -99,6 +106,7 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
if defaultType == "" {
return fmt.Errorf("--type cannot be empty")
}
@ -152,6 +160,7 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error {
}
decisions := make([]*models.Decision, len(decisionsListRaw))
for i, d := range decisionsListRaw {
if d.Value == "" {
return fmt.Errorf("item %d: missing 'value'", i)
@ -222,17 +231,19 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error {
}
log.Infof("Imported %d decisions", len(decisions))
return nil
}
func NewDecisionsImportCmd() *cobra.Command {
var cmdDecisionsImport = &cobra.Command{
func (cli *cliDecisions) newImportCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "import [options]",
Short: "Import decisions from a file or pipe",
Long: "expected format:\n" +
"csv : any of duration,reason,scope,type,value, with a header line\n" +
`json : {"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`,
"json :" + "`{" + `"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"` + "}`",
Args: cobra.NoArgs,
DisableAutoGenTag: true,
Example: `decisions.csv:
duration,scope,value
@ -250,10 +261,10 @@ Raw values, standard input:
$ echo "1.2.3.4" | cscli decisions import -i - --format values
`,
RunE: runDecisionsImport,
RunE: cli.runImport,
}
flags := cmdDecisionsImport.Flags()
flags := cmd.Flags()
flags.SortFlags = false
flags.StringP("input", "i", "", "Input file")
flags.StringP("duration", "d", "4h", "Decision duration: 1h,4h,30m")
@ -263,7 +274,7 @@ $ echo "1.2.3.4" | cscli decisions import -i - --format values
flags.Int("batch", 0, "Split import in batches of N decisions")
flags.String("format", "", "Input format: 'json', 'csv' or 'values' (each line is a value, no headers)")
cmdDecisionsImport.MarkFlagRequired("input")
cmd.MarkFlagRequired("input")
return cmdDecisionsImport
return cmd
}

6
cmd/crowdsec-cli/decisions_table.go
View file

@ -8,13 +8,15 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/models"
)
func decisionsTable(out io.Writer, alerts *models.GetAlertsResponse, printMachine bool) {
func (cli *cliDecisions) decisionsTable(out io.Writer, alerts *models.GetAlertsResponse, printMachine bool) {
t := newTable(out)
t.SetRowLines(false)
header := []string{"ID", "Source", "Scope:Value", "Reason", "Action", "Country", "AS", "Events", "expiration", "Alert ID"}
if printMachine {
header = append(header, "Machine")
}
t.SetHeaders(header...)
for _, alertItem := range *alerts {
@ -22,6 +24,7 @@ func decisionsTable(out io.Writer, alerts *models.GetAlertsResponse, printMachin
if *alertItem.Simulated {
*decisionItem.Type = fmt.Sprintf("(simul)%s", *decisionItem.Type)
}
row := []string{
strconv.Itoa(int(decisionItem.ID)),
*decisionItem.Origin,
@ -42,5 +45,6 @@ func decisionsTable(out io.Writer, alerts *models.GetAlertsResponse, printMachin
t.AddRow(row...)
}
}
t.Render()
}

49
cmd/crowdsec-cli/doc.go Normal file
View file

@ -0,0 +1,49 @@
package main
import (
"fmt"
"path/filepath"
"strings"
"github.com/spf13/cobra"
"github.com/spf13/cobra/doc"
)
type cliDoc struct{}
func NewCLIDoc() *cliDoc {
return &cliDoc{}
}
func (cli cliDoc) NewCommand(rootCmd *cobra.Command) *cobra.Command {
cmd := &cobra.Command{
Use: "doc",
Short: "Generate the documentation in `./doc/`. Directory must exist.",
Args: cobra.ExactArgs(0),
Hidden: true,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
if err := doc.GenMarkdownTreeCustom(rootCmd, "./doc/", cli.filePrepender, cli.linkHandler); err != nil {
return fmt.Errorf("failed to generate cobra doc: %s", err)
}
return nil
},
}
return cmd
}
func (cli cliDoc) filePrepender(filename string) string {
const header = `---
id: %s
title: %s
---
`
name := filepath.Base(filename)
base := strings.TrimSuffix(name, filepath.Ext(name))
return fmt.Sprintf(header, base, strings.ReplaceAll(base, "_", " "))
}
func (cli cliDoc) linkHandler(name string) string {
return fmt.Sprintf("/cscli/%s", name)
}

193
cmd/crowdsec-cli/explain.go
View file

@ -2,6 +2,7 @@ package main
import (
"bufio"
"errors"
"fmt"
"io"
"os"
@ -11,6 +12,7 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/dumps"
"github.com/crowdsecurity/crowdsec/pkg/hubtest"
)
@ -21,14 +23,99 @@ func GetLineCountForFile(filepath string) (int, error) {
}
defer f.Close()
lc := 0
fs := bufio.NewScanner(f)
for fs.Scan() {
lc++
fs := bufio.NewReader(f)
for {
input, err := fs.ReadBytes('\n')
if len(input) > 1 {
lc++
}
if err != nil && err == io.EOF {
break
}
}
return lc, nil
}
func runExplain(cmd *cobra.Command, args []string) error {
type cliExplain struct{}
func NewCLIExplain() *cliExplain {
return &cliExplain{}
}
func (cli cliExplain) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "explain",
Short: "Explain log pipeline",
Long: `
Explain log pipeline
`,
Example: `
cscli explain --file ./myfile.log --type nginx
cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog
cscli explain --dsn "file://myfile.log" --type nginx
tail -n 5 myfile.log | cscli explain --type nginx -f -
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: cli.run,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
logFile, err := flags.GetString("file")
if err != nil {
return err
}
dsn, err := flags.GetString("dsn")
if err != nil {
return err
}
logLine, err := flags.GetString("log")
if err != nil {
return err
}
logType, err := flags.GetString("type")
if err != nil {
return err
}
if logLine == "" && logFile == "" && dsn == "" {
printHelp(cmd)
fmt.Println()
return fmt.Errorf("please provide --log, --file or --dsn flag")
}
if logType == "" {
printHelp(cmd)
fmt.Println()
return fmt.Errorf("please provide --type flag")
}
fileInfo, _ := os.Stdin.Stat()
if logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) {
return fmt.Errorf("the option -f - is intended to work with pipes")
}
return nil
},
}
flags := cmd.Flags()
flags.StringP("file", "f", "", "Log file to test")
flags.StringP("dsn", "d", "", "DSN to test")
flags.StringP("log", "l", "", "Log line to test")
flags.StringP("type", "t", "", "Type of the acquisition to test")
flags.String("labels", "", "Additional labels to add to the acquisition format (key:value,key2:value2)")
flags.BoolP("verbose", "v", false, "Display individual changes")
flags.Bool("failures", false, "Only show failed lines")
flags.Bool("only-successful-parsers", false, "Only show successful parsers")
flags.String("crowdsec", "crowdsec", "Path to crowdsec")
flags.Bool("no-clean", false, "Don't clean runtime environment after tests")
return cmd
}
func (cli cliExplain) run(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
logFile, err := flags.GetString("file")
@ -51,13 +138,18 @@ func runExplain(cmd *cobra.Command, args []string) error {
return err
}
opts := hubtest.DumpOpts{}
opts := dumps.DumpOpts{}
opts.Details, err = flags.GetBool("verbose")
if err != nil {
return err
}
no_clean, err := flags.GetBool("no-clean")
if err != nil {
return err
}
opts.SkipOk, err = flags.GetBool("failures")
if err != nil {
return err
@ -79,19 +171,6 @@ func runExplain(cmd *cobra.Command, args []string) error {
return err
}
fileInfo, _ := os.Stdin.Stat()
if logType == "" || (logLine == "" && logFile == "" && dsn == "") {
printHelp(cmd)
fmt.Println()
fmt.Printf("Please provide --type flag\n")
os.Exit(1)
}
if logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) {
return fmt.Errorf("the option -f - is intended to work with pipes")
}
var f *os.File
// using empty string fallback to /tmp
@ -99,10 +178,19 @@ func runExplain(cmd *cobra.Command, args []string) error {
if err != nil {
return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %s", err)
}
tmpFile := ""
defer func() {
if no_clean {
return
}
if _, err := os.Stat(dir); !os.IsNotExist(err) {
if err := os.RemoveAll(dir); err != nil {
log.Errorf("unable to delete temporary directory '%s': %s", dir, err)
}
}
}()
// we create a temporary log file if a log line/stdin has been provided
if logLine != "" || logFile == "-" {
tmpFile = filepath.Join(dir, "cscli_test_tmp.log")
tmpFile := filepath.Join(dir, "cscli_test_tmp.log")
f, err = os.Create(tmpFile)
if err != nil {
return err
@ -118,16 +206,18 @@ func runExplain(cmd *cobra.Command, args []string) error {
errCount := 0
for {
input, err := reader.ReadBytes('\n')
if err != nil && err == io.EOF {
if err != nil && errors.Is(err, io.EOF) {
break
}
_, err = f.Write(input)
if err != nil {
if len(input) > 1 {
_, err = f.Write(input)
}
if err != nil || len(input) <= 1 {
errCount++
}
}
if errCount > 0 {
log.Warnf("Failed to write %d lines to tmp file", errCount)
log.Warnf("Failed to write %d lines to %s", errCount, tmpFile)
}
}
f.Close()
@ -145,8 +235,12 @@ func runExplain(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
log.Debugf("file %s has %d lines", absolutePath, lineCount)
if lineCount == 0 {
return fmt.Errorf("the log file is empty: %s", absolutePath)
}
if lineCount > 100 {
log.Warnf("The log file contains %d lines. This may take a lot of resources.", lineCount)
log.Warnf("%s contains %d lines. This may take a lot of resources.", absolutePath, lineCount)
}
}
@ -166,63 +260,20 @@ func runExplain(cmd *cobra.Command, args []string) error {
return fmt.Errorf("fail to run crowdsec for test: %v", err)
}
// rm the temporary log file if only a log line/stdin was provided
if tmpFile != "" {
if err := os.Remove(tmpFile); err != nil {
return fmt.Errorf("unable to remove tmp log file '%s': %+v", tmpFile, err)
}
}
parserDumpFile := filepath.Join(dir, hubtest.ParserResultFileName)
bucketStateDumpFile := filepath.Join(dir, hubtest.BucketPourResultFileName)
parserDump, err := hubtest.LoadParserDump(parserDumpFile)
parserDump, err := dumps.LoadParserDump(parserDumpFile)
if err != nil {
return fmt.Errorf("unable to load parser dump result: %s", err)
}
bucketStateDump, err := hubtest.LoadBucketPourDump(bucketStateDumpFile)
bucketStateDump, err := dumps.LoadBucketPourDump(bucketStateDumpFile)
if err != nil {
return fmt.Errorf("unable to load bucket dump result: %s", err)
}
hubtest.DumpTree(*parserDump, *bucketStateDump, opts)
if err := os.RemoveAll(dir); err != nil {
return fmt.Errorf("unable to delete temporary directory '%s': %s", dir, err)
}
dumps.DumpTree(*parserDump, *bucketStateDump, opts)
return nil
}
func NewExplainCmd() *cobra.Command {
cmdExplain := &cobra.Command{
Use: "explain",
Short: "Explain log pipeline",
Long: `
Explain log pipeline
`,
Example: `
cscli explain --file ./myfile.log --type nginx
cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog
cscli explain --dsn "file://myfile.log" --type nginx
tail -n 5 myfile.log | cscli explain --type nginx -f -
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: runExplain,
}
flags := cmdExplain.Flags()
flags.StringP("file", "f", "", "Log file to test")
flags.StringP("dsn", "d", "", "DSN to test")
flags.StringP("log", "l", "", "Log line to test")
flags.StringP("type", "t", "", "Type of the acquisition to test")
flags.String("labels", "", "Additional labels to add to the acquisition format (key:value,key2:value2)")
flags.BoolP("verbose", "v", false, "Display individual changes")
flags.Bool("failures", false, "Only show failed lines")
flags.Bool("only-successful-parsers", false, "Only show successful parsers")
flags.String("crowdsec", "crowdsec", "Path to crowdsec")
return cmdExplain
}

29
cmd/crowdsec-cli/flag.go Normal file
View file

@ -0,0 +1,29 @@
package main
// Custom types for flag validation and conversion.
import (
"errors"
)
type MachinePassword string
func (p *MachinePassword) String() string {
return string(*p)
}
func (p *MachinePassword) Set(v string) error {
// a password can't be more than 72 characters
// due to bcrypt limitations
if len(v) > 72 {
return errors.New("password too long (max 72 characters)")
}
*p = MachinePassword(v)
return nil
}
func (p *MachinePassword) Type() string {
return "string"
}

288
cmd/crowdsec-cli/hub.go
View file

@ -1,162 +1,228 @@
package main
import (
"errors"
"encoding/json"
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewHubCmd() *cobra.Command {
var cmdHub = &cobra.Command{
type cliHub struct {
cfg configGetter
}
func NewCLIHub(cfg configGetter) *cliHub {
return &cliHub{
cfg: cfg,
}
}
func (cli *cliHub) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "hub [action]",
Short: "Manage Hub",
Long: `
Hub management
Short: "Manage hub index",
Long: `Hub management
List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net).
The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.
`,
Example: `
cscli hub list # List all installed configurations
cscli hub update # Download list of available configurations from the hub
`,
The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.`,
Example: `cscli hub list
cscli hub update
cscli hub upgrade`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
return nil
},
}
cmdHub.PersistentFlags().StringVarP(&cwhub.HubBranch, "branch", "b", "", "Use given branch from hub")
cmdHub.AddCommand(NewHubListCmd())
cmdHub.AddCommand(NewHubUpdateCmd())
cmdHub.AddCommand(NewHubUpgradeCmd())
cmd.AddCommand(cli.newListCmd())
cmd.AddCommand(cli.newUpdateCmd())
cmd.AddCommand(cli.newUpgradeCmd())
cmd.AddCommand(cli.newTypesCmd())
return cmdHub
return cmd
}
func NewHubListCmd() *cobra.Command {
var cmdHubList = &cobra.Command{
func (cli *cliHub) list(all bool) error {
hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger())
if err != nil {
return err
}
for _, v := range hub.Warnings {
log.Info(v)
}
for _, line := range hub.ItemStats() {
log.Info(line)
}
items := make(map[string][]*cwhub.Item)
for _, itemType := range cwhub.ItemTypes {
items[itemType], err = selectItems(hub, itemType, nil, !all)
if err != nil {
return err
}
}
err = listItems(color.Output, cwhub.ItemTypes, items, true)
if err != nil {
return err
}
return nil
}
func (cli *cliHub) newListCmd() *cobra.Command {
var all bool
cmd := &cobra.Command{
Use: "list [-a]",
Short: "List installed configs",
Short: "List all installed configurations",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
// use LocalSync to get warnings about tainted / outdated items
_, warn := cwhub.LocalSync(csConfig.Hub)
for _, v := range warn {
log.Info(v)
}
cwhub.DisplaySummary()
ListItems(color.Output, []string{
cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW,
}, args, true, false, all)
return nil
RunE: func(_ *cobra.Command, _ []string) error {
return cli.list(all)
},
}
cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmdHubList
flags := cmd.Flags()
flags.BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmd
}
func NewHubUpdateCmd() *cobra.Command {
var cmdHubUpdate = &cobra.Command{
func (cli *cliHub) update() error {
local := cli.cfg().Hub
remote := require.RemoteHub(cli.cfg())
// don't use require.Hub because if there is no index file, it would fail
hub, err := cwhub.NewHub(local, remote, true, log.StandardLogger())
if err != nil {
return fmt.Errorf("failed to update hub: %w", err)
}
for _, v := range hub.Warnings {
log.Info(v)
}
return nil
}
func (cli *cliHub) newUpdateCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "update",
Short: "Fetch available configs from hub",
Short: "Download the latest index (catalog of available configurations)",
Long: `
Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs.
Fetches the .index.json file from the hub, containing the list of available configs.
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
return err
}
if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
if !errors.Is(err, cwhub.ErrIndexNotFound) {
return fmt.Errorf("failed to get Hub index : %w", err)
}
log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch)
cwhub.HubBranch = "master"
if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
return fmt.Errorf("failed to get Hub index after retry: %w", err)
}
}
//use LocalSync to get warnings about tainted / outdated items
_, warn := cwhub.LocalSync(csConfig.Hub)
for _, v := range warn {
log.Info(v)
}
return nil
RunE: func(_ *cobra.Command, _ []string) error {
return cli.update()
},
}
return cmdHubUpdate
return cmd
}
func NewHubUpgradeCmd() *cobra.Command {
var cmdHubUpgrade = &cobra.Command{
func (cli *cliHub) upgrade(force bool) error {
hub, err := require.Hub(cli.cfg(), require.RemoteHub(cli.cfg()), log.StandardLogger())
if err != nil {
return err
}
for _, itemType := range cwhub.ItemTypes {
items, err := hub.GetInstalledItems(itemType)
if err != nil {
return err
}
updated := 0
log.Infof("Upgrading %s", itemType)
for _, item := range items {
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
updated++
}
}
log.Infof("Upgraded %d %s", updated, itemType)
}
return nil
}
func (cli *cliHub) newUpgradeCmd() *cobra.Command {
var force bool
cmd := &cobra.Command{
Use: "upgrade",
Short: "Upgrade all configs installed from hub",
Short: "Upgrade all configurations to their latest version",
Long: `
Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available.
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
log.Infof("Upgrading collections")
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
log.Infof("Upgrading parsers")
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
log.Infof("Upgrading scenarios")
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
log.Infof("Upgrading postoverflows")
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
return nil
RunE: func(_ *cobra.Command, _ []string) error {
return cli.upgrade(force)
},
}
cmdHubUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
return cmdHubUpgrade
flags := cmd.Flags()
flags.BoolVar(&force, "force", false, "Force upgrade: overwrite tainted and outdated files")
return cmd
}
func (cli *cliHub) types() error {
switch cli.cfg().Cscli.Output {
case "human":
s, err := yaml.Marshal(cwhub.ItemTypes)
if err != nil {
return err
}
fmt.Print(string(s))
case "json":
jsonStr, err := json.Marshal(cwhub.ItemTypes)
if err != nil {
return err
}
fmt.Println(string(jsonStr))
case "raw":
for _, itemType := range cwhub.ItemTypes {
fmt.Println(itemType)
}
}
return nil
}
func (cli *cliHub) newTypesCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "types",
Short: "List supported item types",
Long: `
List the types of supported hub items.
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.types()
},
}
return cmd
}

121
cmd/crowdsec-cli/hubappsec.go Normal file
View file

@ -0,0 +1,121 @@
package main
import (
"fmt"
"os"
"golang.org/x/text/cases"
"golang.org/x/text/language"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/pkg/appsec"
"github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIAppsecConfig() *cliItem {
return &cliItem{
name: cwhub.APPSEC_CONFIGS,
singular: "appsec-config",
oneOrMore: "appsec-config(s)",
help: cliHelp{
example: `cscli appsec-configs list -a
cscli appsec-configs install crowdsecurity/vpatch
cscli appsec-configs inspect crowdsecurity/vpatch
cscli appsec-configs upgrade crowdsecurity/vpatch
cscli appsec-configs remove crowdsecurity/vpatch
`,
},
installHelp: cliHelp{
example: `cscli appsec-configs install crowdsecurity/vpatch`,
},
removeHelp: cliHelp{
example: `cscli appsec-configs remove crowdsecurity/vpatch`,
},
upgradeHelp: cliHelp{
example: `cscli appsec-configs upgrade crowdsecurity/vpatch`,
},
inspectHelp: cliHelp{
example: `cscli appsec-configs inspect crowdsecurity/vpatch`,
},
listHelp: cliHelp{
example: `cscli appsec-configs list
cscli appsec-configs list -a
cscli appsec-configs list crowdsecurity/vpatch`,
},
}
}
func NewCLIAppsecRule() *cliItem {
inspectDetail := func(item *cwhub.Item) error {
// Only show the converted rules in human mode
if csConfig.Cscli.Output != "human" {
return nil
}
appsecRule := appsec.AppsecCollectionConfig{}
yamlContent, err := os.ReadFile(item.State.LocalPath)
if err != nil {
return fmt.Errorf("unable to read file %s : %s", item.State.LocalPath, err)
}
if err := yaml.Unmarshal(yamlContent, &appsecRule); err != nil {
return fmt.Errorf("unable to unmarshal yaml file %s : %s", item.State.LocalPath, err)
}
for _, ruleType := range appsec_rule.SupportedTypes() {
fmt.Printf("\n%s format:\n", cases.Title(language.Und, cases.NoLower).String(ruleType))
for _, rule := range appsecRule.Rules {
convertedRule, _, err := rule.Convert(ruleType, appsecRule.Name)
if err != nil {
return fmt.Errorf("unable to convert rule %s : %s", rule.Name, err)
}
fmt.Println(convertedRule)
}
switch ruleType { //nolint:gocritic
case appsec_rule.ModsecurityRuleType:
for _, rule := range appsecRule.SecLangRules {
fmt.Println(rule)
}
}
}
return nil
}
return &cliItem{
name: "appsec-rules",
singular: "appsec-rule",
oneOrMore: "appsec-rule(s)",
help: cliHelp{
example: `cscli appsec-rules list -a
cscli appsec-rules install crowdsecurity/crs
cscli appsec-rules inspect crowdsecurity/crs
cscli appsec-rules upgrade crowdsecurity/crs
cscli appsec-rules remove crowdsecurity/crs
`,
},
installHelp: cliHelp{
example: `cscli appsec-rules install crowdsecurity/crs`,
},
removeHelp: cliHelp{
example: `cscli appsec-rules remove crowdsecurity/crs`,
},
upgradeHelp: cliHelp{
example: `cscli appsec-rules upgrade crowdsecurity/crs`,
},
inspectHelp: cliHelp{
example: `cscli appsec-rules inspect crowdsecurity/crs`,
},
inspectDetail: inspectDetail,
listHelp: cliHelp{
example: `cscli appsec-rules list
cscli appsec-rules list -a
cscli appsec-rules list crowdsecurity/crs`,
},
}
}

40
cmd/crowdsec-cli/hubcollection.go Normal file
View file

@ -0,0 +1,40 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLICollection() *cliItem {
return &cliItem{
name: cwhub.COLLECTIONS,
singular: "collection",
oneOrMore: "collection(s)",
help: cliHelp{
example: `cscli collections list -a
cscli collections install crowdsecurity/http-cve crowdsecurity/iptables
cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables
cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables
cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
`,
},
installHelp: cliHelp{
example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`,
},
removeHelp: cliHelp{
example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`,
},
upgradeHelp: cliHelp{
example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`,
},
inspectHelp: cliHelp{
example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`,
},
listHelp: cliHelp{
example: `cscli collections list
cscli collections list -a
cscli collections list crowdsecurity/http-cve crowdsecurity/iptables
List only enabled collections unless "-a" or names are specified.`,
},
}
}

40
cmd/crowdsec-cli/hubcontext.go Normal file
View file

@ -0,0 +1,40 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIContext() *cliItem {
return &cliItem{
name: cwhub.CONTEXTS,
singular: "context",
oneOrMore: "context(s)",
help: cliHelp{
example: `cscli contexts list -a
cscli contexts install crowdsecurity/yyy crowdsecurity/zzz
cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz
cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz
cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz
`,
},
installHelp: cliHelp{
example: `cscli contexts install crowdsecurity/yyy crowdsecurity/zzz`,
},
removeHelp: cliHelp{
example: `cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz`,
},
upgradeHelp: cliHelp{
example: `cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz`,
},
inspectHelp: cliHelp{
example: `cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz`,
},
listHelp: cliHelp{
example: `cscli contexts list
cscli contexts list -a
cscli contexts list crowdsecurity/yyy crowdsecurity/zzz
List only enabled contexts unless "-a" or names are specified.`,
},
}
}

40
cmd/crowdsec-cli/hubparser.go Normal file
View file

@ -0,0 +1,40 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIParser() *cliItem {
return &cliItem{
name: cwhub.PARSERS,
singular: "parser",
oneOrMore: "parser(s)",
help: cliHelp{
example: `cscli parsers list -a
cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
`,
},
installHelp: cliHelp{
example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
removeHelp: cliHelp{
example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
upgradeHelp: cliHelp{
example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
inspectHelp: cliHelp{
example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`,
},
listHelp: cliHelp{
example: `cscli parsers list
cscli parsers list -a
cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs
List only enabled parsers unless "-a" or names are specified.`,
},
}
}

40
cmd/crowdsec-cli/hubpostoverflow.go Normal file
View file

@ -0,0 +1,40 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIPostOverflow() *cliItem {
return &cliItem{
name: cwhub.POSTOVERFLOWS,
singular: "postoverflow",
oneOrMore: "postoverflow(s)",
help: cliHelp{
example: `cscli postoverflows list -a
cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
`,
},
installHelp: cliHelp{
example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
removeHelp: cliHelp{
example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
upgradeHelp: cliHelp{
example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
inspectHelp: cliHelp{
example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
listHelp: cliHelp{
example: `cscli postoverflows list
cscli postoverflows list -a
cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns
List only enabled postoverflows unless "-a" or names are specified.`,
},
}
}

40
cmd/crowdsec-cli/hubscenario.go Normal file
View file

@ -0,0 +1,40 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIScenario() *cliItem {
return &cliItem{
name: cwhub.SCENARIOS,
singular: "scenario",
oneOrMore: "scenario(s)",
help: cliHelp{
example: `cscli scenarios list -a
cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
`,
},
installHelp: cliHelp{
example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
removeHelp: cliHelp{
example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
upgradeHelp: cliHelp{
example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
inspectHelp: cliHelp{
example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
listHelp: cliHelp{
example: `cscli scenarios list
cscli scenarios list -a
cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing
List only enabled scenarios unless "-a" or names are specified.`,
},
}
}

427
cmd/crowdsec-cli/hubtest.go
View file

@ -7,6 +7,7 @@ import (
"os"
"path/filepath"
"strings"
"text/template"
"github.com/AlecAivazis/survey/v2"
"github.com/enescakir/emoji"
@ -15,52 +16,70 @@ import (
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/dumps"
"github.com/crowdsecurity/crowdsec/pkg/hubtest"
)
var (
HubTest hubtest.HubTest
)
var HubTest hubtest.HubTest
var HubAppsecTests hubtest.HubTest
var hubPtr *hubtest.HubTest
var isAppsecTest bool
func NewHubTestCmd() *cobra.Command {
type cliHubTest struct{}
func NewCLIHubTest() *cliHubTest {
return &cliHubTest{}
}
func (cli cliHubTest) NewCommand() *cobra.Command {
var hubPath string
var crowdsecPath string
var cscliPath string
var cmdHubTest = &cobra.Command{
cmd := &cobra.Command{
Use: "hubtest",
Short: "Run functional tests on hub configurations",
Long: "Run functional tests on hub configurations (parsers, scenarios, collections...)",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
var err error
HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath)
HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, false)
if err != nil {
return fmt.Errorf("unable to load hubtest: %+v", err)
}
HubAppsecTests, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, true)
if err != nil {
return fmt.Errorf("unable to load appsec specific hubtest: %+v", err)
}
/*commands will use the hubPtr, will point to the default hubTest object, or the one dedicated to appsec tests*/
hubPtr = &HubTest
if isAppsecTest {
hubPtr = &HubAppsecTests
}
return nil
},
}
cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
cmdHubTest.AddCommand(NewHubTestCreateCmd())
cmdHubTest.AddCommand(NewHubTestRunCmd())
cmdHubTest.AddCommand(NewHubTestCleanCmd())
cmdHubTest.AddCommand(NewHubTestInfoCmd())
cmdHubTest.AddCommand(NewHubTestListCmd())
cmdHubTest.AddCommand(NewHubTestCoverageCmd())
cmdHubTest.AddCommand(NewHubTestEvalCmd())
cmdHubTest.AddCommand(NewHubTestExplainCmd())
cmd.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
cmd.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
cmd.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
cmd.PersistentFlags().BoolVar(&isAppsecTest, "appsec", false, "Command relates to appsec tests")
return cmdHubTest
cmd.AddCommand(cli.NewCreateCmd())
cmd.AddCommand(cli.NewRunCmd())
cmd.AddCommand(cli.NewCleanCmd())
cmd.AddCommand(cli.NewInfoCmd())
cmd.AddCommand(cli.NewListCmd())
cmd.AddCommand(cli.NewCoverageCmd())
cmd.AddCommand(cli.NewEvalCmd())
cmd.AddCommand(cli.NewExplainCmd())
return cmd
}
func NewHubTestCreateCmd() *cobra.Command {
func (cli cliHubTest) NewCreateCmd() *cobra.Command {
parsers := []string{}
postoverflows := []string{}
scenarios := []string{}
@ -68,7 +87,7 @@ func NewHubTestCreateCmd() *cobra.Command {
var labels map[string]string
var logType string
var cmdHubTestCreate = &cobra.Command{
cmd := &cobra.Command{
Use: "create",
Short: "create [test_name]",
Example: `cscli hubtest create my-awesome-test --type syslog
@ -76,13 +95,17 @@ cscli hubtest create my-nginx-custom-test --type nginx
cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios crowdsecurity/http-probing`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, args []string) error {
testName := args[0]
testPath := filepath.Join(HubTest.HubTestPath, testName)
testPath := filepath.Join(hubPtr.HubTestPath, testName)
if _, err := os.Stat(testPath); os.IsExist(err) {
return fmt.Errorf("test '%s' already exists in '%s', exiting", testName, testPath)
}
if isAppsecTest {
logType = "appsec"
}
if logType == "" {
return fmt.Errorf("please provide a type (--type) for the test")
}
@ -91,54 +114,84 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
return fmt.Errorf("unable to create folder '%s': %+v", testPath, err)
}
// create empty log file
logFileName := fmt.Sprintf("%s.log", testName)
logFilePath := filepath.Join(testPath, logFileName)
logFile, err := os.Create(logFilePath)
if err != nil {
return err
}
logFile.Close()
// create empty parser assertion file
parserAssertFilePath := filepath.Join(testPath, hubtest.ParserAssertFileName)
parserAssertFile, err := os.Create(parserAssertFilePath)
if err != nil {
return err
}
parserAssertFile.Close()
// create empty scenario assertion file
scenarioAssertFilePath := filepath.Join(testPath, hubtest.ScenarioAssertFileName)
scenarioAssertFile, err := os.Create(scenarioAssertFilePath)
if err != nil {
return err
}
scenarioAssertFile.Close()
parsers = append(parsers, "crowdsecurity/syslog-logs")
parsers = append(parsers, "crowdsecurity/dateparse-enrich")
if len(scenarios) == 0 {
scenarios = append(scenarios, "")
}
if len(postoverflows) == 0 {
postoverflows = append(postoverflows, "")
}
configFileData := &hubtest.HubTestItemConfig{
Parsers: parsers,
Scenarios: scenarios,
PostOVerflows: postoverflows,
LogFile: logFileName,
LogType: logType,
IgnoreParsers: ignoreParsers,
Labels: labels,
}
configFilePath := filepath.Join(testPath, "config.yaml")
fd, err := os.OpenFile(configFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
configFileData := &hubtest.HubTestItemConfig{}
if logType == "appsec" {
//create empty nuclei template file
nucleiFileName := fmt.Sprintf("%s.yaml", testName)
nucleiFilePath := filepath.Join(testPath, nucleiFileName)
nucleiFile, err := os.OpenFile(nucleiFilePath, os.O_RDWR|os.O_CREATE, 0755)
if err != nil {
return err
}
ntpl := template.Must(template.New("nuclei").Parse(hubtest.TemplateNucleiFile))
if ntpl == nil {
return fmt.Errorf("unable to parse nuclei template")
}
ntpl.ExecuteTemplate(nucleiFile, "nuclei", struct{ TestName string }{TestName: testName})
nucleiFile.Close()
configFileData.AppsecRules = []string{"./appsec-rules/<author>/your_rule_here.yaml"}
configFileData.NucleiTemplate = nucleiFileName
fmt.Println()
fmt.Printf(" Test name : %s\n", testName)
fmt.Printf(" Test path : %s\n", testPath)
fmt.Printf(" Config File : %s\n", configFilePath)
fmt.Printf(" Nuclei Template : %s\n", nucleiFilePath)
} else {
// create empty log file
logFileName := fmt.Sprintf("%s.log", testName)
logFilePath := filepath.Join(testPath, logFileName)
logFile, err := os.Create(logFilePath)
if err != nil {
return err
}
logFile.Close()
// create empty parser assertion file
parserAssertFilePath := filepath.Join(testPath, hubtest.ParserAssertFileName)
parserAssertFile, err := os.Create(parserAssertFilePath)
if err != nil {
return err
}
parserAssertFile.Close()
// create empty scenario assertion file
scenarioAssertFilePath := filepath.Join(testPath, hubtest.ScenarioAssertFileName)
scenarioAssertFile, err := os.Create(scenarioAssertFilePath)
if err != nil {
return err
}
scenarioAssertFile.Close()
parsers = append(parsers, "crowdsecurity/syslog-logs")
parsers = append(parsers, "crowdsecurity/dateparse-enrich")
if len(scenarios) == 0 {
scenarios = append(scenarios, "")
}
if len(postoverflows) == 0 {
postoverflows = append(postoverflows, "")
}
configFileData.Parsers = parsers
configFileData.Scenarios = scenarios
configFileData.PostOverflows = postoverflows
configFileData.LogFile = logFileName
configFileData.LogType = logType
configFileData.IgnoreParsers = ignoreParsers
configFileData.Labels = labels
fmt.Println()
fmt.Printf(" Test name : %s\n", testName)
fmt.Printf(" Test path : %s\n", testPath)
fmt.Printf(" Log file : %s (please fill it with logs)\n", logFilePath)
fmt.Printf(" Parser assertion file : %s (please fill it with assertion)\n", parserAssertFilePath)
fmt.Printf(" Scenario assertion file : %s (please fill it with assertion)\n", scenarioAssertFilePath)
fmt.Printf(" Configuration File : %s (please fill it with parsers, scenarios...)\n", configFilePath)
}
fd, err := os.Create(configFilePath)
if err != nil {
return fmt.Errorf("open: %s", err)
}
@ -153,56 +206,52 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
if err := fd.Close(); err != nil {
return fmt.Errorf("close: %s", err)
}
fmt.Println()
fmt.Printf(" Test name : %s\n", testName)
fmt.Printf(" Test path : %s\n", testPath)
fmt.Printf(" Log file : %s (please fill it with logs)\n", logFilePath)
fmt.Printf(" Parser assertion file : %s (please fill it with assertion)\n", parserAssertFilePath)
fmt.Printf(" Scenario assertion file : %s (please fill it with assertion)\n", scenarioAssertFilePath)
fmt.Printf(" Configuration File : %s (please fill it with parsers, scenarios...)\n", configFilePath)
return nil
},
}
cmdHubTestCreate.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test")
cmdHubTestCreate.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test")
cmdHubTestCreate.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test")
cmdHubTestCreate.Flags().StringSliceVarP(&scenarios, "scenarios", "s", scenarios, "Scenarios to add to test")
cmdHubTestCreate.PersistentFlags().BoolVar(&ignoreParsers, "ignore-parsers", false, "Don't run test on parsers")
return cmdHubTestCreate
cmd.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test")
cmd.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test")
cmd.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test")
cmd.Flags().StringSliceVarP(&scenarios, "scenarios", "s", scenarios, "Scenarios to add to test")
cmd.PersistentFlags().BoolVar(&ignoreParsers, "ignore-parsers", false, "Don't run test on parsers")
return cmd
}
func NewHubTestRunCmd() *cobra.Command {
func (cli cliHubTest) NewRunCmd() *cobra.Command {
var noClean bool
var runAll bool
var forceClean bool
var cmdHubTestRun = &cobra.Command{
var NucleiTargetHost string
var AppSecHost string
var cmd = &cobra.Command{
Use: "run",
Short: "run [test_name]",
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if !runAll && len(args) == 0 {
printHelp(cmd)
return fmt.Errorf("Please provide test to run or --all flag")
return fmt.Errorf("please provide test to run or --all flag")
}
hubPtr.NucleiTargetHost = NucleiTargetHost
hubPtr.AppSecHost = AppSecHost
if runAll {
if err := HubTest.LoadAllTests(); err != nil {
if err := hubPtr.LoadAllTests(); err != nil {
return fmt.Errorf("unable to load all tests: %+v", err)
}
} else {
for _, testName := range args {
_, err := HubTest.LoadTestItem(testName)
_, err := hubPtr.LoadTestItem(testName)
if err != nil {
return fmt.Errorf("unable to load test '%s': %s", testName, err)
}
}
}
for _, test := range HubTest.Tests {
// set timezone to avoid DST issues
os.Setenv("TZ", "UTC")
for _, test := range hubPtr.Tests {
if csConfig.Cscli.Output == "human" {
log.Infof("Running test '%s'", test.Name)
}
@ -214,11 +263,11 @@ func NewHubTestRunCmd() *cobra.Command {
return nil
},
PersistentPostRunE: func(cmd *cobra.Command, args []string) error {
PersistentPostRunE: func(_ *cobra.Command, _ []string) error {
success := true
testResult := make(map[string]bool)
for _, test := range HubTest.Tests {
if test.AutoGen {
for _, test := range hubPtr.Tests {
if test.AutoGen && !isAppsecTest {
if test.ParserAssert.AutoGenAssert {
log.Warningf("Assert file '%s' is empty, generating assertion:", test.ParserAssert.File)
fmt.Println()
@ -293,9 +342,11 @@ func NewHubTestRunCmd() *cobra.Command {
}
}
}
if csConfig.Cscli.Output == "human" {
switch csConfig.Cscli.Output {
case "human":
hubTestResultTable(color.Output, testResult)
} else if csConfig.Cscli.Output == "json" {
case "json":
jsonResult := make(map[string][]string, 0)
jsonResult["success"] = make([]string, 0)
jsonResult["fail"] = make([]string, 0)
@ -311,6 +362,8 @@ func NewHubTestRunCmd() *cobra.Command {
return fmt.Errorf("unable to json test result: %s", err)
}
fmt.Println(string(jsonStr))
default:
return fmt.Errorf("only human/json output modes are supported")
}
if !success {
@ -320,23 +373,25 @@ func NewHubTestRunCmd() *cobra.Command {
return nil
},
}
cmdHubTestRun.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
cmdHubTestRun.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
cmdHubTestRun.Flags().BoolVar(&runAll, "all", false, "Run all tests")
return cmdHubTestRun
cmd.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
cmd.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
cmd.Flags().StringVar(&NucleiTargetHost, "target", hubtest.DefaultNucleiTarget, "Target for AppSec Test")
cmd.Flags().StringVar(&AppSecHost, "host", hubtest.DefaultAppsecHost, "Address to expose AppSec for hubtest")
cmd.Flags().BoolVar(&runAll, "all", false, "Run all tests")
return cmd
}
func NewHubTestCleanCmd() *cobra.Command {
var cmdHubTestClean = &cobra.Command{
func (cli cliHubTest) NewCleanCmd() *cobra.Command {
var cmd = &cobra.Command{
Use: "clean",
Short: "clean [test_name]",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, args []string) error {
for _, testName := range args {
test, err := HubTest.LoadTestItem(testName)
test, err := hubPtr.LoadTestItem(testName)
if err != nil {
return fmt.Errorf("unable to load test '%s': %s", testName, err)
}
@ -349,28 +404,32 @@ func NewHubTestCleanCmd() *cobra.Command {
},
}
return cmdHubTestClean
return cmd
}
func NewHubTestInfoCmd() *cobra.Command {
var cmdHubTestInfo = &cobra.Command{
func (cli cliHubTest) NewInfoCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "info",
Short: "info [test_name]",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, args []string) error {
for _, testName := range args {
test, err := HubTest.LoadTestItem(testName)
test, err := hubPtr.LoadTestItem(testName)
if err != nil {
return fmt.Errorf("unable to load test '%s': %s", testName, err)
}
fmt.Println()
fmt.Printf(" Test name : %s\n", test.Name)
fmt.Printf(" Test path : %s\n", test.Path)
fmt.Printf(" Log file : %s\n", filepath.Join(test.Path, test.Config.LogFile))
fmt.Printf(" Parser assertion file : %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName))
fmt.Printf(" Scenario assertion file : %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName))
if isAppsecTest {
fmt.Printf(" Nuclei Template : %s\n", test.Config.NucleiTemplate)
fmt.Printf(" Appsec Rules : %s\n", strings.Join(test.Config.AppsecRules, ", "))
} else {
fmt.Printf(" Log file : %s\n", filepath.Join(test.Path, test.Config.LogFile))
fmt.Printf(" Parser assertion file : %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName))
fmt.Printf(" Scenario assertion file : %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName))
}
fmt.Printf(" Configuration File : %s\n", filepath.Join(test.Path, "config.yaml"))
}
@ -378,25 +437,24 @@ func NewHubTestInfoCmd() *cobra.Command {
},
}
return cmdHubTestInfo
return cmd
}
func NewHubTestListCmd() *cobra.Command {
var cmdHubTestList = &cobra.Command{
func (cli cliHubTest) NewListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "list",
Short: "list",
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if err := HubTest.LoadAllTests(); err != nil {
RunE: func(_ *cobra.Command, _ []string) error {
if err := hubPtr.LoadAllTests(); err != nil {
return fmt.Errorf("unable to load all tests: %s", err)
}
switch csConfig.Cscli.Output {
case "human":
hubTestListTable(color.Output, HubTest.Tests)
hubTestListTable(color.Output, hubPtr.Tests)
case "json":
j, err := json.MarshalIndent(HubTest.Tests, " ", " ")
j, err := json.MarshalIndent(hubPtr.Tests, " ", " ")
if err != nil {
return err
}
@ -409,31 +467,34 @@ func NewHubTestListCmd() *cobra.Command {
},
}
return cmdHubTestList
return cmd
}
func NewHubTestCoverageCmd() *cobra.Command {
func (cli cliHubTest) NewCoverageCmd() *cobra.Command {
var showParserCov bool
var showScenarioCov bool
var showOnlyPercent bool
var showAppsecCov bool
var cmdHubTestCoverage = &cobra.Command{
cmd := &cobra.Command{
Use: "coverage",
Short: "coverage",
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
//for this one we explicitly don't do for appsec
if err := HubTest.LoadAllTests(); err != nil {
return fmt.Errorf("unable to load all tests: %+v", err)
}
var err error
scenarioCoverage := []hubtest.ScenarioCoverage{}
parserCoverage := []hubtest.ParserCoverage{}
scenarioCoverage := []hubtest.Coverage{}
parserCoverage := []hubtest.Coverage{}
appsecRuleCoverage := []hubtest.Coverage{}
scenarioCoveragePercent := 0
parserCoveragePercent := 0
appsecRuleCoveragePercent := 0
// if both are false (flag by default), show both
showAll := !showScenarioCov && !showParserCov
showAll := !showScenarioCov && !showParserCov && !showAppsecCov
if showParserCov || showAll {
parserCoverage, err = HubTest.GetParsersCoverage()
@ -443,7 +504,7 @@ func NewHubTestCoverageCmd() *cobra.Command {
parserTested := 0
for _, test := range parserCoverage {
if test.TestsCount > 0 {
parserTested += 1
parserTested++
}
}
parserCoveragePercent = int(math.Round((float64(parserTested) / float64(len(parserCoverage)) * 100)))
@ -454,27 +515,47 @@ func NewHubTestCoverageCmd() *cobra.Command {
if err != nil {
return fmt.Errorf("while getting scenario coverage: %s", err)
}
scenarioTested := 0
for _, test := range scenarioCoverage {
if test.TestsCount > 0 {
scenarioTested += 1
scenarioTested++
}
}
scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100)))
}
if showAppsecCov || showAll {
appsecRuleCoverage, err = HubTest.GetAppsecCoverage()
if err != nil {
return fmt.Errorf("while getting scenario coverage: %s", err)
}
appsecRuleTested := 0
for _, test := range appsecRuleCoverage {
if test.TestsCount > 0 {
appsecRuleTested++
}
}
appsecRuleCoveragePercent = int(math.Round((float64(appsecRuleTested) / float64(len(appsecRuleCoverage)) * 100)))
}
if showOnlyPercent {
if showAll {
fmt.Printf("parsers=%d%%\nscenarios=%d%%", parserCoveragePercent, scenarioCoveragePercent)
fmt.Printf("parsers=%d%%\nscenarios=%d%%\nappsec_rules=%d%%", parserCoveragePercent, scenarioCoveragePercent, appsecRuleCoveragePercent)
} else if showParserCov {
fmt.Printf("parsers=%d%%", parserCoveragePercent)
} else if showScenarioCov {
fmt.Printf("scenarios=%d%%", scenarioCoveragePercent)
} else if showAppsecCov {
fmt.Printf("appsec_rules=%d%%", appsecRuleCoveragePercent)
}
os.Exit(0)
}
if csConfig.Cscli.Output == "human" {
switch csConfig.Cscli.Output {
case "human":
if showParserCov || showAll {
hubTestParserCoverageTable(color.Output, parserCoverage)
}
@ -482,6 +563,11 @@ func NewHubTestCoverageCmd() *cobra.Command {
if showScenarioCov || showAll {
hubTestScenarioCoverageTable(color.Output, scenarioCoverage)
}
if showAppsecCov || showAll {
hubTestAppsecRuleCoverageTable(color.Output, appsecRuleCoverage)
}
fmt.Println()
if showParserCov || showAll {
fmt.Printf("PARSERS : %d%% of coverage\n", parserCoveragePercent)
@ -489,7 +575,10 @@ func NewHubTestCoverageCmd() *cobra.Command {
if showScenarioCov || showAll {
fmt.Printf("SCENARIOS : %d%% of coverage\n", scenarioCoveragePercent)
}
} else if csConfig.Cscli.Output == "json" {
if showAppsecCov || showAll {
fmt.Printf("APPSEC RULES : %d%% of coverage\n", appsecRuleCoveragePercent)
}
case "json":
dump, err := json.MarshalIndent(parserCoverage, "", " ")
if err != nil {
return err
@ -500,61 +589,71 @@ func NewHubTestCoverageCmd() *cobra.Command {
return err
}
fmt.Printf("%s", dump)
} else {
dump, err = json.MarshalIndent(appsecRuleCoverage, "", " ")
if err != nil {
return err
}
fmt.Printf("%s", dump)
default:
return fmt.Errorf("only human/json output modes are supported")
}
return nil
},
}
cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
return cmdHubTestCoverage
cmd.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
cmd.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
cmd.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
cmd.PersistentFlags().BoolVar(&showAppsecCov, "appsec", false, "Show only appsec coverage")
return cmd
}
func NewHubTestEvalCmd() *cobra.Command {
func (cli cliHubTest) NewEvalCmd() *cobra.Command {
var evalExpression string
var cmdHubTestEval = &cobra.Command{
cmd := &cobra.Command{
Use: "eval",
Short: "eval [test_name]",
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, args []string) error {
for _, testName := range args {
test, err := HubTest.LoadTestItem(testName)
test, err := hubPtr.LoadTestItem(testName)
if err != nil {
return fmt.Errorf("can't load test: %+v", err)
}
err = test.ParserAssert.LoadTest(test.ParserResultFile)
if err != nil {
return fmt.Errorf("can't load test results from '%s': %+v", test.ParserResultFile, err)
}
output, err := test.ParserAssert.EvalExpression(evalExpression)
if err != nil {
return err
}
fmt.Print(output)
}
return nil
},
}
cmdHubTestEval.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval")
return cmdHubTestEval
cmd.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval")
return cmd
}
func NewHubTestExplainCmd() *cobra.Command {
var cmdHubTestExplain = &cobra.Command{
func (cli cliHubTest) NewExplainCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "explain",
Short: "explain [test_name]",
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, args []string) error {
for _, testName := range args {
test, err := HubTest.LoadTestItem(testName)
if err != nil {
@ -562,34 +661,32 @@ func NewHubTestExplainCmd() *cobra.Command {
}
err = test.ParserAssert.LoadTest(test.ParserResultFile)
if err != nil {
err := test.Run()
if err != nil {
if err = test.Run(); err != nil {
return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
}
err = test.ParserAssert.LoadTest(test.ParserResultFile)
if err != nil {
if err = test.ParserAssert.LoadTest(test.ParserResultFile); err != nil {
return fmt.Errorf("unable to load parser result after run: %s", err)
}
}
err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
if err != nil {
err := test.Run()
if err != nil {
if err = test.Run(); err != nil {
return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
}
err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
if err != nil {
if err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile); err != nil {
return fmt.Errorf("unable to load scenario result after run: %s", err)
}
}
opts := hubtest.DumpOpts{}
hubtest.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData, opts)
opts := dumps.DumpOpts{}
dumps.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData, opts)
}
return nil
},
}
return cmdHubTestExplain
return cmd
}

30
cmd/crowdsec-cli/hubtest_table.go
View file

@ -41,39 +41,61 @@ func hubTestListTable(out io.Writer, tests []*hubtest.HubTestItem) {
t.Render()
}
func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.ParserCoverage) {
func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
t := newLightTable(out)
t.SetHeaders("Parser", "Status", "Number of tests")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
parserTested := 0
for _, test := range coverage {
status := emoji.RedCircle.String()
if test.TestsCount > 0 {
status = emoji.GreenCircle.String()
parserTested++
}
t.AddRow(test.Parser, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
}
t.Render()
}
func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.ScenarioCoverage) {
func hubTestAppsecRuleCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
t := newLightTable(out)
t.SetHeaders("Appsec Rule", "Status", "Number of tests")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
parserTested := 0
for _, test := range coverage {
status := emoji.RedCircle.String()
if test.TestsCount > 0 {
status = emoji.GreenCircle.String()
parserTested++
}
t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
}
t.Render()
}
func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
t := newLightTable(out)
t.SetHeaders("Scenario", "Status", "Number of tests")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
parserTested := 0
for _, test := range coverage {
status := emoji.RedCircle.String()
if test.TestsCount > 0 {
status = emoji.GreenCircle.String()
parserTested++
}
t.AddRow(test.Scenario, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
}
t.Render()

297
cmd/crowdsec-cli/item_metrics.go Normal file
View file

@ -0,0 +1,297 @@
package main
import (
"fmt"
"math"
"net/http"
"strconv"
"strings"
"time"
"github.com/fatih/color"
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/prom2json"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func ShowMetrics(hubItem *cwhub.Item) error {
switch hubItem.Type {
case cwhub.PARSERS:
metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
parserMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.SCENARIOS:
metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
scenarioMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.COLLECTIONS:
for _, sub := range hubItem.SubItems() {
if err := ShowMetrics(sub); err != nil {
return err
}
}
case cwhub.APPSEC_RULES:
metrics := GetAppsecRuleMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
appsecMetricsTable(color.Output, hubItem.Name, metrics)
default: // no metrics for this item type
}
return nil
}
// GetParserMetric is a complete rip from prom2json
func GetParserMetric(url string, itemName string) map[string]map[string]int {
stats := make(map[string]map[string]int)
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
source, ok := metric.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v", metric.Labels)
} else {
if srctype, ok := metric.Labels["type"]; ok {
source = srctype + ":" + source
}
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_reader_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
stats[source]["parsed"] = 0
stats[source]["reads"] = 0
stats[source]["unparsed"] = 0
stats[source]["hits"] = 0
}
stats[source]["reads"] += ival
case "cs_parser_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_parser_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
case "cs_node_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["hits"] += ival
case "cs_node_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_node_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
default:
continue
}
}
}
return stats
}
func GetScenarioMetric(url string, itemName string) map[string]int {
stats := make(map[string]int)
stats["instantiation"] = 0
stats["curr_count"] = 0
stats["overflow"] = 0
stats["pour"] = 0
stats["underflow"] = 0
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_bucket_created_total":
stats["instantiation"] += ival
case "cs_buckets":
stats["curr_count"] += ival
case "cs_bucket_overflowed_total":
stats["overflow"] += ival
case "cs_bucket_poured_total":
stats["pour"] += ival
case "cs_bucket_underflowed_total":
stats["underflow"] += ival
default:
continue
}
}
}
return stats
}
func GetAppsecRuleMetric(url string, itemName string) map[string]int {
stats := make(map[string]int)
stats["inband_hits"] = 0
stats["outband_hits"] = 0
results := GetPrometheusMetric(url)
for idx, fam := range results {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["rule_name"]
if !ok {
log.Debugf("no rule_name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
band, ok := metric.Labels["type"]
if !ok {
log.Debugf("no type in Metric %v", metric.Labels)
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_appsec_rule_hits":
switch band {
case "inband":
stats["inband_hits"] += ival
case "outband":
stats["outband_hits"] += ival
default:
continue
}
default:
continue
}
}
}
return stats
}
func GetPrometheusMetric(url string) []*prom2json.Family {
mfChan := make(chan *dto.MetricFamily, 1024)
// Start with the DefaultTransport for sane defaults.
transport := http.DefaultTransport.(*http.Transport).Clone()
// Conservatively disable HTTP keep-alives as this program will only
// ever need a single HTTP request.
transport.DisableKeepAlives = true
// Timeout early if the server doesn't even return the headers.
transport.ResponseHeaderTimeout = time.Minute
go func() {
defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
err := prom2json.FetchMetricFamilies(url, mfChan, transport)
if err != nil {
log.Fatalf("failed to fetch prometheus metrics : %v", err)
}
}()
result := []*prom2json.Family{}
for mf := range mfChan {
result = append(result, prom2json.NewFamily(mf))
}
log.Debugf("Finished reading prometheus output, %d entries", len(result))
return result
}
type unit struct {
value int64
symbol string
}
var ranges = []unit{
{value: 1e18, symbol: "E"},
{value: 1e15, symbol: "P"},
{value: 1e12, symbol: "T"},
{value: 1e9, symbol: "G"},
{value: 1e6, symbol: "M"},
{value: 1e3, symbol: "k"},
{value: 1, symbol: ""},
}
func formatNumber(num int) string {
goodUnit := unit{}
for _, u := range ranges {
if int64(num) >= u.value {
goodUnit = u
break
}
}
if goodUnit.value == 1 {
return fmt.Sprintf("%d%s", num, goodUnit.symbol)
}
res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
}

85
cmd/crowdsec-cli/item_suggest.go Normal file
View file

@ -0,0 +1,85 @@
package main
import (
"fmt"
"slices"
"strings"
"github.com/agext/levenshtein"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
// suggestNearestMessage returns a message with the most similar item name, if one is found
func suggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string {
const maxDistance = 7
score := 100
nearest := ""
for _, item := range hub.GetItemMap(itemType) {
d := levenshtein.Distance(itemName, item.Name, nil)
if d < score {
score = d
nearest = item.Name
}
}
msg := fmt.Sprintf("can't find '%s' in %s", itemName, itemType)
if score < maxDistance {
msg += fmt.Sprintf(", did you mean '%s'?", nearest)
}
return msg
}
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
hub, err := require.Hub(csConfig, nil, nil)
if err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
for _, item := range hub.GetItemMap(itemType) {
if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
comp = append(comp, item.Name)
}
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
hub, err := require.Hub(csConfig, nil, nil)
if err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
items, err := hub.GetInstalledItemNames(itemType)
if err != nil {
cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
if toComplete != "" {
for _, item := range items {
if strings.Contains(item, toComplete) {
comp = append(comp, item)
}
}
} else {
comp = items
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}

534
cmd/crowdsec-cli/itemcli.go Normal file
View file

@ -0,0 +1,534 @@
package main
import (
"fmt"
"os"
"strings"
"github.com/fatih/color"
"github.com/hexops/gotextdiff"
"github.com/hexops/gotextdiff/myers"
"github.com/hexops/gotextdiff/span"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/coalesce"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
type cliHelp struct {
// Example is required, the others have a default value
// generated from the item type
use string
short string
long string
example string
}
type cliItem struct {
name string // plural, as used in the hub index
singular string
oneOrMore string // parenthetical pluralizaion: "parser(s)"
help cliHelp
installHelp cliHelp
removeHelp cliHelp
upgradeHelp cliHelp
inspectHelp cliHelp
inspectDetail func(item *cwhub.Item) error
listHelp cliHelp
}
func (cli cliItem) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: coalesce.String(cli.help.use, fmt.Sprintf("%s <action> [item]...", cli.name)),
Short: coalesce.String(cli.help.short, fmt.Sprintf("Manage hub %s", cli.name)),
Long: cli.help.long,
Example: cli.help.example,
Args: cobra.MinimumNArgs(1),
Aliases: []string{cli.singular},
DisableAutoGenTag: true,
}
cmd.AddCommand(cli.newInstallCmd())
cmd.AddCommand(cli.newRemoveCmd())
cmd.AddCommand(cli.newUpgradeCmd())
cmd.AddCommand(cli.newInspectCmd())
cmd.AddCommand(cli.newListCmd())
return cmd
}
func (cli cliItem) install(args []string, downloadOnly bool, force bool, ignoreError bool) error {
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), log.StandardLogger())
if err != nil {
return err
}
for _, name := range args {
item := hub.GetItem(cli.name, name)
if item == nil {
msg := suggestNearestMessage(hub, cli.name, name)
if !ignoreError {
return fmt.Errorf(msg)
}
log.Errorf(msg)
continue
}
if err := item.Install(force, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", item.Name, err)
}
log.Errorf("Error while installing '%s': %s", item.Name, err)
}
}
log.Infof(ReloadMessage())
return nil
}
func (cli cliItem) newInstallCmd() *cobra.Command {
var (
downloadOnly bool
force bool
ignoreError bool
)
cmd := &cobra.Command{
Use: coalesce.String(cli.installHelp.use, "install [item]..."),
Short: coalesce.String(cli.installHelp.short, fmt.Sprintf("Install given %s", cli.oneOrMore)),
Long: coalesce.String(cli.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", cli.name)),
Example: cli.installHelp.example,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cli.name, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
return cli.install(args, downloadOnly, force, ignoreError)
},
}
flags := cmd.Flags()
flags.BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
flags.BoolVar(&force, "force", false, "Force install: overwrite tainted and outdated files")
flags.BoolVar(&ignoreError, "ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", cli.name))
return cmd
}
// return the names of the installed parents of an item, used to check if we can remove it
func istalledParentNames(item *cwhub.Item) []string {
ret := make([]string, 0)
for _, parent := range item.Ancestors() {
if parent.State.Installed {
ret = append(ret, parent.Name)
}
}
return ret
}
func (cli cliItem) remove(args []string, purge bool, force bool, all bool) error {
hub, err := require.Hub(csConfig, nil, log.StandardLogger())
if err != nil {
return err
}
if all {
getter := hub.GetInstalledItems
if purge {
getter = hub.GetAllItems
}
items, err := getter(cli.name)
if err != nil {
return err
}
removed := 0
for _, item := range items {
didRemove, err := item.Remove(purge, force)
if err != nil {
return err
}
if didRemove {
log.Infof("Removed %s", item.Name)
removed++
}
}
log.Infof("Removed %d %s", removed, cli.name)
if removed > 0 {
log.Infof(ReloadMessage())
}
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one %s to remove or '--all'", cli.singular)
}
removed := 0
for _, itemName := range args {
item := hub.GetItem(cli.name, itemName)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", itemName, cli.name)
}
parents := istalledParentNames(item)
if !force && len(parents) > 0 {
log.Warningf("%s belongs to collections: %s", item.Name, parents)
log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, cli.singular)
continue
}
didRemove, err := item.Remove(purge, force)
if err != nil {
return err
}
if didRemove {
log.Infof("Removed %s", item.Name)
removed++
}
}
log.Infof("Removed %d %s", removed, cli.name)
if removed > 0 {
log.Infof(ReloadMessage())
}
return nil
}
func (cli cliItem) newRemoveCmd() *cobra.Command {
var (
purge bool
force bool
all bool
)
cmd := &cobra.Command{
Use: coalesce.String(cli.removeHelp.use, "remove [item]..."),
Short: coalesce.String(cli.removeHelp.short, fmt.Sprintf("Remove given %s", cli.oneOrMore)),
Long: coalesce.String(cli.removeHelp.long, fmt.Sprintf("Remove one or more %s", cli.name)),
Example: cli.removeHelp.example,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cli.name, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
return cli.remove(args, purge, force, all)
},
}
flags := cmd.Flags()
flags.BoolVar(&purge, "purge", false, "Delete source file too")
flags.BoolVar(&force, "force", false, "Force remove: remove tainted and outdated files")
flags.BoolVar(&all, "all", false, fmt.Sprintf("Remove all the %s", cli.name))
return cmd
}
func (cli cliItem) upgrade(args []string, force bool, all bool) error {
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), log.StandardLogger())
if err != nil {
return err
}
if all {
items, err := hub.GetInstalledItems(cli.name)
if err != nil {
return err
}
updated := 0
for _, item := range items {
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
updated++
}
}
log.Infof("Updated %d %s", updated, cli.name)
if updated > 0 {
log.Infof(ReloadMessage())
}
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one %s to upgrade or '--all'", cli.singular)
}
updated := 0
for _, itemName := range args {
item := hub.GetItem(cli.name, itemName)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", itemName, cli.name)
}
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
log.Infof("Updated %s", item.Name)
updated++
}
}
if updated > 0 {
log.Infof(ReloadMessage())
}
return nil
}
func (cli cliItem) newUpgradeCmd() *cobra.Command {
var (
all bool
force bool
)
cmd := &cobra.Command{
Use: coalesce.String(cli.upgradeHelp.use, "upgrade [item]..."),
Short: coalesce.String(cli.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", cli.oneOrMore)),
Long: coalesce.String(cli.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", cli.name)),
Example: cli.upgradeHelp.example,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cli.name, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
return cli.upgrade(args, force, all)
},
}
flags := cmd.Flags()
flags.BoolVarP(&all, "all", "a", false, fmt.Sprintf("Upgrade all the %s", cli.name))
flags.BoolVar(&force, "force", false, "Force upgrade: overwrite tainted and outdated files")
return cmd
}
func (cli cliItem) inspect(args []string, url string, diff bool, rev bool, noMetrics bool) error {
if rev && !diff {
return fmt.Errorf("--rev can only be used with --diff")
}
if url != "" {
csConfig.Cscli.PrometheusUrl = url
}
remote := (*cwhub.RemoteHubCfg)(nil)
if diff {
remote = require.RemoteHub(csConfig)
}
hub, err := require.Hub(csConfig, remote, log.StandardLogger())
if err != nil {
return err
}
for _, name := range args {
item := hub.GetItem(cli.name, name)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", name, cli.name)
}
if diff {
fmt.Println(cli.whyTainted(hub, item, rev))
continue
}
if err = inspectItem(item, !noMetrics); err != nil {
return err
}
if cli.inspectDetail != nil {
if err = cli.inspectDetail(item); err != nil {
return err
}
}
}
return nil
}
func (cli cliItem) newInspectCmd() *cobra.Command {
var (
url string
diff bool
rev bool
noMetrics bool
)
cmd := &cobra.Command{
Use: coalesce.String(cli.inspectHelp.use, "inspect [item]..."),
Short: coalesce.String(cli.inspectHelp.short, fmt.Sprintf("Inspect given %s", cli.oneOrMore)),
Long: coalesce.String(cli.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", cli.name)),
Example: cli.inspectHelp.example,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cli.name, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
return cli.inspect(args, url, diff, rev, noMetrics)
},
}
flags := cmd.Flags()
flags.StringVarP(&url, "url", "u", "", "Prometheus url")
flags.BoolVar(&diff, "diff", false, "Show diff with latest version (for tainted items)")
flags.BoolVar(&rev, "rev", false, "Reverse diff output")
flags.BoolVar(&noMetrics, "no-metrics", false, "Don't show metrics (when cscli.output=human)")
return cmd
}
func (cli cliItem) list(args []string, all bool) error {
hub, err := require.Hub(csConfig, nil, log.StandardLogger())
if err != nil {
return err
}
items := make(map[string][]*cwhub.Item)
items[cli.name], err = selectItems(hub, cli.name, args, !all)
if err != nil {
return err
}
if err = listItems(color.Output, []string{cli.name}, items, false); err != nil {
return err
}
return nil
}
func (cli cliItem) newListCmd() *cobra.Command {
var all bool
cmd := &cobra.Command{
Use: coalesce.String(cli.listHelp.use, "list [item... | -a]"),
Short: coalesce.String(cli.listHelp.short, fmt.Sprintf("List %s", cli.oneOrMore)),
Long: coalesce.String(cli.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", cli.name)),
Example: cli.listHelp.example,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
return cli.list(args, all)
},
}
flags := cmd.Flags()
flags.BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmd
}
// return the diff between the installed version and the latest version
func (cli cliItem) itemDiff(item *cwhub.Item, reverse bool) (string, error) {
if !item.State.Installed {
return "", fmt.Errorf("'%s' is not installed", item.FQName())
}
latestContent, remoteURL, err := item.FetchLatest()
if err != nil {
return "", err
}
localContent, err := os.ReadFile(item.State.LocalPath)
if err != nil {
return "", fmt.Errorf("while reading %s: %w", item.State.LocalPath, err)
}
file1 := item.State.LocalPath
file2 := remoteURL
content1 := string(localContent)
content2 := string(latestContent)
if reverse {
file1, file2 = file2, file1
content1, content2 = content2, content1
}
edits := myers.ComputeEdits(span.URIFromPath(file1), content1, content2)
diff := gotextdiff.ToUnified(file1, file2, content1, edits)
return fmt.Sprintf("%s", diff), nil
}
func (cli cliItem) whyTainted(hub *cwhub.Hub, item *cwhub.Item, reverse bool) string {
if !item.State.Installed {
return fmt.Sprintf("# %s is not installed", item.FQName())
}
if !item.State.Tainted {
return fmt.Sprintf("# %s is not tainted", item.FQName())
}
if len(item.State.TaintedBy) == 0 {
return fmt.Sprintf("# %s is tainted but we don't know why. please report this as a bug", item.FQName())
}
ret := []string{
fmt.Sprintf("# Let's see why %s is tainted.", item.FQName()),
}
for _, fqsub := range item.State.TaintedBy {
ret = append(ret, fmt.Sprintf("\n-> %s\n", fqsub))
sub, err := hub.GetItemFQ(fqsub)
if err != nil {
ret = append(ret, err.Error())
}
diff, err := cli.itemDiff(sub, reverse)
if err != nil {
ret = append(ret, err.Error())
}
if diff != "" {
ret = append(ret, diff)
} else if len(sub.State.TaintedBy) > 0 {
taintList := strings.Join(sub.State.TaintedBy, ", ")
if sub.FQName() == taintList {
// hack: avoid message "item is tainted by itself"
continue
}
ret = append(ret, fmt.Sprintf("# %s is tainted by %s", sub.FQName(), taintList))
}
}
return strings.Join(ret, "\n")
}

183
cmd/crowdsec-cli/items.go Normal file
View file

@ -0,0 +1,183 @@
package main
import (
"encoding/csv"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"slices"
"strings"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
// selectItems returns a slice of items of a given type, selected by name and sorted by case-insensitive name
func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]*cwhub.Item, error) {
itemNames := hub.GetItemNames(itemType)
notExist := []string{}
if len(args) > 0 {
for _, arg := range args {
if !slices.Contains(itemNames, arg) {
notExist = append(notExist, arg)
}
}
}
if len(notExist) > 0 {
return nil, fmt.Errorf("item(s) '%s' not found in %s", strings.Join(notExist, ", "), itemType)
}
if len(args) > 0 {
itemNames = args
installedOnly = false
}
items := make([]*cwhub.Item, 0, len(itemNames))
for _, itemName := range itemNames {
item := hub.GetItem(itemType, itemName)
if installedOnly && !item.State.Installed {
continue
}
items = append(items, item)
}
cwhub.SortItemSlice(items)
return items, nil
}
func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item, omitIfEmpty bool) error {
switch csConfig.Cscli.Output {
case "human":
nothingToDisplay := true
for _, itemType := range itemTypes {
if omitIfEmpty && len(items[itemType]) == 0 {
continue
}
listHubItemTable(out, "\n"+strings.ToUpper(itemType), items[itemType])
nothingToDisplay = false
}
if nothingToDisplay {
fmt.Println("No items to display")
}
case "json":
type itemHubStatus struct {
Name string `json:"name"`
LocalVersion string `json:"local_version"`
LocalPath string `json:"local_path"`
Description string `json:"description"`
UTF8Status string `json:"utf8_status"`
Status string `json:"status"`
}
hubStatus := make(map[string][]itemHubStatus)
for _, itemType := range itemTypes {
// empty slice in case there are no items of this type
hubStatus[itemType] = make([]itemHubStatus, len(items[itemType]))
for i, item := range items[itemType] {
status := item.State.Text()
statusEmo := item.State.Emoji()
hubStatus[itemType][i] = itemHubStatus{
Name: item.Name,
LocalVersion: item.State.LocalVersion,
LocalPath: item.State.LocalPath,
Description: item.Description,
Status: status,
UTF8Status: fmt.Sprintf("%v %s", statusEmo, status),
}
}
}
x, err := json.MarshalIndent(hubStatus, "", " ")
if err != nil {
return fmt.Errorf("failed to unmarshal: %w", err)
}
out.Write(x)
case "raw":
csvwriter := csv.NewWriter(out)
header := []string{"name", "status", "version", "description"}
if len(itemTypes) > 1 {
header = append(header, "type")
}
if err := csvwriter.Write(header); err != nil {
return fmt.Errorf("failed to write header: %s", err)
}
for _, itemType := range itemTypes {
for _, item := range items[itemType] {
row := []string{
item.Name,
item.State.Text(),
item.State.LocalVersion,
item.Description,
}
if len(itemTypes) > 1 {
row = append(row, itemType)
}
if err := csvwriter.Write(row); err != nil {
return fmt.Errorf("failed to write raw output: %s", err)
}
}
}
csvwriter.Flush()
}
return nil
}
func inspectItem(item *cwhub.Item, showMetrics bool) error {
switch csConfig.Cscli.Output {
case "human", "raw":
enc := yaml.NewEncoder(os.Stdout)
enc.SetIndent(2)
if err := enc.Encode(item); err != nil {
return fmt.Errorf("unable to encode item: %s", err)
}
case "json":
b, err := json.MarshalIndent(*item, "", " ")
if err != nil {
return fmt.Errorf("unable to marshal item: %s", err)
}
fmt.Print(string(b))
}
if csConfig.Cscli.Output != "human" {
return nil
}
if item.State.Tainted {
fmt.Println()
fmt.Printf(`This item is tainted. Use "%s %s inspect --diff %s" to see why.`, filepath.Base(os.Args[0]), item.Type, item.Name)
fmt.Println()
}
if showMetrics {
fmt.Printf("\nCurrent metrics: \n")
if err := ShowMetrics(item); err != nil {
return err
}
}
return nil
}

206
cmd/crowdsec-cli/lapi.go
View file

@ -2,10 +2,10 @@ package main
import (
"context"
"errors"
"fmt"
"net/url"
"os"
"slices"
"sort"
"strings"
@ -13,6 +13,7 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"slices"
"github.com/crowdsecurity/go-cs-lib/version"
@ -26,25 +27,24 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/parser"
)
var LAPIURLPrefix string = "v1"
const LAPIURLPrefix = "v1"
func runLapiStatus(cmd *cobra.Command, args []string) error {
var err error
password := strfmt.Password(csConfig.API.Client.Credentials.Password)
apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL)
login := csConfig.API.Client.Credentials.Login
if err != nil {
log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
return fmt.Errorf("parsing api url: %w", err)
}
if err := require.Hub(csConfig); err != nil {
log.Fatal(err)
}
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
hub, err := require.Hub(csConfig, nil, nil)
if err != nil {
log.Fatalf("failed to get scenarios : %s", err)
return err
}
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("failed to get scenarios: %w", err)
}
Client, err = apiclient.NewDefaultClient(apiurl,
@ -52,28 +52,27 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
fmt.Sprintf("crowdsec/%s", version.String()),
nil)
if err != nil {
log.Fatalf("init default client: %s", err)
return fmt.Errorf("init default client: %w", err)
}
t := models.WatcherAuthRequest{
MachineID: &login,
Password: &password,
Scenarios: scenarios,
}
log.Infof("Loaded credentials from %s", csConfig.API.Client.CredentialsFilePath)
log.Infof("Trying to authenticate with username %s on %s", login, apiurl)
_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
if err != nil {
log.Fatalf("Failed to authenticate to Local API (LAPI) : %s", err)
} else {
log.Infof("You can successfully interact with Local API (LAPI)")
return fmt.Errorf("failed to authenticate to Local API (LAPI): %w", err)
}
log.Infof("You can successfully interact with Local API (LAPI)")
return nil
}
func runLapiRegister(cmd *cobra.Command, args []string) error {
var err error
flags := cmd.Flags()
apiURL, err := flags.GetString("url")
@ -94,16 +93,15 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
if lapiUser == "" {
lapiUser, err = generateID("")
if err != nil {
log.Fatalf("unable to generate machine id: %s", err)
return fmt.Errorf("unable to generate machine id: %w", err)
}
}
password := strfmt.Password(generatePassword(passwordLength))
if apiURL == "" {
if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" {
apiURL = csConfig.API.Client.Credentials.URL
} else {
log.Fatalf("No Local API URL. Please provide it in your configuration or with the -u parameter")
if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil || csConfig.API.Client.Credentials.URL == "" {
return fmt.Errorf("no Local API URL. Please provide it in your configuration or with the -u parameter")
}
apiURL = csConfig.API.Client.Credentials.URL
}
/*URL needs to end with /, but user doesn't care*/
if !strings.HasSuffix(apiURL, "/") {
@ -115,7 +113,7 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
}
apiurl, err := url.Parse(apiURL)
if err != nil {
log.Fatalf("parsing api url: %s", err)
return fmt.Errorf("parsing api url: %w", err)
}
_, err = apiclient.RegisterClient(&apiclient.Config{
MachineID: lapiUser,
@ -126,7 +124,7 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
}, nil)
if err != nil {
log.Fatalf("api client register: %s", err)
return fmt.Errorf("api client register: %w", err)
}
log.Printf("Successfully registered to Local API (LAPI)")
@ -146,14 +144,14 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
}
apiConfigDump, err := yaml.Marshal(apiCfg)
if err != nil {
log.Fatalf("unable to marshal api credentials: %s", err)
return fmt.Errorf("unable to marshal api credentials: %w", err)
}
if dumpFile != "" {
err = os.WriteFile(dumpFile, apiConfigDump, 0644)
err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
if err != nil {
log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
return fmt.Errorf("write api credentials to '%s' failed: %w", dumpFile, err)
}
log.Printf("Local API credentials dumped to '%s'", dumpFile)
log.Printf("Local API credentials written to '%s'", dumpFile)
} else {
fmt.Printf("%s\n", string(apiConfigDump))
}
@ -194,7 +192,7 @@ Keep in mind the machine needs to be validated by an administrator on LAPI side
}
func NewLapiCmd() *cobra.Command {
var cmdLapi = &cobra.Command{
cmdLapi := &cobra.Command{
Use: "lapi [action]",
Short: "Manage interaction with Local API (LAPI)",
Args: cobra.MinimumNArgs(1),
@ -220,6 +218,7 @@ func AddContext(key string, values []string) error {
}
if _, ok := csConfig.Crowdsec.ContextToSend[key]; !ok {
csConfig.Crowdsec.ContextToSend[key] = make([]string, 0)
log.Infof("key '%s' added", key)
}
data := csConfig.Crowdsec.ContextToSend[key]
@ -246,11 +245,11 @@ func NewLapiContextCmd() *cobra.Command {
if err := csConfig.LoadCrowdsec(); err != nil {
fileNotFoundMessage := fmt.Sprintf("failed to open context file: open %s: no such file or directory", csConfig.Crowdsec.ConsoleContextPath)
if err.Error() != fileNotFoundMessage {
log.Fatalf("Unable to load CrowdSec Agent: %s", err)
return fmt.Errorf("unable to load CrowdSec agent configuration: %w", err)
}
}
if csConfig.DisableAgent {
log.Fatalf("Agent is disabled and lapi context can only be used on the agent")
return errors.New("agent is disabled and lapi context can only be used on the agent")
}
return nil
@ -270,12 +269,21 @@ cscli lapi context add --key file_source --value evt.Line.Src
cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
hub, err := require.Hub(csConfig, nil, nil)
if err != nil {
return err
}
if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil {
return fmt.Errorf("while loading context: %w", err)
}
if keyToAdd != "" {
if err := AddContext(keyToAdd, valuesToAdd); err != nil {
log.Fatalf(err.Error())
return err
}
return
return nil
}
for _, v := range valuesToAdd {
@ -283,9 +291,11 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
key := keySlice[len(keySlice)-1]
value := []string{v}
if err := AddContext(key, value); err != nil {
log.Fatalf(err.Error())
return err
}
}
return nil
},
}
cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
@ -297,19 +307,29 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
Use: "status",
Short: "List context to send with alerts",
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
hub, err := require.Hub(csConfig, nil, nil)
if err != nil {
return err
}
if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil {
return fmt.Errorf("while loading context: %w", err)
}
if len(csConfig.Crowdsec.ContextToSend) == 0 {
fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.")
return
return nil
}
dump, err := yaml.Marshal(csConfig.Crowdsec.ContextToSend)
if err != nil {
log.Fatalf("unable to show context status: %s", err)
return fmt.Errorf("unable to show context status: %w", err)
}
fmt.Println(string(dump))
fmt.Print(string(dump))
return nil
},
}
cmdContext.AddCommand(cmdContextStatus)
@ -322,30 +342,27 @@ cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
cscli lapi context detect crowdsecurity/sshd-logs
`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
var err error
RunE: func(cmd *cobra.Command, args []string) error {
if !detectAll && len(args) == 0 {
log.Infof("Please provide parsers to detect or --all flag.")
printHelp(cmd)
}
// to avoid all the log.Info from the loaders functions
log.SetLevel(log.ErrorLevel)
log.SetLevel(log.WarnLevel)
err = exprhelpers.Init(nil)
if err := exprhelpers.Init(nil); err != nil {
return fmt.Errorf("failed to init expr helpers: %w", err)
}
hub, err := require.Hub(csConfig, nil, nil)
if err != nil {
log.Fatalf("Failed to init expr helpers : %s", err)
return err
}
// Populate cwhub package tools
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to load hub index : %s", err)
}
csParsers := parser.NewParsers()
csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil {
log.Fatalf("unable to load parsers: %s", err)
return fmt.Errorf("unable to load parsers: %w", err)
}
fieldByParsers := make(map[string][]string)
@ -365,7 +382,6 @@ cscli lapi context detect crowdsecurity/sshd-logs
fieldByParsers[node.Name] = append(fieldByParsers[node.Name], field)
}
}
}
fmt.Printf("Acquisition :\n\n")
@ -398,59 +414,25 @@ cscli lapi context detect crowdsecurity/sshd-logs
log.Errorf("parser '%s' not found, can't detect fields", parserNotFound)
}
}
return nil
},
}
cmdContextDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser")
cmdContext.AddCommand(cmdContextDetect)
var keysToDelete []string
var valuesToDelete []string
cmdContextDelete := &cobra.Command{
Use: "delete",
Short: "Delete context to send with alerts",
Example: `cscli lapi context delete --key source_ip
cscli lapi context delete --value evt.Line.Src
`,
Use: "delete",
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if len(keysToDelete) == 0 && len(valuesToDelete) == 0 {
log.Fatalf("please provide at least a key or a value to delete")
RunE: func(_ *cobra.Command, _ []string) error {
filePath := csConfig.Crowdsec.ConsoleContextPath
if filePath == "" {
filePath = "the context file"
}
for _, key := range keysToDelete {
if _, ok := csConfig.Crowdsec.ContextToSend[key]; ok {
delete(csConfig.Crowdsec.ContextToSend, key)
log.Infof("key '%s' has been removed", key)
} else {
log.Warningf("key '%s' doesn't exist", key)
}
}
for _, value := range valuesToDelete {
valueFound := false
for key, context := range csConfig.Crowdsec.ContextToSend {
if slices.Contains(context, value) {
valueFound = true
csConfig.Crowdsec.ContextToSend[key] = removeFromSlice(value, context)
log.Infof("value '%s' has been removed from key '%s'", value, key)
}
if len(csConfig.Crowdsec.ContextToSend[key]) == 0 {
delete(csConfig.Crowdsec.ContextToSend, key)
}
}
if !valueFound {
log.Warningf("value '%s' not found", value)
}
}
if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
log.Fatalf(err.Error())
}
fmt.Printf("Command \"delete\" is deprecated, please manually edit %s.", filePath)
return nil
},
}
cmdContextDelete.Flags().StringSliceVarP(&keysToDelete, "key", "k", []string{}, "The keys to delete")
cmdContextDelete.Flags().StringSliceVar(&valuesToDelete, "value", []string{}, "The expr fields to delete")
cmdContext.AddCommand(cmdContextDelete)
return cmdContext
@ -458,6 +440,7 @@ cscli lapi context delete --value evt.Line.Src
func detectStaticField(GrokStatics []parser.ExtraField) []string {
ret := make([]string, 0)
for _, static := range GrokStatics {
if static.Parsed != "" {
fieldName := fmt.Sprintf("evt.Parsed.%s", static.Parsed)
@ -486,7 +469,8 @@ func detectStaticField(GrokStatics []parser.ExtraField) []string {
}
func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
var ret = make([]string, 0)
ret := make([]string, 0)
if node.Grok.RunTimeRegexp != nil {
for _, capturedField := range node.Grok.RunTimeRegexp.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
@ -498,13 +482,13 @@ func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
if node.Grok.RegexpName != "" {
grokCompiled, err := parserCTX.Grok.Get(node.Grok.RegexpName)
if err != nil {
log.Warningf("Can't get subgrok: %s", err)
}
for _, capturedField := range grokCompiled.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
if !slices.Contains(ret, fieldName) {
ret = append(ret, fieldName)
// ignore error (parser does not exist?)
if err == nil {
for _, capturedField := range grokCompiled.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
if !slices.Contains(ret, fieldName) {
ret = append(ret, fieldName)
}
}
}
}
@ -544,13 +528,13 @@ func detectSubNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
}
if subnode.Grok.RegexpName != "" {
grokCompiled, err := parserCTX.Grok.Get(subnode.Grok.RegexpName)
if err != nil {
log.Warningf("Can't get subgrok: %s", err)
}
for _, capturedField := range grokCompiled.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
if !slices.Contains(ret, fieldName) {
ret = append(ret, fieldName)
if err == nil {
// ignore error (parser does not exist?)
for _, capturedField := range grokCompiled.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
if !slices.Contains(ret, fieldName) {
ret = append(ret, fieldName)
}
}
}
}

736
cmd/crowdsec-cli/machines.go
View file

@ -5,7 +5,6 @@ import (
"encoding/csv"
"encoding/json"
"fmt"
"io"
"math/big"
"os"
"slices"
@ -18,21 +17,18 @@ import (
"github.com/google/uuid"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/machineid"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/database/ent"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
var (
passwordLength = 64
)
const passwordLength = 64
func generatePassword(length int) string {
upper := "ABCDEFGHIJKLMNOPQRSTUVWXY"
@ -43,11 +39,13 @@ func generatePassword(length int) string {
charsetLength := len(charset)
buf := make([]byte, length)
for i := 0; i < length; i++ {
rInt, err := saferand.Int(saferand.Reader, big.NewInt(int64(charsetLength)))
if err != nil {
log.Fatalf("failed getting data from prng for password generation : %s", err)
}
buf[i] = charset[rInt.Int64()]
}
@ -62,12 +60,14 @@ func generateIDPrefix() (string, error) {
if err == nil {
return prefix, nil
}
log.Debugf("failed to get machine-id with usual files: %s", err)
bId, err := uuid.NewRandom()
bID, err := uuid.NewRandom()
if err == nil {
return bId.String(), nil
return bID.String(), nil
}
return "", fmt.Errorf("generating machine id: %w", err)
}
@ -78,11 +78,14 @@ func generateID(prefix string) (string, error) {
if prefix == "" {
prefix, err = generateIDPrefix()
}
if err != nil {
return "", err
}
prefix = strings.ReplaceAll(prefix, "-", "")[:32]
suffix := generatePassword(16)
return prefix + suffix, nil
}
@ -103,346 +106,19 @@ func getLastHeartbeat(m *ent.Machine) (string, bool) {
return hb, true
}
func getAgents(out io.Writer, dbClient *database.Client) error {
machines, err := dbClient.ListMachines()
if err != nil {
return fmt.Errorf("unable to list machines: %s", err)
}
if csConfig.Cscli.Output == "human" {
getAgentsTable(out, machines)
} else if csConfig.Cscli.Output == "json" {
enc := json.NewEncoder(out)
enc.SetIndent("", " ")
if err := enc.Encode(machines); err != nil {
return fmt.Errorf("failed to marshal")
}
return nil
} else if csConfig.Cscli.Output == "raw" {
csvwriter := csv.NewWriter(out)
err := csvwriter.Write([]string{"machine_id", "ip_address", "updated_at", "validated", "version", "auth_type", "last_heartbeat"})
if err != nil {
return fmt.Errorf("failed to write header: %s", err)
}
for _, m := range machines {
var validated string
if m.IsValidated {
validated = "true"
} else {
validated = "false"
}
hb, _ := getLastHeartbeat(m)
err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb})
if err != nil {
return fmt.Errorf("failed to write raw output: %w", err)
}
}
csvwriter.Flush()
} else {
log.Errorf("unknown output '%s'", csConfig.Cscli.Output)
}
return nil
type cliMachines struct {
db *database.Client
cfg configGetter
}
func NewMachinesListCmd() *cobra.Command {
cmdMachinesList := &cobra.Command{
Use: "list",
Short: "list all machines in the database",
Long: `list all machines in the database with their status and last heartbeat`,
Example: `cscli machines list`,
Args: cobra.NoArgs,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
err := getAgents(color.Output, dbClient)
if err != nil {
return fmt.Errorf("unable to list machines: %s", err)
}
return nil
},
func NewCLIMachines(cfg configGetter) *cliMachines {
return &cliMachines{
cfg: cfg,
}
return cmdMachinesList
}
func NewMachinesAddCmd() *cobra.Command {
cmdMachinesAdd := &cobra.Command{
Use: "add",
Short: "add a single machine to the database",
DisableAutoGenTag: true,
Long: `Register a new machine in the database. cscli should be on the same machine as LAPI.`,
Example: `
cscli machines add --auto
cscli machines add MyTestMachine --auto
cscli machines add MyTestMachine --password MyPassword
`,
RunE: runMachinesAdd,
}
flags := cmdMachinesAdd.Flags()
flags.StringP("password", "p", "", "machine password to login to the API")
flags.StringP("file", "f", "", "output file destination (defaults to "+csconfig.DefaultConfigPath("local_api_credentials.yaml")+")")
flags.StringP("url", "u", "", "URL of the local API")
flags.BoolP("interactive", "i", false, "interfactive mode to enter the password")
flags.BoolP("auto", "a", false, "automatically generate password (and username if not provided)")
flags.Bool("force", false, "will force add the machine if it already exist")
return cmdMachinesAdd
}
func runMachinesAdd(cmd *cobra.Command, args []string) error {
var dumpFile string
var err error
flags := cmd.Flags()
machinePassword, err := flags.GetString("password")
if err != nil {
return err
}
outputFile, err := flags.GetString("file")
if err != nil {
return err
}
apiURL, err := flags.GetString("url")
if err != nil {
return err
}
interactive, err := flags.GetBool("interactive")
if err != nil {
return err
}
autoAdd, err := flags.GetBool("auto")
if err != nil {
return err
}
forceAdd, err := flags.GetBool("force")
if err != nil {
return err
}
var machineID string
// create machineID if not specified by user
if len(args) == 0 {
if !autoAdd {
printHelp(cmd)
return nil
}
machineID, err = generateID("")
if err != nil {
return fmt.Errorf("unable to generate machine id: %s", err)
}
} else {
machineID = args[0]
}
/*check if file already exists*/
if outputFile != "" {
dumpFile = outputFile
} else if csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" {
dumpFile = csConfig.API.Client.CredentialsFilePath
}
// create a password if it's not specified by user
if machinePassword == "" && !interactive {
if !autoAdd {
printHelp(cmd)
return nil
}
machinePassword = generatePassword(passwordLength)
} else if machinePassword == "" && interactive {
qs := &survey.Password{
Message: "Please provide a password for the machine",
}
survey.AskOne(qs, &machinePassword)
}
password := strfmt.Password(machinePassword)
_, err = dbClient.CreateMachine(&machineID, &password, "", true, forceAdd, types.PasswordAuthType)
if err != nil {
return fmt.Errorf("unable to create machine: %s", err)
}
log.Infof("Machine '%s' successfully added to the local API", machineID)
if apiURL == "" {
if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" {
apiURL = csConfig.API.Client.Credentials.URL
} else if csConfig.API.Server != nil && csConfig.API.Server.ListenURI != "" {
apiURL = "http://" + csConfig.API.Server.ListenURI
} else {
return fmt.Errorf("unable to dump an api URL. Please provide it in your configuration or with the -u parameter")
}
}
apiCfg := csconfig.ApiCredentialsCfg{
Login: machineID,
Password: password.String(),
URL: apiURL,
}
apiConfigDump, err := yaml.Marshal(apiCfg)
if err != nil {
return fmt.Errorf("unable to marshal api credentials: %s", err)
}
if dumpFile != "" && dumpFile != "-" {
err = os.WriteFile(dumpFile, apiConfigDump, 0644)
if err != nil {
return fmt.Errorf("write api credentials in '%s' failed: %s", dumpFile, err)
}
log.Printf("API credentials dumped to '%s'", dumpFile)
} else {
fmt.Printf("%s\n", string(apiConfigDump))
}
return nil
}
func NewMachinesDeleteCmd() *cobra.Command {
cmdMachinesDelete := &cobra.Command{
Use: "delete [machine_name]...",
Short: "delete machine(s) by name",
Example: `cscli machines delete "machine1" "machine2"`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"remove"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
machines, err := dbClient.ListMachines()
if err != nil {
cobra.CompError("unable to list machines " + err.Error())
}
ret := make([]string, 0)
for _, machine := range machines {
if strings.Contains(machine.MachineId, toComplete) && !slices.Contains(args, machine.MachineId) {
ret = append(ret, machine.MachineId)
}
}
return ret, cobra.ShellCompDirectiveNoFileComp
},
RunE: runMachinesDelete,
}
return cmdMachinesDelete
}
func runMachinesDelete(cmd *cobra.Command, args []string) error {
for _, machineID := range args {
err := dbClient.DeleteWatcher(machineID)
if err != nil {
log.Errorf("unable to delete machine '%s': %s", machineID, err)
return nil
}
log.Infof("machine '%s' deleted successfully", machineID)
}
return nil
}
func NewMachinesPruneCmd() *cobra.Command {
var parsedDuration time.Duration
cmdMachinesPrune := &cobra.Command{
Use: "prune",
Short: "prune multiple machines from the database",
Long: `prune multiple machines that are not validated or have not connected to the local API in a given duration.`,
Example: `cscli machines prune
cscli machines prune --duration 1h
cscli machines prune --not-validated-only --force`,
Args: cobra.NoArgs,
DisableAutoGenTag: true,
PreRunE: func(cmd *cobra.Command, args []string) error {
dur, _ := cmd.Flags().GetString("duration")
var err error
parsedDuration, err = time.ParseDuration(fmt.Sprintf("-%s", dur))
if err != nil {
return fmt.Errorf("unable to parse duration '%s': %s", dur, err)
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
notValidOnly, _ := cmd.Flags().GetBool("not-validated-only")
force, _ := cmd.Flags().GetBool("force")
if parsedDuration >= 0-60*time.Second && !notValidOnly {
var answer bool
prompt := &survey.Confirm{
Message: "The duration you provided is less than or equal 60 seconds this can break installations do you want to continue ?",
Default: false,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask about prune check: %s", err)
}
if !answer {
fmt.Println("user aborted prune no changes were made")
return nil
}
}
machines := make([]*ent.Machine, 0)
if pending, err := dbClient.QueryPendingMachine(); err == nil {
machines = append(machines, pending...)
}
if !notValidOnly {
if pending, err := dbClient.QueryLastValidatedHeartbeatLT(time.Now().UTC().Add(parsedDuration)); err == nil {
machines = append(machines, pending...)
}
}
if len(machines) == 0 {
fmt.Println("no machines to prune")
return nil
}
getAgentsTable(color.Output, machines)
if !force {
var answer bool
prompt := &survey.Confirm{
Message: "You are about to PERMANENTLY remove the above machines from the database these will NOT be recoverable, continue ?",
Default: false,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask about prune check: %s", err)
}
if !answer {
fmt.Println("user aborted prune no changes were made")
return nil
}
}
nbDeleted, err := dbClient.BulkDeleteWatchers(machines)
if err != nil {
return fmt.Errorf("unable to prune machines: %s", err)
}
fmt.Printf("successfully delete %d machines\n", nbDeleted)
return nil
},
}
cmdMachinesPrune.Flags().StringP("duration", "d", "10m", "duration of time since validated machine last heartbeat")
cmdMachinesPrune.Flags().Bool("not-validated-only", false, "only prune machines that are not validated")
cmdMachinesPrune.Flags().Bool("force", false, "force prune without asking for confirmation")
return cmdMachinesPrune
}
func NewMachinesValidateCmd() *cobra.Command {
cmdMachinesValidate := &cobra.Command{
Use: "validate",
Short: "validate a machine to access the local API",
Long: `validate a machine to access the local API.`,
Example: `cscli machines validate "machine_name"`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
machineID := args[0]
if err := dbClient.ValidateMachine(machineID); err != nil {
return fmt.Errorf("unable to validate machine '%s': %s", machineID, err)
}
log.Infof("machine '%s' validated successfully", machineID)
return nil
},
}
return cmdMachinesValidate
}
func NewMachinesCmd() *cobra.Command {
var cmdMachines = &cobra.Command{
func (cli *cliMachines) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "machines [action]",
Short: "Manage local API machines [requires local API]",
Long: `To list/add/delete/validate/prune machines.
@ -451,24 +127,378 @@ Note: This command requires database direct access, so is intended to be run on
Example: `cscli machines [action]`,
DisableAutoGenTag: true,
Aliases: []string{"machine"},
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
var err error
if err := require.LAPI(csConfig); err != nil {
if err = require.LAPI(cli.cfg()); err != nil {
return err
}
dbClient, err = database.NewClient(csConfig.DbConfig)
cli.db, err = database.NewClient(cli.cfg().DbConfig)
if err != nil {
return fmt.Errorf("unable to create new database client: %s", err)
}
return nil
},
}
cmdMachines.AddCommand(NewMachinesListCmd())
cmdMachines.AddCommand(NewMachinesAddCmd())
cmdMachines.AddCommand(NewMachinesDeleteCmd())
cmdMachines.AddCommand(NewMachinesValidateCmd())
cmdMachines.AddCommand(NewMachinesPruneCmd())
cmd.AddCommand(cli.newListCmd())
cmd.AddCommand(cli.newAddCmd())
cmd.AddCommand(cli.newDeleteCmd())
cmd.AddCommand(cli.newValidateCmd())
cmd.AddCommand(cli.newPruneCmd())
return cmdMachines
return cmd
}
func (cli *cliMachines) list() error {
out := color.Output
machines, err := cli.db.ListMachines()
if err != nil {
return fmt.Errorf("unable to list machines: %s", err)
}
switch cli.cfg().Cscli.Output {
case "human":
getAgentsTable(out, machines)
case "json":
enc := json.NewEncoder(out)
enc.SetIndent("", " ")
if err := enc.Encode(machines); err != nil {
return fmt.Errorf("failed to marshal")
}
return nil
case "raw":
csvwriter := csv.NewWriter(out)
err := csvwriter.Write([]string{"machine_id", "ip_address", "updated_at", "validated", "version", "auth_type", "last_heartbeat"})
if err != nil {
return fmt.Errorf("failed to write header: %s", err)
}
for _, m := range machines {
validated := "false"
if m.IsValidated {
validated = "true"
}
hb, _ := getLastHeartbeat(m)
if err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb}); err != nil {
return fmt.Errorf("failed to write raw output: %w", err)
}
}
csvwriter.Flush()
}
return nil
}
func (cli *cliMachines) newListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "list",
Short: "list all machines in the database",
Long: `list all machines in the database with their status and last heartbeat`,
Example: `cscli machines list`,
Args: cobra.NoArgs,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.list()
},
}
return cmd
}
func (cli *cliMachines) newAddCmd() *cobra.Command {
var (
password MachinePassword
dumpFile string
apiURL string
interactive bool
autoAdd bool
force bool
)
cmd := &cobra.Command{
Use: "add",
Short: "add a single machine to the database",
DisableAutoGenTag: true,
Long: `Register a new machine in the database. cscli should be on the same machine as LAPI.`,
Example: `cscli machines add --auto
cscli machines add MyTestMachine --auto
cscli machines add MyTestMachine --password MyPassword
cscli machines add -f- --auto > /tmp/mycreds.yaml`,
RunE: func(_ *cobra.Command, args []string) error {
return cli.add(args, string(password), dumpFile, apiURL, interactive, autoAdd, force)
},
}
flags := cmd.Flags()
flags.VarP(&password, "password", "p", "machine password to login to the API")
flags.StringVarP(&dumpFile, "file", "f", "", "output file destination (defaults to "+csconfig.DefaultConfigPath("local_api_credentials.yaml")+")")
flags.StringVarP(&apiURL, "url", "u", "", "URL of the local API")
flags.BoolVarP(&interactive, "interactive", "i", false, "interfactive mode to enter the password")
flags.BoolVarP(&autoAdd, "auto", "a", false, "automatically generate password (and username if not provided)")
flags.BoolVar(&force, "force", false, "will force add the machine if it already exist")
return cmd
}
func (cli *cliMachines) add(args []string, machinePassword string, dumpFile string, apiURL string, interactive bool, autoAdd bool, force bool) error {
var (
err error
machineID string
)
// create machineID if not specified by user
if len(args) == 0 {
if !autoAdd {
return fmt.Errorf("please specify a machine name to add, or use --auto")
}
machineID, err = generateID("")
if err != nil {
return fmt.Errorf("unable to generate machine id: %s", err)
}
} else {
machineID = args[0]
}
clientCfg := cli.cfg().API.Client
serverCfg := cli.cfg().API.Server
/*check if file already exists*/
if dumpFile == "" && clientCfg != nil && clientCfg.CredentialsFilePath != "" {
credFile := clientCfg.CredentialsFilePath
// use the default only if the file does not exist
_, err = os.Stat(credFile)
switch {
case os.IsNotExist(err) || force:
dumpFile = credFile
case err != nil:
return fmt.Errorf("unable to stat '%s': %s", credFile, err)
default:
return fmt.Errorf(`credentials file '%s' already exists: please remove it, use "--force" or specify a different file with "-f" ("-f -" for standard output)`, credFile)
}
}
if dumpFile == "" {
return fmt.Errorf(`please specify a file to dump credentials to, with -f ("-f -" for standard output)`)
}
// create a password if it's not specified by user
if machinePassword == "" && !interactive {
if !autoAdd {
return fmt.Errorf("please specify a password with --password or use --auto")
}
machinePassword = generatePassword(passwordLength)
} else if machinePassword == "" && interactive {
qs := &survey.Password{
Message: "Please provide a password for the machine:",
}
survey.AskOne(qs, &machinePassword)
}
password := strfmt.Password(machinePassword)
_, err = cli.db.CreateMachine(&machineID, &password, "", true, force, types.PasswordAuthType)
if err != nil {
return fmt.Errorf("unable to create machine: %s", err)
}
fmt.Fprintf(os.Stderr, "Machine '%s' successfully added to the local API.\n", machineID)
if apiURL == "" {
if clientCfg != nil && clientCfg.Credentials != nil && clientCfg.Credentials.URL != "" {
apiURL = clientCfg.Credentials.URL
} else if serverCfg != nil && serverCfg.ListenURI != "" {
apiURL = "http://" + serverCfg.ListenURI
} else {
return fmt.Errorf("unable to dump an api URL. Please provide it in your configuration or with the -u parameter")
}
}
apiCfg := csconfig.ApiCredentialsCfg{
Login: machineID,
Password: password.String(),
URL: apiURL,
}
apiConfigDump, err := yaml.Marshal(apiCfg)
if err != nil {
return fmt.Errorf("unable to marshal api credentials: %s", err)
}
if dumpFile != "" && dumpFile != "-" {
if err = os.WriteFile(dumpFile, apiConfigDump, 0o600); err != nil {
return fmt.Errorf("write api credentials in '%s' failed: %s", dumpFile, err)
}
fmt.Fprintf(os.Stderr, "API credentials written to '%s'.\n", dumpFile)
} else {
fmt.Print(string(apiConfigDump))
}
return nil
}
func (cli *cliMachines) deleteValid(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
machines, err := cli.db.ListMachines()
if err != nil {
cobra.CompError("unable to list machines " + err.Error())
}
ret := []string{}
for _, machine := range machines {
if strings.Contains(machine.MachineId, toComplete) && !slices.Contains(args, machine.MachineId) {
ret = append(ret, machine.MachineId)
}
}
return ret, cobra.ShellCompDirectiveNoFileComp
}
func (cli *cliMachines) delete(machines []string) error {
for _, machineID := range machines {
if err := cli.db.DeleteWatcher(machineID); err != nil {
log.Errorf("unable to delete machine '%s': %s", machineID, err)
return nil
}
log.Infof("machine '%s' deleted successfully", machineID)
}
return nil
}
func (cli *cliMachines) newDeleteCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "delete [machine_name]...",
Short: "delete machine(s) by name",
Example: `cscli machines delete "machine1" "machine2"`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"remove"},
DisableAutoGenTag: true,
ValidArgsFunction: cli.deleteValid,
RunE: func(_ *cobra.Command, args []string) error {
return cli.delete(args)
},
}
return cmd
}
func (cli *cliMachines) prune(duration time.Duration, notValidOnly bool, force bool) error {
if duration < 2*time.Minute && !notValidOnly {
if yes, err := askYesNo(
"The duration you provided is less than 2 minutes. " +
"This can break installations if the machines are only temporarily disconnected. Continue?", false); err != nil {
return err
} else if !yes {
fmt.Println("User aborted prune. No changes were made.")
return nil
}
}
machines := []*ent.Machine{}
if pending, err := cli.db.QueryPendingMachine(); err == nil {
machines = append(machines, pending...)
}
if !notValidOnly {
if pending, err := cli.db.QueryLastValidatedHeartbeatLT(time.Now().UTC().Add(duration)); err == nil {
machines = append(machines, pending...)
}
}
if len(machines) == 0 {
fmt.Println("no machines to prune")
return nil
}
getAgentsTable(color.Output, machines)
if !force {
if yes, err := askYesNo(
"You are about to PERMANENTLY remove the above machines from the database. " +
"These will NOT be recoverable. Continue?", false); err != nil {
return err
} else if !yes {
fmt.Println("User aborted prune. No changes were made.")
return nil
}
}
deleted, err := cli.db.BulkDeleteWatchers(machines)
if err != nil {
return fmt.Errorf("unable to prune machines: %s", err)
}
fmt.Fprintf(os.Stderr, "successfully delete %d machines\n", deleted)
return nil
}
func (cli *cliMachines) newPruneCmd() *cobra.Command {
var (
duration time.Duration
notValidOnly bool
force bool
)
const defaultDuration = 10 * time.Minute
cmd := &cobra.Command{
Use: "prune",
Short: "prune multiple machines from the database",
Long: `prune multiple machines that are not validated or have not connected to the local API in a given duration.`,
Example: `cscli machines prune
cscli machines prune --duration 1h
cscli machines prune --not-validated-only --force`,
Args: cobra.NoArgs,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.prune(duration, notValidOnly, force)
},
}
flags := cmd.Flags()
flags.DurationVarP(&duration, "duration", "d", defaultDuration, "duration of time since validated machine last heartbeat")
flags.BoolVar(&notValidOnly, "not-validated-only", false, "only prune machines that are not validated")
flags.BoolVar(&force, "force", false, "force prune without asking for confirmation")
return cmd
}
func (cli *cliMachines) validate(machineID string) error {
if err := cli.db.ValidateMachine(machineID); err != nil {
return fmt.Errorf("unable to validate machine '%s': %s", machineID, err)
}
log.Infof("machine '%s' validated successfully", machineID)
return nil
}
func (cli *cliMachines) newValidateCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "validate",
Short: "validate a machine to access the local API",
Long: `validate a machine to access the local API.`,
Example: `cscli machines validate "machine_name"`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
return cli.validate(args[0])
},
}
return cmd
}

318
cmd/crowdsec-cli/main.go
View file

@ -1,68 +1,102 @@
package main
import (
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"time"
"github.com/fatih/color"
cc "github.com/ivanpirog/coloredcobra"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/cobra/doc"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
)
var trace_lvl, dbg_lvl, nfo_lvl, wrn_lvl, err_lvl bool
var ConfigFilePath string
var csConfig *csconfig.Config
var dbClient *database.Client
var OutputFormat string
var OutputColor string
var downloadOnly bool
var forceAction bool
var purge bool
var all bool
var prometheusURL string
type configGetter func() *csconfig.Config
var mergedConfig string
func initConfig() {
var err error
if trace_lvl {
log.SetLevel(log.TraceLevel)
} else if dbg_lvl {
log.SetLevel(log.DebugLevel)
} else if nfo_lvl {
log.SetLevel(log.InfoLevel)
} else if wrn_lvl {
log.SetLevel(log.WarnLevel)
} else if err_lvl {
log.SetLevel(log.ErrorLevel)
type cliRoot struct {
logTrace bool
logDebug bool
logInfo bool
logWarn bool
logErr bool
outputColor string
outputFormat string
// flagBranch overrides the value in csConfig.Cscli.HubBranch
flagBranch string
}
func newCliRoot() *cliRoot {
return &cliRoot{}
}
// cfg() is a helper function to get the configuration loaded from config.yaml,
// we pass it to subcommands because the file is not read until the Execute() call
func (cli *cliRoot) cfg() *csconfig.Config {
return csConfig
}
// wantedLogLevel returns the log level requested in the command line flags.
func (cli *cliRoot) wantedLogLevel() log.Level {
switch {
case cli.logTrace:
return log.TraceLevel
case cli.logDebug:
return log.DebugLevel
case cli.logInfo:
return log.InfoLevel
case cli.logWarn:
return log.WarnLevel
case cli.logErr:
return log.ErrorLevel
default:
return log.InfoLevel
}
}
// loadConfigFor loads the configuration file for the given sub-command.
// If the sub-command does not need it, it returns a default configuration.
func loadConfigFor(command string) (*csconfig.Config, string, error) {
noNeedConfig := []string{
"doc",
"help",
"completion",
"version",
"hubtest",
}
if !slices.Contains(NoNeedConfig, os.Args[1]) {
if !slices.Contains(noNeedConfig, command) {
log.Debugf("Using %s as configuration file", ConfigFilePath)
csConfig, mergedConfig, err = csconfig.NewConfig(ConfigFilePath, false, false, true)
config, merged, err := csconfig.NewConfig(ConfigFilePath, false, false, true)
if err != nil {
log.Fatal(err)
return nil, "", err
}
if err := csConfig.LoadCSCLI(); err != nil {
log.Fatal(err)
}
} else {
csConfig = csconfig.NewDefaultConfig()
return config, merged, nil
}
return csconfig.NewDefaultConfig(), "", nil
}
// initialize is called before the subcommand is executed.
func (cli *cliRoot) initialize() {
var err error
log.SetLevel(cli.wantedLogLevel())
csConfig, mergedConfig, err = loadConfigFor(os.Args[1])
if err != nil {
log.Fatal(err)
}
// recap of the enabled feature flags, because logging
@ -71,22 +105,22 @@ func initConfig() {
log.Debugf("Enabled feature flags: %s", fflist)
}
if csConfig.Cscli == nil {
log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath)
if cli.flagBranch != "" {
csConfig.Cscli.HubBranch = cli.flagBranch
}
if cwhub.HubBranch == "" && csConfig.Cscli.HubBranch != "" {
cwhub.HubBranch = csConfig.Cscli.HubBranch
}
if OutputFormat != "" {
csConfig.Cscli.Output = OutputFormat
if OutputFormat != "json" && OutputFormat != "raw" && OutputFormat != "human" {
log.Fatalf("output format %s unknown", OutputFormat)
}
if cli.outputFormat != "" {
csConfig.Cscli.Output = cli.outputFormat
}
if csConfig.Cscli.Output == "" {
csConfig.Cscli.Output = "human"
}
if csConfig.Cscli.Output != "human" && csConfig.Cscli.Output != "json" && csConfig.Cscli.Output != "raw" {
log.Fatalf("output format '%s' not supported: must be one of human, json, raw", csConfig.Cscli.Output)
}
if csConfig.Cscli.Output == "json" {
log.SetFormatter(&log.JSONFormatter{})
log.SetLevel(log.ErrorLevel)
@ -94,47 +128,44 @@ func initConfig() {
log.SetLevel(log.ErrorLevel)
}
if OutputColor != "" {
csConfig.Cscli.Color = OutputColor
if OutputColor != "yes" && OutputColor != "no" && OutputColor != "auto" {
log.Fatalf("output color %s unknown", OutputColor)
if cli.outputColor != "" {
csConfig.Cscli.Color = cli.outputColor
if cli.outputColor != "yes" && cli.outputColor != "no" && cli.outputColor != "auto" {
log.Fatalf("output color %s unknown", cli.outputColor)
}
}
}
// list of valid subcommands for the shell completion
var validArgs = []string{
"scenarios", "parsers", "collections", "capi", "lapi", "postoverflows", "machines",
"metrics", "bouncers", "alerts", "decisions", "simulation", "hub", "dashboard",
"config", "completion", "version", "console", "notifications", "support",
"alerts", "appsec-configs", "appsec-rules", "bouncers", "capi", "collections",
"completion", "config", "console", "contexts", "dashboard", "decisions", "explain",
"hub", "hubtest", "lapi", "machines", "metrics", "notifications", "parsers",
"postoverflows", "scenarios", "simulation", "support", "version",
}
func prepender(filename string) string {
const header = `---
id: %s
title: %s
---
`
name := filepath.Base(filename)
base := strings.TrimSuffix(name, filepath.Ext(name))
return fmt.Sprintf(header, base, strings.ReplaceAll(base, "_", " "))
func (cli *cliRoot) colorize(cmd *cobra.Command) {
cc.Init(&cc.Config{
RootCmd: cmd,
Headings: cc.Yellow,
Commands: cc.Green + cc.Bold,
CmdShortDescr: cc.Cyan,
Example: cc.Italic,
ExecName: cc.Bold,
Aliases: cc.Bold + cc.Italic,
FlagsDataType: cc.White,
Flags: cc.Green,
FlagsDescr: cc.Cyan,
NoExtraNewlines: true,
NoBottomNewline: true,
})
cmd.SetOut(color.Output)
}
func linkHandler(name string) string {
return fmt.Sprintf("/cscli/%s", name)
}
var (
NoNeedConfig = []string{
"help",
"completion",
"version",
"hubtest",
}
)
func main() {
func (cli *cliRoot) NewCommand() *cobra.Command {
// set the formatter asap and worry about level later
logFormatter := &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}
logFormatter := &log.TextFormatter{TimestampFormat: time.RFC3339, FullTimestamp: true}
log.SetFormatter(logFormatter)
if err := fflag.RegisterAllFeatures(); err != nil {
@ -145,7 +176,7 @@ func main() {
log.Fatalf("failed to set feature flags from env: %s", err)
}
var rootCmd = &cobra.Command{
cmd := &cobra.Command{
Use: "cscli",
Short: "cscli allows you to manage crowdsec",
Long: `cscli is the main command to interact with your crowdsec service, scenarios & db.
@ -157,57 +188,25 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
/*TBD examples*/
}
cc.Init(&cc.Config{
RootCmd: rootCmd,
Headings: cc.Yellow,
Commands: cc.Green + cc.Bold,
CmdShortDescr: cc.Cyan,
Example: cc.Italic,
ExecName: cc.Bold,
Aliases: cc.Bold + cc.Italic,
FlagsDataType: cc.White,
Flags: cc.Green,
FlagsDescr: cc.Cyan,
})
rootCmd.SetOut(color.Output)
cli.colorize(cmd)
var cmdDocGen = &cobra.Command{
Use: "doc",
Short: "Generate the documentation in `./doc/`. Directory must exist.",
Args: cobra.ExactArgs(0),
Hidden: true,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if err := doc.GenMarkdownTreeCustom(rootCmd, "./doc/", prepender, linkHandler); err != nil {
return fmt.Errorf("Failed to generate cobra doc: %s", err)
}
return nil
},
}
rootCmd.AddCommand(cmdDocGen)
/*usage*/
var cmdVersion = &cobra.Command{
Use: "version",
Short: "Display version",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
cwversion.Show()
},
}
rootCmd.AddCommand(cmdVersion)
/*don't sort flags so we can enforce order*/
cmd.Flags().SortFlags = false
rootCmd.PersistentFlags().StringVarP(&ConfigFilePath, "config", "c", csconfig.DefaultConfigPath("config.yaml"), "path to crowdsec config file")
rootCmd.PersistentFlags().StringVarP(&OutputFormat, "output", "o", "", "Output format: human, json, raw")
rootCmd.PersistentFlags().StringVarP(&OutputColor, "color", "", "auto", "Output color: yes, no, auto")
rootCmd.PersistentFlags().BoolVar(&dbg_lvl, "debug", false, "Set logging to debug")
rootCmd.PersistentFlags().BoolVar(&nfo_lvl, "info", false, "Set logging to info")
rootCmd.PersistentFlags().BoolVar(&wrn_lvl, "warning", false, "Set logging to warning")
rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error")
rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace")
pflags := cmd.PersistentFlags()
pflags.SortFlags = false
rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "", "Override hub branch on github")
if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil {
pflags.StringVarP(&ConfigFilePath, "config", "c", csconfig.DefaultConfigPath("config.yaml"), "path to crowdsec config file")
pflags.StringVarP(&cli.outputFormat, "output", "o", "", "Output format: human, json, raw")
pflags.StringVarP(&cli.outputColor, "color", "", "auto", "Output color: yes, no, auto")
pflags.BoolVar(&cli.logDebug, "debug", false, "Set logging to debug")
pflags.BoolVar(&cli.logInfo, "info", false, "Set logging to info")
pflags.BoolVar(&cli.logWarn, "warning", false, "Set logging to warning")
pflags.BoolVar(&cli.logErr, "error", false, "Set logging to error")
pflags.BoolVar(&cli.logTrace, "trace", false, "Set logging to trace")
pflags.StringVar(&cli.flagBranch, "branch", "", "Override hub branch on github")
if err := pflags.MarkHidden("branch"); err != nil {
log.Fatalf("failed to hide flag: %s", err)
}
@ -227,44 +226,47 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
}
if len(os.Args) > 1 {
cobra.OnInitialize(initConfig)
cobra.OnInitialize(cli.initialize)
}
/*don't sort flags so we can enforce order*/
rootCmd.Flags().SortFlags = false
rootCmd.PersistentFlags().SortFlags = false
rootCmd.AddCommand(NewConfigCmd())
rootCmd.AddCommand(NewHubCmd())
rootCmd.AddCommand(NewMetricsCmd())
rootCmd.AddCommand(NewDashboardCmd())
rootCmd.AddCommand(NewDecisionsCmd())
rootCmd.AddCommand(NewAlertsCmd())
rootCmd.AddCommand(NewSimulationCmds())
rootCmd.AddCommand(NewBouncersCmd())
rootCmd.AddCommand(NewMachinesCmd())
rootCmd.AddCommand(NewParsersCmd())
rootCmd.AddCommand(NewScenariosCmd())
rootCmd.AddCommand(NewCollectionsCmd())
rootCmd.AddCommand(NewPostOverflowsCmd())
rootCmd.AddCommand(NewCapiCmd())
rootCmd.AddCommand(NewLapiCmd())
rootCmd.AddCommand(NewCompletionCmd())
rootCmd.AddCommand(NewConsoleCmd())
rootCmd.AddCommand(NewExplainCmd())
rootCmd.AddCommand(NewHubTestCmd())
rootCmd.AddCommand(NewNotificationsCmd())
rootCmd.AddCommand(NewSupportCmd())
cmd.AddCommand(NewCLIDoc().NewCommand(cmd))
cmd.AddCommand(NewCLIVersion().NewCommand())
cmd.AddCommand(NewConfigCmd())
cmd.AddCommand(NewCLIHub(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIMetrics(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIDashboard(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIDecisions(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIAlerts().NewCommand())
cmd.AddCommand(NewCLISimulation(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIBouncers(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIMachines(cli.cfg).NewCommand())
cmd.AddCommand(NewCLICapi().NewCommand())
cmd.AddCommand(NewLapiCmd())
cmd.AddCommand(NewCompletionCmd())
cmd.AddCommand(NewConsoleCmd())
cmd.AddCommand(NewCLIExplain().NewCommand())
cmd.AddCommand(NewCLIHubTest().NewCommand())
cmd.AddCommand(NewCLINotifications().NewCommand())
cmd.AddCommand(NewCLISupport().NewCommand())
cmd.AddCommand(NewCLIPapi(cli.cfg).NewCommand())
cmd.AddCommand(NewCLICollection().NewCommand())
cmd.AddCommand(NewCLIParser().NewCommand())
cmd.AddCommand(NewCLIScenario().NewCommand())
cmd.AddCommand(NewCLIPostOverflow().NewCommand())
cmd.AddCommand(NewCLIContext().NewCommand())
cmd.AddCommand(NewCLIAppsecConfig().NewCommand())
cmd.AddCommand(NewCLIAppsecRule().NewCommand())
if fflag.CscliSetup.IsEnabled() {
rootCmd.AddCommand(NewSetupCmd())
cmd.AddCommand(NewSetupCmd())
}
if fflag.PapiClient.IsEnabled() {
rootCmd.AddCommand(NewPapiCmd())
}
return cmd
}
if err := rootCmd.Execute(); err != nil {
func main() {
cmd := newCliRoot().NewCommand()
if err := cmd.Execute(); err != nil {
log.Fatal(err)
}
}

507
cmd/crowdsec-cli/metrics.go
View file

@ -2,6 +2,7 @@ package main
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
@ -16,11 +17,64 @@ import (
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/go-cs-lib/maptools"
"github.com/crowdsecurity/go-cs-lib/trace"
)
// FormatPrometheusMetrics is a complete rip from prom2json
func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error {
type (
statAcquis map[string]map[string]int
statParser map[string]map[string]int
statBucket map[string]map[string]int
statWhitelist map[string]map[string]map[string]int
statLapi map[string]map[string]int
statLapiMachine map[string]map[string]map[string]int
statLapiBouncer map[string]map[string]map[string]int
statLapiDecision map[string]struct {
NonEmpty int
Empty int
}
statDecision map[string]map[string]map[string]int
statAppsecEngine map[string]map[string]int
statAppsecRule map[string]map[string]map[string]int
statAlert map[string]int
statStash map[string]struct {
Type string
Count int
}
)
var (
ErrMissingConfig = errors.New("prometheus section missing, can't show metrics")
ErrMetricsDisabled = errors.New("prometheus is not enabled, can't show metrics")
)
type metricSection interface {
Table(out io.Writer, noUnit bool, showEmpty bool)
Description() (string, string)
}
type metricStore map[string]metricSection
func NewMetricStore() metricStore {
return metricStore{
"acquisition": statAcquis{},
"buckets": statBucket{},
"parsers": statParser{},
"lapi": statLapi{},
"lapi-machine": statLapiMachine{},
"lapi-bouncer": statLapiBouncer{},
"lapi-decisions": statLapiDecision{},
"decisions": statDecision{},
"alerts": statAlert{},
"stash": statStash{},
"appsec-engine": statAppsecEngine{},
"appsec-rule": statAppsecRule{},
"whitelists": statWhitelist{},
}
}
func (ms metricStore) Fetch(url string) error {
mfChan := make(chan *dto.MetricFamily, 1024)
errChan := make(chan error, 1)
@ -33,9 +87,10 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
transport.ResponseHeaderTimeout = time.Minute
go func() {
defer trace.CatchPanic("crowdsec/ShowPrometheus")
err := prom2json.FetchMetricFamilies(url, mfChan, transport)
if err != nil {
errChan <- fmt.Errorf("failed to fetch prometheus metrics: %w", err)
errChan <- fmt.Errorf("failed to fetch metrics: %w", err)
return
}
errChan <- nil
@ -50,40 +105,42 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
return err
}
log.Debugf("Finished reading prometheus output, %d entries", len(result))
log.Debugf("Finished reading metrics output, %d entries", len(result))
/*walk*/
lapi_decisions_stats := map[string]struct {
NonEmpty int
Empty int
}{}
acquis_stats := map[string]map[string]int{}
parsers_stats := map[string]map[string]int{}
buckets_stats := map[string]map[string]int{}
lapi_stats := map[string]map[string]int{}
lapi_machine_stats := map[string]map[string]map[string]int{}
lapi_bouncer_stats := map[string]map[string]map[string]int{}
decisions_stats := map[string]map[string]map[string]int{}
alerts_stats := map[string]int{}
stash_stats := map[string]struct {
Type string
Count int
}{}
mAcquis := ms["acquisition"].(statAcquis)
mParser := ms["parsers"].(statParser)
mBucket := ms["buckets"].(statBucket)
mLapi := ms["lapi"].(statLapi)
mLapiMachine := ms["lapi-machine"].(statLapiMachine)
mLapiBouncer := ms["lapi-bouncer"].(statLapiBouncer)
mLapiDecision := ms["lapi-decisions"].(statLapiDecision)
mDecision := ms["decisions"].(statDecision)
mAppsecEngine := ms["appsec-engine"].(statAppsecEngine)
mAppsecRule := ms["appsec-rule"].(statAppsecRule)
mAlert := ms["alerts"].(statAlert)
mStash := ms["stash"].(statStash)
mWhitelist := ms["whitelists"].(statWhitelist)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
source, ok := metric.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v for %s", metric.Labels, fam.Name)
@ -104,173 +161,142 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
origin := metric.Labels["origin"]
action := metric.Labels["action"]
appsecEngine := metric.Labels["appsec_engine"]
appsecRule := metric.Labels["rule_name"]
mtype := metric.Labels["type"]
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
}
ival := int(fval)
switch fam.Name {
/*buckets*/
//
// buckets
//
case "cs_bucket_created_total":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
buckets_stats[name]["instantiation"] += ival
mBucket.Process(name, "instantiation", ival)
case "cs_buckets":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
buckets_stats[name]["curr_count"] += ival
mBucket.Process(name, "curr_count", ival)
case "cs_bucket_overflowed_total":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
buckets_stats[name]["overflow"] += ival
mBucket.Process(name, "overflow", ival)
case "cs_bucket_poured_total":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
}
buckets_stats[name]["pour"] += ival
acquis_stats[source]["pour"] += ival
mBucket.Process(name, "pour", ival)
mAcquis.Process(source, "pour", ival)
case "cs_bucket_underflowed_total":
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
buckets_stats[name]["underflow"] += ival
/*acquis*/
mBucket.Process(name, "underflow", ival)
//
// parsers
//
case "cs_parser_hits_total":
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
}
acquis_stats[source]["reads"] += ival
mAcquis.Process(source, "reads", ival)
case "cs_parser_hits_ok_total":
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
}
acquis_stats[source]["parsed"] += ival
mAcquis.Process(source, "parsed", ival)
case "cs_parser_hits_ko_total":
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
}
acquis_stats[source]["unparsed"] += ival
mAcquis.Process(source, "unparsed", ival)
case "cs_node_hits_total":
if _, ok := parsers_stats[name]; !ok {
parsers_stats[name] = make(map[string]int)
}
parsers_stats[name]["hits"] += ival
mParser.Process(name, "hits", ival)
case "cs_node_hits_ok_total":
if _, ok := parsers_stats[name]; !ok {
parsers_stats[name] = make(map[string]int)
}
parsers_stats[name]["parsed"] += ival
mParser.Process(name, "parsed", ival)
case "cs_node_hits_ko_total":
if _, ok := parsers_stats[name]; !ok {
parsers_stats[name] = make(map[string]int)
}
parsers_stats[name]["unparsed"] += ival
mParser.Process(name, "unparsed", ival)
//
// whitelists
//
case "cs_node_wl_hits_total":
mWhitelist.Process(name, reason, "hits", ival)
case "cs_node_wl_hits_ok_total":
mWhitelist.Process(name, reason, "whitelisted", ival)
// track as well whitelisted lines at acquis level
mAcquis.Process(source, "whitelisted", ival)
//
// lapi
//
case "cs_lapi_route_requests_total":
if _, ok := lapi_stats[route]; !ok {
lapi_stats[route] = make(map[string]int)
}
lapi_stats[route][method] += ival
mLapi.Process(route, method, ival)
case "cs_lapi_machine_requests_total":
if _, ok := lapi_machine_stats[machine]; !ok {
lapi_machine_stats[machine] = make(map[string]map[string]int)
}
if _, ok := lapi_machine_stats[machine][route]; !ok {
lapi_machine_stats[machine][route] = make(map[string]int)
}
lapi_machine_stats[machine][route][method] += ival
mLapiMachine.Process(machine, route, method, ival)
case "cs_lapi_bouncer_requests_total":
if _, ok := lapi_bouncer_stats[bouncer]; !ok {
lapi_bouncer_stats[bouncer] = make(map[string]map[string]int)
}
if _, ok := lapi_bouncer_stats[bouncer][route]; !ok {
lapi_bouncer_stats[bouncer][route] = make(map[string]int)
}
lapi_bouncer_stats[bouncer][route][method] += ival
mLapiBouncer.Process(bouncer, route, method, ival)
case "cs_lapi_decisions_ko_total", "cs_lapi_decisions_ok_total":
if _, ok := lapi_decisions_stats[bouncer]; !ok {
lapi_decisions_stats[bouncer] = struct {
NonEmpty int
Empty int
}{}
}
x := lapi_decisions_stats[bouncer]
if fam.Name == "cs_lapi_decisions_ko_total" {
x.Empty += ival
} else if fam.Name == "cs_lapi_decisions_ok_total" {
x.NonEmpty += ival
}
lapi_decisions_stats[bouncer] = x
mLapiDecision.Process(bouncer, fam.Name, ival)
//
// decisions
//
case "cs_active_decisions":
if _, ok := decisions_stats[reason]; !ok {
decisions_stats[reason] = make(map[string]map[string]int)
}
if _, ok := decisions_stats[reason][origin]; !ok {
decisions_stats[reason][origin] = make(map[string]int)
}
decisions_stats[reason][origin][action] += ival
mDecision.Process(reason, origin, action, ival)
case "cs_alerts":
/*if _, ok := alerts_stats[scenario]; !ok {
alerts_stats[scenario] = make(map[string]int)
}*/
alerts_stats[reason] += ival
mAlert.Process(reason, ival)
//
// stash
//
case "cs_cache_size":
stash_stats[name] = struct {
Type string
Count int
}{Type: mtype, Count: ival}
mStash.Process(name, mtype, ival)
//
// appsec
//
case "cs_appsec_reqs_total":
mAppsecEngine.Process(appsecEngine, "processed", ival)
case "cs_appsec_block_total":
mAppsecEngine.Process(appsecEngine, "blocked", ival)
case "cs_appsec_rule_hits":
mAppsecRule.Process(appsecEngine, appsecRule, "triggered", ival)
default:
log.Debugf("unknown: %+v", fam.Name)
continue
}
}
}
if formatType == "human" {
acquisStatsTable(out, acquis_stats)
bucketStatsTable(out, buckets_stats)
parserStatsTable(out, parsers_stats)
lapiStatsTable(out, lapi_stats)
lapiMachineStatsTable(out, lapi_machine_stats)
lapiBouncerStatsTable(out, lapi_bouncer_stats)
lapiDecisionStatsTable(out, lapi_decisions_stats)
decisionStatsTable(out, decisions_stats)
alertStatsTable(out, alerts_stats)
stashStatsTable(out, stash_stats)
return nil
return nil
}
type cliMetrics struct {
cfg configGetter
}
func NewCLIMetrics(cfg configGetter) *cliMetrics {
return &cliMetrics{
cfg: cfg,
}
}
func (ms metricStore) Format(out io.Writer, sections []string, formatType string, noUnit bool) error {
// copy only the sections we want
want := map[string]metricSection{}
// if explicitly asking for sections, we want to show empty tables
showEmpty := len(sections) > 0
// if no sections are specified, we want all of them
if len(sections) == 0 {
for section := range ms {
sections = append(sections, section)
}
}
stats := make(map[string]any)
stats["acquisition"] = acquis_stats
stats["buckets"] = buckets_stats
stats["parsers"] = parsers_stats
stats["lapi"] = lapi_stats
stats["lapi_machine"] = lapi_machine_stats
stats["lapi_bouncer"] = lapi_bouncer_stats
stats["lapi_decisions"] = lapi_decisions_stats
stats["decisions"] = decisions_stats
stats["alerts"] = alerts_stats
stats["stash"] = stash_stats
for _, section := range sections {
want[section] = ms[section]
}
switch formatType {
case "human":
for section := range want {
want[section].Table(out, noUnit, showEmpty)
}
case "json":
x, err := json.MarshalIndent(stats, "", " ")
x, err := json.MarshalIndent(want, "", " ")
if err != nil {
return fmt.Errorf("failed to unmarshal metrics : %v", err)
return fmt.Errorf("failed to marshal metrics: %w", err)
}
out.Write(x)
case "raw":
x, err := yaml.Marshal(stats)
x, err := yaml.Marshal(want)
if err != nil {
return fmt.Errorf("failed to unmarshal metrics : %v", err)
return fmt.Errorf("failed to marshal metrics: %w", err)
}
out.Write(x)
default:
@ -280,49 +306,190 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
return nil
}
var noUnit bool
func (cli *cliMetrics) show(sections []string, url string, noUnit bool) error {
cfg := cli.cfg()
func runMetrics(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadPrometheus(); err != nil {
return fmt.Errorf("failed to load prometheus config: %w", err)
if url != "" {
cfg.Cscli.PrometheusUrl = url
}
if csConfig.Prometheus == nil {
return fmt.Errorf("prometheus section missing, can't show metrics")
if cfg.Prometheus == nil {
return ErrMissingConfig
}
if !csConfig.Prometheus.Enabled {
return fmt.Errorf("prometheus is not enabled, can't show metrics")
if !cfg.Prometheus.Enabled {
return ErrMetricsDisabled
}
if prometheusURL == "" {
prometheusURL = csConfig.Cscli.PrometheusUrl
ms := NewMetricStore()
if err := ms.Fetch(cfg.Cscli.PrometheusUrl); err != nil {
return err
}
if prometheusURL == "" {
return fmt.Errorf("no prometheus url, please specify in %s or via -u", *csConfig.FilePath)
// any section that we don't have in the store is an error
for _, section := range sections {
if _, ok := ms[section]; !ok {
return fmt.Errorf("unknown metrics type: %s", section)
}
}
err := FormatPrometheusMetrics(color.Output, prometheusURL+"/metrics", csConfig.Cscli.Output)
if err != nil {
return fmt.Errorf("could not fetch prometheus metrics: %w", err)
if err := ms.Format(color.Output, sections, cfg.Cscli.Output, noUnit); err != nil {
return err
}
return nil
}
func (cli *cliMetrics) NewCommand() *cobra.Command {
var (
url string
noUnit bool
)
func NewMetricsCmd() *cobra.Command {
cmdMetrics := &cobra.Command{
Use: "metrics",
Short: "Display crowdsec prometheus metrics.",
Long: `Fetch metrics from the prometheus server and display them in a human-friendly way`,
cmd := &cobra.Command{
Use: "metrics",
Short: "Display crowdsec prometheus metrics.",
Long: `Fetch metrics from a Local API server and display them`,
Example: `# Show all Metrics, skip empty tables (same as "cecli metrics show")
cscli metrics
# Show only some metrics, connect to a different url
cscli metrics --url http://lapi.local:6060/metrics show acquisition parsers
# List available metric types
cscli metrics list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: runMetrics,
RunE: func(cmd *cobra.Command, args []string) error {
return cli.show(nil, url, noUnit)
},
}
cmdMetrics.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
cmdMetrics.PersistentFlags().BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
return cmdMetrics
flags := cmd.Flags()
flags.StringVarP(&url, "url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
flags.BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
cmd.AddCommand(cli.newShowCmd())
cmd.AddCommand(cli.newListCmd())
return cmd
}
// expandAlias returns a list of sections. The input can be a list of sections or alias.
func (cli *cliMetrics) expandSectionGroups(args []string) []string {
ret := []string{}
for _, section := range args {
switch section {
case "engine":
ret = append(ret, "acquisition", "parsers", "buckets", "stash", "whitelists")
case "lapi":
ret = append(ret, "alerts", "decisions", "lapi", "lapi-bouncer", "lapi-decisions", "lapi-machine")
case "appsec":
ret = append(ret, "appsec-engine", "appsec-rule")
default:
ret = append(ret, section)
}
}
return ret
}
func (cli *cliMetrics) newShowCmd() *cobra.Command {
var (
url string
noUnit bool
)
cmd := &cobra.Command{
Use: "show [type]...",
Short: "Display all or part of the available metrics.",
Long: `Fetch metrics from a Local API server and display them, optionally filtering on specific types.`,
Example: `# Show all Metrics, skip empty tables
cscli metrics show
# Use an alias: "engine", "lapi" or "appsec" to show a group of metrics
cscli metrics show engine
# Show some specific metrics, show empty tables, connect to a different url
cscli metrics show acquisition parsers buckets stash --url http://lapi.local:6060/metrics
# Show metrics in json format
cscli metrics show acquisition parsers buckets stash -o json`,
// Positional args are optional
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
args = cli.expandSectionGroups(args)
return cli.show(args, url, noUnit)
},
}
flags := cmd.Flags()
flags.StringVarP(&url, "url", "u", "", "Metrics url (http://<ip>:<port>/metrics)")
flags.BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
return cmd
}
func (cli *cliMetrics) list() error {
type metricType struct {
Type string `json:"type" yaml:"type"`
Title string `json:"title" yaml:"title"`
Description string `json:"description" yaml:"description"`
}
var allMetrics []metricType
ms := NewMetricStore()
for _, section := range maptools.SortedKeys(ms) {
title, description := ms[section].Description()
allMetrics = append(allMetrics, metricType{
Type: section,
Title: title,
Description: description,
})
}
switch cli.cfg().Cscli.Output {
case "human":
t := newTable(color.Output)
t.SetRowLines(true)
t.SetHeaders("Type", "Title", "Description")
for _, metric := range allMetrics {
t.AddRow(metric.Type, metric.Title, metric.Description)
}
t.Render()
case "json":
x, err := json.MarshalIndent(allMetrics, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal metric types: %w", err)
}
fmt.Println(string(x))
case "raw":
x, err := yaml.Marshal(allMetrics)
if err != nil {
return fmt.Errorf("failed to marshal metric types: %w", err)
}
fmt.Println(string(x))
}
return nil
}
func (cli *cliMetrics) newListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "list",
Short: "List available types of metrics.",
Long: `List available types of metrics.`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.list()
},
}
return cmd
}

464
cmd/crowdsec-cli/metrics_table.go
View file

@ -4,22 +4,29 @@ import (
"fmt"
"io"
"sort"
"strconv"
"github.com/aquasecurity/table"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/maptools"
)
// ErrNilTable means a nil pointer was passed instead of a table instance. This is a programming error.
var ErrNilTable = fmt.Errorf("nil table")
func lapiMetricsToTable(t *table.Table, stats map[string]map[string]map[string]int) int {
// stats: machine -> route -> method -> count
// sort keys to keep consistent order when printing
machineKeys := []string{}
for k := range stats {
machineKeys = append(machineKeys, k)
}
sort.Strings(machineKeys)
numRows := 0
for _, machine := range machineKeys {
// oneRow: route -> method -> count
machineRow := stats[machine]
@ -31,41 +38,77 @@ func lapiMetricsToTable(t *table.Table, stats map[string]map[string]map[string]i
methodName,
}
if count != 0 {
row = append(row, fmt.Sprintf("%d", count))
row = append(row, strconv.Itoa(count))
} else {
row = append(row, "-")
}
t.AddRow(row...)
numRows++
}
}
}
return numRows
}
func metricsToTable(t *table.Table, stats map[string]map[string]int, keys []string) (int, error) {
func wlMetricsToTable(t *table.Table, stats map[string]map[string]map[string]int, noUnit bool) (int, error) {
if t == nil {
return 0, fmt.Errorf("nil table")
return 0, ErrNilTable
}
// sort keys to keep consistent order when printing
sortedKeys := []string{}
for k := range stats {
sortedKeys = append(sortedKeys, k)
}
sort.Strings(sortedKeys)
numRows := 0
for _, alabel := range sortedKeys {
for _, name := range maptools.SortedKeys(stats) {
for _, reason := range maptools.SortedKeys(stats[name]) {
row := []string{
name,
reason,
"-",
"-",
}
for _, action := range maptools.SortedKeys(stats[name][reason]) {
value := stats[name][reason][action]
switch action {
case "whitelisted":
row[3] = strconv.Itoa(value)
case "hits":
row[2] = strconv.Itoa(value)
default:
log.Debugf("unexpected counter '%s' for whitelists = %d", action, value)
}
}
t.AddRow(row...)
numRows++
}
}
return numRows, nil
}
func metricsToTable(t *table.Table, stats map[string]map[string]int, keys []string, noUnit bool) (int, error) {
if t == nil {
return 0, ErrNilTable
}
numRows := 0
for _, alabel := range maptools.SortedKeys(stats) {
astats, ok := stats[alabel]
if !ok {
continue
}
row := []string{
alabel,
}
for _, sl := range keys {
if v, ok := astats[sl]; ok && v != 0 {
numberToShow := fmt.Sprintf("%d", v)
numberToShow := strconv.Itoa(v)
if !noUnit {
numberToShow = formatNumber(v)
}
@ -75,13 +118,29 @@ func metricsToTable(t *table.Table, stats map[string]map[string]int, keys []stri
row = append(row, "-")
}
}
t.AddRow(row...)
numRows++
}
return numRows, nil
}
func bucketStatsTable(out io.Writer, stats map[string]map[string]int) {
func (s statBucket) Description() (string, string) {
return "Bucket Metrics",
`Measure events in different scenarios. Current count is the number of buckets during metrics collection. ` +
`Overflows are past event-producing buckets, while Expired are the ones that didn’t receive enough events to Overflow.`
}
func (s statBucket) Process(bucket, metric string, val int) {
if _, ok := s[bucket]; !ok {
s[bucket] = make(map[string]int)
}
s[bucket][metric] += val
}
func (s statBucket) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Bucket", "Current Count", "Overflows", "Instantiated", "Poured", "Expired")
@ -89,31 +148,161 @@ func bucketStatsTable(out io.Writer, stats map[string]map[string]int) {
keys := []string{"curr_count", "overflow", "instantiation", "pour", "underflow"}
if numRows, err := metricsToTable(t, stats, keys); err != nil {
log.Warningf("while collecting acquis stats: %s", err)
} else if numRows > 0 {
renderTableTitle(out, "\nBucket Metrics:")
if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
log.Warningf("while collecting bucket stats: %s", err)
} else if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func acquisStatsTable(out io.Writer, stats map[string]map[string]int) {
func (s statAcquis) Description() (string, string) {
return "Acquisition Metrics",
`Measures the lines read, parsed, and unparsed per datasource. ` +
`Zero read lines indicate a misconfigured or inactive datasource. ` +
`Zero parsed lines mean the parser(s) failed. ` +
`Non-zero parsed lines are fine as crowdsec selects relevant lines.`
}
func (s statAcquis) Process(source, metric string, val int) {
if _, ok := s[source]; !ok {
s[source] = make(map[string]int)
}
s[source][metric] += val
}
func (s statAcquis) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Source", "Lines read", "Lines parsed", "Lines unparsed", "Lines poured to bucket")
t.SetHeaders("Source", "Lines read", "Lines parsed", "Lines unparsed", "Lines poured to bucket", "Lines whitelisted")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
keys := []string{"reads", "parsed", "unparsed", "pour"}
keys := []string{"reads", "parsed", "unparsed", "pour", "whitelisted"}
if numRows, err := metricsToTable(t, stats, keys); err != nil {
if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
log.Warningf("while collecting acquis stats: %s", err)
} else if numRows > 0 {
renderTableTitle(out, "\nAcquisition Metrics:")
} else if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func parserStatsTable(out io.Writer, stats map[string]map[string]int) {
func (s statAppsecEngine) Description() (string, string) {
return "Appsec Metrics",
`Measures the number of parsed and blocked requests by the AppSec Component.`
}
func (s statAppsecEngine) Process(appsecEngine, metric string, val int) {
if _, ok := s[appsecEngine]; !ok {
s[appsecEngine] = make(map[string]int)
}
s[appsecEngine][metric] += val
}
func (s statAppsecEngine) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Appsec Engine", "Processed", "Blocked")
t.SetAlignment(table.AlignLeft, table.AlignLeft)
keys := []string{"processed", "blocked"}
if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
log.Warningf("while collecting appsec stats: %s", err)
} else if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func (s statAppsecRule) Description() (string, string) {
return "Appsec Rule Metrics",
`Provides “per AppSec Component” information about the number of matches for loaded AppSec Rules.`
}
func (s statAppsecRule) Process(appsecEngine, appsecRule string, metric string, val int) {
if _, ok := s[appsecEngine]; !ok {
s[appsecEngine] = make(map[string]map[string]int)
}
if _, ok := s[appsecEngine][appsecRule]; !ok {
s[appsecEngine][appsecRule] = make(map[string]int)
}
s[appsecEngine][appsecRule][metric] += val
}
func (s statAppsecRule) Table(out io.Writer, noUnit bool, showEmpty bool) {
for appsecEngine, appsecEngineRulesStats := range s {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Rule ID", "Triggered")
t.SetAlignment(table.AlignLeft, table.AlignLeft)
keys := []string{"triggered"}
if numRows, err := metricsToTable(t, appsecEngineRulesStats, keys, noUnit); err != nil {
log.Warningf("while collecting appsec rules stats: %s", err)
} else if numRows > 0 || showEmpty {
renderTableTitle(out, fmt.Sprintf("\nAppsec '%s' Rules Metrics:", appsecEngine))
t.Render()
}
}
}
func (s statWhitelist) Description() (string, string) {
return "Whitelist Metrics",
`Tracks the number of events processed and possibly whitelisted by each parser whitelist.`
}
func (s statWhitelist) Process(whitelist, reason, metric string, val int) {
if _, ok := s[whitelist]; !ok {
s[whitelist] = make(map[string]map[string]int)
}
if _, ok := s[whitelist][reason]; !ok {
s[whitelist][reason] = make(map[string]int)
}
s[whitelist][reason][metric] += val
}
func (s statWhitelist) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Whitelist", "Reason", "Hits", "Whitelisted")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
if numRows, err := wlMetricsToTable(t, s, noUnit); err != nil {
log.Warningf("while collecting parsers stats: %s", err)
} else if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func (s statParser) Description() (string, string) {
return "Parser Metrics",
`Tracks the number of events processed by each parser and indicates success of failure. ` +
`Zero parsed lines means the parer(s) failed. ` +
`Non-zero unparsed lines are fine as crowdsec select relevant lines.`
}
func (s statParser) Process(parser, metric string, val int) {
if _, ok := s[parser]; !ok {
s[parser] = make(map[string]int)
}
s[parser][metric] += val
}
func (s statParser) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Parsers", "Hits", "Parsed", "Unparsed")
@ -121,187 +310,296 @@ func parserStatsTable(out io.Writer, stats map[string]map[string]int) {
keys := []string{"hits", "parsed", "unparsed"}
if numRows, err := metricsToTable(t, stats, keys); err != nil {
log.Warningf("while collecting acquis stats: %s", err)
} else if numRows > 0 {
renderTableTitle(out, "\nParser Metrics:")
if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
log.Warningf("while collecting parsers stats: %s", err)
} else if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func stashStatsTable(out io.Writer, stats map[string]struct {
Type string
Count int
}) {
func (s statStash) Description() (string, string) {
return "Parser Stash Metrics",
`Tracks the status of stashes that might be created by various parsers and scenarios.`
}
func (s statStash) Process(name, mtype string, val int) {
s[name] = struct {
Type string
Count int
}{
Type: mtype,
Count: val,
}
}
func (s statStash) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Name", "Type", "Items")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
// unfortunately, we can't reuse metricsToTable as the structure is too different :/
sortedKeys := []string{}
for k := range stats {
sortedKeys = append(sortedKeys, k)
}
sort.Strings(sortedKeys)
numRows := 0
for _, alabel := range sortedKeys {
astats := stats[alabel]
for _, alabel := range maptools.SortedKeys(s) {
astats := s[alabel]
row := []string{
alabel,
astats.Type,
fmt.Sprintf("%d", astats.Count),
strconv.Itoa(astats.Count),
}
t.AddRow(row...)
numRows++
}
if numRows > 0 {
renderTableTitle(out, "\nParser Stash Metrics:")
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func lapiStatsTable(out io.Writer, stats map[string]map[string]int) {
func (s statLapi) Description() (string, string) {
return "Local API Metrics",
`Monitors the requests made to local API routes.`
}
func (s statLapi) Process(route, method string, val int) {
if _, ok := s[route]; !ok {
s[route] = make(map[string]int)
}
s[route][method] += val
}
func (s statLapi) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Route", "Method", "Hits")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
// unfortunately, we can't reuse metricsToTable as the structure is too different :/
sortedKeys := []string{}
for k := range stats {
sortedKeys = append(sortedKeys, k)
}
sort.Strings(sortedKeys)
numRows := 0
for _, alabel := range sortedKeys {
astats := stats[alabel]
for _, alabel := range maptools.SortedKeys(s) {
astats := s[alabel]
subKeys := []string{}
for skey := range astats {
subKeys = append(subKeys, skey)
}
sort.Strings(subKeys)
for _, sl := range subKeys {
row := []string{
alabel,
sl,
fmt.Sprintf("%d", astats[sl]),
strconv.Itoa(astats[sl]),
}
t.AddRow(row...)
numRows++
}
}
if numRows > 0 {
renderTableTitle(out, "\nLocal API Metrics:")
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func lapiMachineStatsTable(out io.Writer, stats map[string]map[string]map[string]int) {
func (s statLapiMachine) Description() (string, string) {
return "Local API Machines Metrics",
`Tracks the number of calls to the local API from each registered machine.`
}
func (s statLapiMachine) Process(machine, route, method string, val int) {
if _, ok := s[machine]; !ok {
s[machine] = make(map[string]map[string]int)
}
if _, ok := s[machine][route]; !ok {
s[machine][route] = make(map[string]int)
}
s[machine][route][method] += val
}
func (s statLapiMachine) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Machine", "Route", "Method", "Hits")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
numRows := lapiMetricsToTable(t, stats)
numRows := lapiMetricsToTable(t, s)
if numRows > 0 {
renderTableTitle(out, "\nLocal API Machines Metrics:")
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func lapiBouncerStatsTable(out io.Writer, stats map[string]map[string]map[string]int) {
func (s statLapiBouncer) Description() (string, string) {
return "Local API Bouncers Metrics",
`Tracks total hits to remediation component related API routes.`
}
func (s statLapiBouncer) Process(bouncer, route, method string, val int) {
if _, ok := s[bouncer]; !ok {
s[bouncer] = make(map[string]map[string]int)
}
if _, ok := s[bouncer][route]; !ok {
s[bouncer][route] = make(map[string]int)
}
s[bouncer][route][method] += val
}
func (s statLapiBouncer) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Bouncer", "Route", "Method", "Hits")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
numRows := lapiMetricsToTable(t, stats)
numRows := lapiMetricsToTable(t, s)
if numRows > 0 {
renderTableTitle(out, "\nLocal API Bouncers Metrics:")
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func lapiDecisionStatsTable(out io.Writer, stats map[string]struct {
NonEmpty int
Empty int
},
) {
func (s statLapiDecision) Description() (string, string) {
return "Local API Bouncers Decisions",
`Tracks the number of empty/non-empty answers from LAPI to bouncers that are working in "live" mode.`
}
func (s statLapiDecision) Process(bouncer, fam string, val int) {
if _, ok := s[bouncer]; !ok {
s[bouncer] = struct {
NonEmpty int
Empty int
}{}
}
x := s[bouncer]
switch fam {
case "cs_lapi_decisions_ko_total":
x.Empty += val
case "cs_lapi_decisions_ok_total":
x.NonEmpty += val
}
s[bouncer] = x
}
func (s statLapiDecision) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Bouncer", "Empty answers", "Non-empty answers")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
numRows := 0
for bouncer, hits := range stats {
for bouncer, hits := range s {
t.AddRow(
bouncer,
fmt.Sprintf("%d", hits.Empty),
fmt.Sprintf("%d", hits.NonEmpty),
strconv.Itoa(hits.Empty),
strconv.Itoa(hits.NonEmpty),
)
numRows++
}
if numRows > 0 {
renderTableTitle(out, "\nLocal API Bouncers Decisions:")
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func decisionStatsTable(out io.Writer, stats map[string]map[string]map[string]int) {
func (s statDecision) Description() (string, string) {
return "Local API Decisions",
`Provides information about all currently active decisions. ` +
`Includes both local (crowdsec) and global decisions (CAPI), and lists subscriptions (lists).`
}
func (s statDecision) Process(reason, origin, action string, val int) {
if _, ok := s[reason]; !ok {
s[reason] = make(map[string]map[string]int)
}
if _, ok := s[reason][origin]; !ok {
s[reason][origin] = make(map[string]int)
}
s[reason][origin][action] += val
}
func (s statDecision) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Reason", "Origin", "Action", "Count")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
numRows := 0
for reason, origins := range stats {
for reason, origins := range s {
for origin, actions := range origins {
for action, hits := range actions {
t.AddRow(
reason,
origin,
action,
fmt.Sprintf("%d", hits),
strconv.Itoa(hits),
)
numRows++
}
}
}
if numRows > 0 {
renderTableTitle(out, "\nLocal API Decisions:")
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func alertStatsTable(out io.Writer, stats map[string]int) {
func (s statAlert) Description() (string, string) {
return "Local API Alerts",
`Tracks the total number of past and present alerts for the installed scenarios.`
}
func (s statAlert) Process(reason string, val int) {
s[reason] += val
}
func (s statAlert) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Reason", "Count")
t.SetAlignment(table.AlignLeft, table.AlignLeft)
numRows := 0
for scenario, hits := range stats {
for scenario, hits := range s {
t.AddRow(
scenario,
fmt.Sprintf("%d", hits),
strconv.Itoa(hits),
)
numRows++
}
if numRows > 0 {
renderTableTitle(out, "\nLocal API Alerts:")
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}

308
cmd/crowdsec-cli/notifications.go
View file

@ -18,15 +18,19 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/csplugin"
"github.com/crowdsecurity/crowdsec/pkg/csprofiles"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/models"
)
type NotificationsCfg struct {
@ -35,8 +39,14 @@ type NotificationsCfg struct {
ids []uint
}
func NewNotificationsCmd() *cobra.Command {
var cmdNotifications = &cobra.Command{
type cliNotifications struct{}
func NewCLINotifications() *cliNotifications {
return &cliNotifications{}
}
func (cli cliNotifications) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "notifications [action]",
Short: "Helper for notification plugin configuration",
Long: "To list/inspect/test notification template",
@ -47,8 +57,8 @@ func NewNotificationsCmd() *cobra.Command {
if err := require.LAPI(csConfig); err != nil {
return err
}
if err := require.Profiles(csConfig); err != nil {
return err
if err := csConfig.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
}
if err := require.Notifications(csConfig); err != nil {
return err
@ -58,14 +68,15 @@ func NewNotificationsCmd() *cobra.Command {
},
}
cmdNotifications.AddCommand(NewNotificationsListCmd())
cmdNotifications.AddCommand(NewNotificationsInspectCmd())
cmdNotifications.AddCommand(NewNotificationsReinjectCmd())
cmd.AddCommand(cli.NewListCmd())
cmd.AddCommand(cli.NewInspectCmd())
cmd.AddCommand(cli.NewReinjectCmd())
cmd.AddCommand(cli.NewTestCmd())
return cmdNotifications
return cmd
}
func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
func getPluginConfigs() (map[string]csplugin.PluginConfig, error) {
pcfgs := map[string]csplugin.PluginConfig{}
wf := func(path string, info fs.FileInfo, err error) error {
if info == nil {
@ -78,6 +89,7 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
return fmt.Errorf("loading notifification plugin configuration with %s: %w", name, err)
}
for _, t := range ts {
csplugin.SetRequiredFields(&t)
pcfgs[t.Name] = t
}
}
@ -87,57 +99,53 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
if err := filepath.Walk(csConfig.ConfigPaths.NotificationDir, wf); err != nil {
return nil, fmt.Errorf("while loading notifification plugin configuration: %w", err)
}
return pcfgs, nil
}
func getProfilesConfigs() (map[string]NotificationsCfg, error) {
// A bit of a tricky stuf now: reconcile profiles and notification plugins
pcfgs, err := getPluginConfigs()
if err != nil {
return nil, err
}
ncfgs := map[string]NotificationsCfg{}
for _, pc := range pcfgs {
ncfgs[pc.Name] = NotificationsCfg{
Config: pc,
}
}
profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
if err != nil {
return nil, fmt.Errorf("while extracting profiles from configuration: %w", err)
}
for profileID, profile := range profiles {
loop:
for _, notif := range profile.Cfg.Notifications {
for name, pc := range pcfgs {
if notif == name {
if _, ok := ncfgs[pc.Name]; !ok {
ncfgs[pc.Name] = NotificationsCfg{
Config: pc,
Profiles: []*csconfig.ProfileCfg{profile.Cfg},
ids: []uint{uint(profileID)},
}
continue loop
}
tmp := ncfgs[pc.Name]
for _, pr := range tmp.Profiles {
var profiles []*csconfig.ProfileCfg
if pr.Name == profile.Cfg.Name {
continue
}
profiles = append(tmp.Profiles, profile.Cfg)
ids := append(tmp.ids, uint(profileID))
ncfgs[pc.Name] = NotificationsCfg{
Config: tmp.Config,
Profiles: profiles,
ids: ids,
}
}
}
pc, ok := pcfgs[notif]
if !ok {
return nil, fmt.Errorf("notification plugin '%s' does not exist", notif)
}
tmp, ok := ncfgs[pc.Name]
if !ok {
return nil, fmt.Errorf("notification plugin '%s' does not exist", pc.Name)
}
tmp.Profiles = append(tmp.Profiles, profile.Cfg)
tmp.ids = append(tmp.ids, uint(profileID))
ncfgs[pc.Name] = tmp
}
}
return ncfgs, nil
}
func NewNotificationsListCmd() *cobra.Command {
var cmdNotificationsList = &cobra.Command{
func (cli cliNotifications) NewListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "list",
Short: "List active notifications plugins",
Long: `List active notifications plugins`,
Short: "list active notifications plugins",
Long: `list active notifications plugins`,
Example: `cscli notifications list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, arg []string) error {
ncfgs, err := getNotificationsConfiguration()
ncfgs, err := getProfilesConfigs()
if err != nil {
return fmt.Errorf("can't build profiles configuration: %w", err)
}
@ -172,36 +180,32 @@ func NewNotificationsListCmd() *cobra.Command {
},
}
return cmdNotificationsList
return cmd
}
func NewNotificationsInspectCmd() *cobra.Command {
var cmdNotificationsInspect = &cobra.Command{
func (cli cliNotifications) NewInspectCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "inspect",
Short: "Inspect active notifications plugin configuration",
Long: `Inspect active notifications plugin and show configuration`,
Example: `cscli notifications inspect <plugin_name>`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, arg []string) error {
var (
cfg NotificationsCfg
ok bool
)
pluginName := arg[0]
if pluginName == "" {
PreRunE: func(cmd *cobra.Command, args []string) error {
if args[0] == "" {
return fmt.Errorf("please provide a plugin name to inspect")
}
ncfgs, err := getNotificationsConfiguration()
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
ncfgs, err := getProfilesConfigs()
if err != nil {
return fmt.Errorf("can't build profiles configuration: %w", err)
}
if cfg, ok = ncfgs[pluginName]; !ok {
return fmt.Errorf("plugin '%s' does not exist or is not active", pluginName)
cfg, ok := ncfgs[args[0]]
if !ok {
return fmt.Errorf("plugin '%s' does not exist or is not active", args[0])
}
if csConfig.Cscli.Output == "human" || csConfig.Cscli.Output == "raw" {
fmt.Printf(" - %15s: %15s\n", "Type", cfg.Config.Type)
fmt.Printf(" - %15s: %15s\n", "Name", cfg.Config.Name)
@ -221,78 +225,128 @@ func NewNotificationsInspectCmd() *cobra.Command {
},
}
return cmdNotificationsInspect
return cmd
}
func NewNotificationsReinjectCmd() *cobra.Command {
var remediation bool
var alertOverride string
func (cli cliNotifications) NewTestCmd() *cobra.Command {
var (
pluginBroker csplugin.PluginBroker
pluginTomb tomb.Tomb
alertOverride string
)
cmd := &cobra.Command{
Use: "test [plugin name]",
Short: "send a generic test alert to notification plugin",
Long: `send a generic test alert to a notification plugin to test configuration even if is not active`,
Example: `cscli notifications test [plugin_name]`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
PreRunE: func(cmd *cobra.Command, args []string) error {
pconfigs, err := getPluginConfigs()
if err != nil {
return fmt.Errorf("can't build profiles configuration: %w", err)
}
cfg, ok := pconfigs[args[0]]
if !ok {
return fmt.Errorf("plugin name: '%s' does not exist", args[0])
}
//Create a single profile with plugin name as notification name
return pluginBroker.Init(csConfig.PluginConfig, []*csconfig.ProfileCfg{
{
Notifications: []string{
cfg.Name,
},
},
}, csConfig.ConfigPaths)
},
RunE: func(cmd *cobra.Command, args []string) error {
pluginTomb.Go(func() error {
pluginBroker.Run(&pluginTomb)
return nil
})
alert := &models.Alert{
Capacity: ptr.Of(int32(0)),
Decisions: []*models.Decision{{
Duration: ptr.Of("4h"),
Scope: ptr.Of("Ip"),
Value: ptr.Of("10.10.10.10"),
Type: ptr.Of("ban"),
Scenario: ptr.Of("test alert"),
Origin: ptr.Of(types.CscliOrigin),
}},
Events: []*models.Event{},
EventsCount: ptr.Of(int32(1)),
Leakspeed: ptr.Of("0"),
Message: ptr.Of("test alert"),
ScenarioHash: ptr.Of(""),
Scenario: ptr.Of("test alert"),
ScenarioVersion: ptr.Of(""),
Simulated: ptr.Of(false),
Source: &models.Source{
AsName: "",
AsNumber: "",
Cn: "",
IP: "10.10.10.10",
Range: "",
Scope: ptr.Of("Ip"),
Value: ptr.Of("10.10.10.10"),
},
StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
CreatedAt: time.Now().UTC().Format(time.RFC3339),
}
if err := yaml.Unmarshal([]byte(alertOverride), alert); err != nil {
return fmt.Errorf("failed to unmarshal alert override: %w", err)
}
pluginBroker.PluginChannel <- csplugin.ProfileAlert{
ProfileID: uint(0),
Alert: alert,
}
//time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
pluginTomb.Kill(fmt.Errorf("terminating"))
pluginTomb.Wait()
return nil
},
}
cmd.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the generic alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
var cmdNotificationsReinject = &cobra.Command{
return cmd
}
func (cli cliNotifications) NewReinjectCmd() *cobra.Command {
var alertOverride string
var alert *models.Alert
cmd := &cobra.Command{
Use: "reinject",
Short: "reinject alert into notifications system",
Long: `Reinject alert into notifications system`,
Short: "reinject an alert into profiles to trigger notifications",
Long: `reinject an alert into profiles to be evaluated by the filter and sent to matched notifications plugins`,
Example: `
cscli notifications reinject <alert_id>
cscli notifications reinject <alert_id> --remediation
cscli notifications reinject <alert_id> -a '{"remediation": false,"scenario":"notification/test"}'
cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"notification/test"}'
`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
PreRunE: func(cmd *cobra.Command, args []string) error {
var err error
alert, err = FetchAlertFromArgString(args[0])
if err != nil {
return err
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
var (
pluginBroker csplugin.PluginBroker
pluginTomb tomb.Tomb
)
if len(args) != 1 {
printHelp(cmd)
return fmt.Errorf("wrong number of argument: there should be one argument")
}
//first: get the alert
id, err := strconv.Atoi(args[0])
if err != nil {
return fmt.Errorf("bad alert id %s", args[0])
}
if err := csConfig.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
}
if csConfig.API.Client == nil {
return fmt.Errorf("missing configuration on 'api_client:'")
}
if csConfig.API.Client.Credentials == nil {
return fmt.Errorf("missing API credentials in '%s'", csConfig.API.Client.CredentialsFilePath)
}
apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
if err != nil {
return fmt.Errorf("error parsing the URL of the API: %w", err)
}
client, err := apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
Password: strfmt.Password(csConfig.API.Client.Credentials.Password),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
})
if err != nil {
return fmt.Errorf("error creating the client for the API: %w", err)
}
alert, _, err := client.Alerts.GetByID(context.Background(), id)
if err != nil {
return fmt.Errorf("can't find alert with id %s: %w", args[0], err)
}
if alertOverride != "" {
if err = json.Unmarshal([]byte(alertOverride), alert); err != nil {
if err := json.Unmarshal([]byte(alertOverride), alert); err != nil {
return fmt.Errorf("can't unmarshal data in the alert flag: %w", err)
}
}
if !remediation {
alert.Remediation = true
}
// second we start plugins
err = pluginBroker.Init(csConfig.PluginConfig, csConfig.API.Server.Profiles, csConfig.ConfigPaths)
err := pluginBroker.Init(csConfig.PluginConfig, csConfig.API.Server.Profiles, csConfig.ConfigPaths)
if err != nil {
return fmt.Errorf("can't initialize plugins: %w", err)
}
@ -302,8 +356,6 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
return nil
})
//third: get the profile(s), and process the whole stuff
profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
if err != nil {
return fmt.Errorf("cannot extract profiles from configuration: %w", err)
@ -338,15 +390,39 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
break
}
}
// time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
//time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
pluginTomb.Kill(fmt.Errorf("terminating"))
pluginTomb.Wait()
return nil
},
}
cmdNotificationsReinject.Flags().BoolVarP(&remediation, "remediation", "r", false, "Set Alert.Remediation to false in the reinjected alert (see your profile filter configuration)")
cmdNotificationsReinject.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the reinjected alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
cmd.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the reinjected alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
return cmdNotificationsReinject
return cmd
}
func FetchAlertFromArgString(toParse string) (*models.Alert, error) {
id, err := strconv.Atoi(toParse)
if err != nil {
return nil, fmt.Errorf("bad alert id %s", toParse)
}
apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
if err != nil {
return nil, fmt.Errorf("error parsing the URL of the API: %w", err)
}
client, err := apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
Password: strfmt.Password(csConfig.API.Client.Credentials.Password),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
})
if err != nil {
return nil, fmt.Errorf("error creating the client for the API: %w", err)
}
alert, _, err := client.Alerts.GetByID(context.Background(), id)
if err != nil {
return nil, fmt.Errorf("can't find alert with id %d: %w", id, err)
}
return alert, nil
}

26
cmd/crowdsec-cli/notifications_table.go
View file

@ -2,24 +2,36 @@ package main
import (
"io"
"sort"
"strings"
"github.com/aquasecurity/table"
"github.com/enescakir/emoji"
)
func notificationListTable(out io.Writer, ncfgs map[string]NotificationsCfg) {
t := newLightTable(out)
t.SetHeaders("Name", "Type", "Profile name")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
for _, b := range ncfgs {
t.SetHeaders("Active", "Name", "Type", "Profile name")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
keys := make([]string, 0, len(ncfgs))
for k := range ncfgs {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool {
return len(ncfgs[keys[i]].Profiles) > len(ncfgs[keys[j]].Profiles)
})
for _, k := range keys {
b := ncfgs[k]
profilesList := []string{}
for _, p := range b.Profiles {
profilesList = append(profilesList, p.Name)
}
t.AddRow(b.Config.Name, b.Config.Type, strings.Join(profilesList, ", "))
active := emoji.CheckMark.String()
if len(profilesList) == 0 {
active = emoji.Prohibited.String()
}
t.AddRow(active, b.Config.Name, b.Config.Type, strings.Join(profilesList, ", "))
}
t.Render()
}

87
cmd/crowdsec-cli/papi.go
View file

@ -1,6 +1,7 @@
package main
import (
"fmt"
"time"
log "github.com/sirupsen/logrus"
@ -9,67 +10,79 @@ import (
"github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiserver"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
func NewPapiCmd() *cobra.Command {
var cmdLapi = &cobra.Command{
type cliPapi struct {
cfg configGetter
}
func NewCLIPapi(cfg configGetter) *cliPapi {
return &cliPapi{
cfg: cfg,
}
}
func (cli *cliPapi) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "papi [action]",
Short: "Manage interaction with Polling API (PAPI)",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.LAPI(csConfig); err != nil {
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := require.LAPI(cfg); err != nil {
return err
}
if err := require.CAPI(csConfig); err != nil {
if err := require.CAPI(cfg); err != nil {
return err
}
if err := require.PAPI(csConfig); err != nil {
if err := require.PAPI(cfg); err != nil {
return err
}
return nil
},
}
cmdLapi.AddCommand(NewPapiStatusCmd())
cmdLapi.AddCommand(NewPapiSyncCmd())
cmd.AddCommand(cli.NewStatusCmd())
cmd.AddCommand(cli.NewSyncCmd())
return cmdLapi
return cmd
}
func NewPapiStatusCmd() *cobra.Command {
cmdCapiStatus := &cobra.Command{
func (cli *cliPapi) NewStatusCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "status",
Short: "Get status of the Polling API",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(_ *cobra.Command, _ []string) error {
var err error
dbClient, err = database.NewClient(csConfig.DbConfig)
cfg := cli.cfg()
dbClient, err = database.NewClient(cfg.DbConfig)
if err != nil {
log.Fatalf("unable to initialize database client : %s", err)
return fmt.Errorf("unable to initialize database client: %s", err)
}
apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig, csConfig.API.Server.CapiWhitelists)
apic, err := apiserver.NewAPIC(cfg.API.Server.OnlineClient, dbClient, cfg.API.Server.ConsoleConfig, cfg.API.Server.CapiWhitelists)
if err != nil {
log.Fatalf("unable to initialize API client : %s", err)
return fmt.Errorf("unable to initialize API client: %s", err)
}
papi, err := apiserver.NewPAPI(apic, dbClient, csConfig.API.Server.ConsoleConfig, log.GetLevel())
papi, err := apiserver.NewPAPI(apic, dbClient, cfg.API.Server.ConsoleConfig, log.GetLevel())
if err != nil {
log.Fatalf("unable to initialize PAPI client : %s", err)
return fmt.Errorf("unable to initialize PAPI client: %s", err)
}
perms, err := papi.GetPermissions()
if err != nil {
log.Fatalf("unable to get PAPI permissions: %s", err)
return fmt.Errorf("unable to get PAPI permissions: %s", err)
}
var lastTimestampStr *string
lastTimestampStr, err = dbClient.GetConfigItem(apiserver.PapiPullKey)
@ -84,45 +97,48 @@ func NewPapiStatusCmd() *cobra.Command {
for _, sub := range perms.Categories {
log.Infof(" - %s", sub)
}
return nil
},
}
return cmdCapiStatus
return cmd
}
func NewPapiSyncCmd() *cobra.Command {
cmdCapiSync := &cobra.Command{
func (cli *cliPapi) NewSyncCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "sync",
Short: "Sync with the Polling API, pulling all non-expired orders for the instance",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(_ *cobra.Command, _ []string) error {
var err error
cfg := cli.cfg()
t := tomb.Tomb{}
dbClient, err = database.NewClient(csConfig.DbConfig)
dbClient, err = database.NewClient(cfg.DbConfig)
if err != nil {
log.Fatalf("unable to initialize database client : %s", err)
return fmt.Errorf("unable to initialize database client: %s", err)
}
apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig, csConfig.API.Server.CapiWhitelists)
apic, err := apiserver.NewAPIC(cfg.API.Server.OnlineClient, dbClient, cfg.API.Server.ConsoleConfig, cfg.API.Server.CapiWhitelists)
if err != nil {
log.Fatalf("unable to initialize API client : %s", err)
return fmt.Errorf("unable to initialize API client: %s", err)
}
t.Go(apic.Push)
papi, err := apiserver.NewPAPI(apic, dbClient, csConfig.API.Server.ConsoleConfig, log.GetLevel())
papi, err := apiserver.NewPAPI(apic, dbClient, cfg.API.Server.ConsoleConfig, log.GetLevel())
if err != nil {
log.Fatalf("unable to initialize PAPI client : %s", err)
return fmt.Errorf("unable to initialize PAPI client: %s", err)
}
t.Go(papi.SyncDecisions)
err = papi.PullOnce(time.Time{}, true)
if err != nil {
log.Fatalf("unable to sync decisions: %s", err)
return fmt.Errorf("unable to sync decisions: %s", err)
}
log.Infof("Sending acknowledgements to CAPI")
@ -132,8 +148,9 @@ func NewPapiSyncCmd() *cobra.Command {
t.Wait()
time.Sleep(5 * time.Second) //FIXME: the push done by apic.Push is run inside a sub goroutine, sleep to make sure it's done
return nil
},
}
return cmdCapiSync
return cmd
}

194
cmd/crowdsec-cli/parsers.go
View file

@ -1,194 +0,0 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewParsersCmd() *cobra.Command {
var cmdParsers = &cobra.Command{
Use: "parsers [action] [config]",
Short: "Install/Remove/Upgrade/Inspect parser(s) from hub",
Example: `cscli parsers install crowdsecurity/sshd-logs
cscli parsers inspect crowdsecurity/sshd-logs
cscli parsers upgrade crowdsecurity/sshd-logs
cscli parsers list
cscli parsers remove crowdsecurity/sshd-logs
`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"parser"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
cmdParsers.AddCommand(NewParsersInstallCmd())
cmdParsers.AddCommand(NewParsersRemoveCmd())
cmdParsers.AddCommand(NewParsersUpgradeCmd())
cmdParsers.AddCommand(NewParsersInspectCmd())
cmdParsers.AddCommand(NewParsersListCmd())
return cmdParsers
}
func NewParsersInstallCmd() *cobra.Command {
var ignoreError bool
var cmdParsersInstall = &cobra.Command{
Use: "install [config]",
Short: "Install given parser(s)",
Long: `Fetch and install given parser(s) from hub`,
Example: `cscli parsers install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.PARSERS, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
for _, name := range args {
t := cwhub.GetItem(cwhub.PARSERS, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.PARSERS, name)
Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
}
}
return nil
},
}
cmdParsersInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdParsersInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdParsersInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple parsers")
return cmdParsersInstall
}
func NewParsersRemoveCmd() *cobra.Command {
cmdParsersRemove := &cobra.Command{
Use: "remove [config]",
Short: "Remove given parser(s)",
Long: `Remove given parse(s) from hub`,
Example: `cscli parsers remove crowdsec/xxx crowdsec/xyz`,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, forceAction)
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one parser to remove or '--all'")
}
for _, name := range args {
cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, forceAction)
}
return nil
},
}
cmdParsersRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdParsersRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdParsersRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the parsers")
return cmdParsersRemove
}
func NewParsersUpgradeCmd() *cobra.Command {
cmdParsersUpgrade := &cobra.Command{
Use: "upgrade [config]",
Short: "Upgrade given parser(s)",
Long: `Fetch and upgrade given parser(s) from hub`,
Example: `cscli parsers upgrade crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
} else {
if len(args) == 0 {
return fmt.Errorf("specify at least one parser to upgrade or '--all'")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, forceAction)
}
}
return nil
},
}
cmdParsersUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the parsers")
cmdParsersUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
return cmdParsersUpgrade
}
func NewParsersInspectCmd() *cobra.Command {
var cmdParsersInspect = &cobra.Command{
Use: "inspect [name]",
Short: "Inspect given parser",
Long: `Inspect given parser`,
Example: `cscli parsers inspect crowdsec/xxx`,
DisableAutoGenTag: true,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
InspectItem(args[0], cwhub.PARSERS)
},
}
cmdParsersInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
return cmdParsersInspect
}
func NewParsersListCmd() *cobra.Command {
var cmdParsersList = &cobra.Command{
Use: "list [name]",
Short: "List all parsers or given one",
Long: `List all parsers or given one`,
Example: `cscli parsers list
cscli parser list crowdsecurity/xxx`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all)
},
}
cmdParsersList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmdParsersList
}

191
cmd/crowdsec-cli/postoverflows.go
View file

@ -1,191 +0,0 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewPostOverflowsCmd() *cobra.Command {
cmdPostOverflows := &cobra.Command{
Use: "postoverflows [action] [config]",
Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub",
Example: `cscli postoverflows install crowdsecurity/cdn-whitelist
cscli postoverflows inspect crowdsecurity/cdn-whitelist
cscli postoverflows upgrade crowdsecurity/cdn-whitelist
cscli postoverflows list
cscli postoverflows remove crowdsecurity/cdn-whitelist`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"postoverflow"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsListCmd())
return cmdPostOverflows
}
func NewPostOverflowsInstallCmd() *cobra.Command {
var ignoreError bool
cmdPostOverflowsInstall := &cobra.Command{
Use: "install [config]",
Short: "Install given postoverflow(s)",
Long: `Fetch and install given postoverflow(s) from hub`,
Example: `cscli postoverflows install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
for _, name := range args {
t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.PARSERS_OVFLW, name)
Suggest(cwhub.PARSERS_OVFLW, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
}
}
return nil
},
}
cmdPostOverflowsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdPostOverflowsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdPostOverflowsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple postoverflows")
return cmdPostOverflowsInstall
}
func NewPostOverflowsRemoveCmd() *cobra.Command {
cmdPostOverflowsRemove := &cobra.Command{
Use: "remove [config]",
Short: "Remove given postoverflow(s)",
Long: `remove given postoverflow(s)`,
Example: `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, forceAction)
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one postoverflow to remove or '--all'")
}
for _, name := range args {
cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, forceAction)
}
return nil
},
}
cmdPostOverflowsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdPostOverflowsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdPostOverflowsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the postoverflows")
return cmdPostOverflowsRemove
}
func NewPostOverflowsUpgradeCmd() *cobra.Command {
cmdPostOverflowsUpgrade := &cobra.Command{
Use: "upgrade [config]",
Short: "Upgrade given postoverflow(s)",
Long: `Fetch and Upgrade given postoverflow(s) from hub`,
Example: `cscli postoverflows upgrade crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
} else {
if len(args) == 0 {
return fmt.Errorf("specify at least one postoverflow to upgrade or '--all'")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, forceAction)
}
}
return nil
},
}
cmdPostOverflowsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the postoverflows")
cmdPostOverflowsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
return cmdPostOverflowsUpgrade
}
func NewPostOverflowsInspectCmd() *cobra.Command {
cmdPostOverflowsInspect := &cobra.Command{
Use: "inspect [config]",
Short: "Inspect given postoverflow",
Long: `Inspect given postoverflow`,
Example: `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
InspectItem(args[0], cwhub.PARSERS_OVFLW)
},
}
return cmdPostOverflowsInspect
}
func NewPostOverflowsListCmd() *cobra.Command {
cmdPostOverflowsList := &cobra.Command{
Use: "list [config]",
Short: "List all postoverflows or given one",
Long: `List all postoverflows or given one`,
Example: `cscli postoverflows list
cscli postoverflows list crowdsecurity/xxx`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all)
},
}
cmdPostOverflowsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmdPostOverflowsList
}

62
cmd/crowdsec-cli/require/branch.go Normal file
View file

@ -0,0 +1,62 @@
package require
// Set the appropriate hub branch according to config settings and crowdsec version
import (
log "github.com/sirupsen/logrus"
"golang.org/x/mod/semver"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
func chooseBranch(cfg *csconfig.Config) string {
// this was set from config.yaml or flag
if cfg.Cscli.HubBranch != "" {
log.Debugf("Hub override from config: branch '%s'", cfg.Cscli.HubBranch)
return cfg.Cscli.HubBranch
}
latest, err := cwversion.Latest()
if err != nil {
log.Warningf("Unable to retrieve latest crowdsec version: %s, using hub branch 'master'", err)
return "master"
}
csVersion := cwversion.VersionStrip()
if csVersion == latest {
log.Debugf("Latest crowdsec version (%s), using hub branch 'master'", csVersion)
return "master"
}
// if current version is greater than the latest we are in pre-release
if semver.Compare(csVersion, latest) == 1 {
log.Debugf("Your current crowdsec version seems to be a pre-release (%s), using hub branch 'master'", csVersion)
return "master"
}
if csVersion == "" {
log.Warning("Crowdsec version is not set, using hub branch 'master'")
return "master"
}
log.Warnf("A new CrowdSec release is available (%s). "+
"Your version is '%s'. Please update it to use new parsers/scenarios/collections.",
latest, csVersion)
return csVersion
}
// HubBranch sets the branch (in cscli config) and returns its value
// It can be "master", or the branch corresponding to the current crowdsec version, or the value overridden in config/flag
func HubBranch(cfg *csconfig.Config) string {
branch := chooseBranch(cfg)
cfg.Cscli.HubBranch = branch
return branch
}
func HubURLTemplate(cfg *csconfig.Config) string {
return cfg.Cscli.HubURLTemplate
}

64
cmd/crowdsec-cli/require/require.go
View file

@ -2,13 +2,16 @@ package require
import (
"fmt"
"io"
"github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func LAPI(c *csconfig.Config) error {
if err := c.LoadAPIServer(); err != nil {
if err := c.LoadAPIServer(true); err != nil {
return fmt.Errorf("failed to load Local API: %w", err)
}
@ -23,6 +26,7 @@ func CAPI(c *csconfig.Config) error {
if c.API.Server.OnlineClient == nil {
return fmt.Errorf("no configuration for Central API (CAPI) in '%s'", *c.FilePath)
}
return nil
}
@ -30,6 +34,7 @@ func PAPI(c *csconfig.Config) error {
if c.API.Server.OnlineClient.Credentials.PapiURL == "" {
return fmt.Errorf("no PAPI URL in configuration")
}
return nil
}
@ -42,16 +47,9 @@ func CAPIRegistered(c *csconfig.Config) error {
}
func DB(c *csconfig.Config) error {
if err := c.LoadDBConfig(); err != nil {
if err := c.LoadDBConfig(true); err != nil {
return fmt.Errorf("this command requires direct database access (must be run on the local API machine): %w", err)
}
return nil
}
func Profiles(c *csconfig.Config) error {
if err := c.API.Server.LoadProfiles(); err != nil {
return fmt.Errorf("while loading profiles: %w", err)
}
return nil
}
@ -64,22 +62,38 @@ func Notifications(c *csconfig.Config) error {
return nil
}
func Hub (c *csconfig.Config) error {
if err := c.LoadHub(); err != nil {
return err
// RemoteHub returns the configuration required to download hub index and items: url, branch, etc.
func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg {
// set branch in config, and log if necessary
branch := HubBranch(c)
urlTemplate := HubURLTemplate(c)
remote := &cwhub.RemoteHubCfg{
Branch: branch,
URLTemplate: urlTemplate,
IndexPath: ".index.json",
}
if c.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("while setting hub branch: %w", err)
}
if err := cwhub.GetHubIdx(c.Hub); err != nil {
return fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err)
}
return nil
return remote
}
// Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items.
// If no remote parameter is provided, the hub can only be used for local operations.
func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg, logger *logrus.Logger) (*cwhub.Hub, error) {
local := c.Hub
if local == nil {
return nil, fmt.Errorf("you must configure cli before interacting with hub")
}
if logger == nil {
logger = logrus.New()
logger.SetOutput(io.Discard)
}
hub, err := cwhub.NewHub(local, remote, false, logger)
if err != nil {
return nil, fmt.Errorf("failed to read Hub index: %w. Run 'sudo cscli hub update' to download the index again", err)
}
return hub, nil
}

188
cmd/crowdsec-cli/scenarios.go
View file

@ -1,188 +0,0 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewScenariosCmd() *cobra.Command {
var cmdScenarios = &cobra.Command{
Use: "scenarios [action] [config]",
Short: "Install/Remove/Upgrade/Inspect scenario(s) from hub",
Example: `cscli scenarios list [-a]
cscli scenarios install crowdsecurity/ssh-bf
cscli scenarios inspect crowdsecurity/ssh-bf
cscli scenarios upgrade crowdsecurity/ssh-bf
cscli scenarios remove crowdsecurity/ssh-bf
`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"scenario"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
return err
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
cmdScenarios.AddCommand(NewCmdScenariosInstall())
cmdScenarios.AddCommand(NewCmdScenariosRemove())
cmdScenarios.AddCommand(NewCmdScenariosUpgrade())
cmdScenarios.AddCommand(NewCmdScenariosInspect())
cmdScenarios.AddCommand(NewCmdScenariosList())
return cmdScenarios
}
func NewCmdScenariosInstall() *cobra.Command {
var ignoreError bool
var cmdScenariosInstall = &cobra.Command{
Use: "install [config]",
Short: "Install given scenario(s)",
Long: `Fetch and install given scenario(s) from hub`,
Example: `cscli scenarios install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
for _, name := range args {
t := cwhub.GetItem(cwhub.SCENARIOS, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.SCENARIOS, name)
Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
}
}
return nil
},
}
cmdScenariosInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdScenariosInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdScenariosInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple scenarios")
return cmdScenariosInstall
}
func NewCmdScenariosRemove() *cobra.Command {
var cmdScenariosRemove = &cobra.Command{
Use: "remove [config]",
Short: "Remove given scenario(s)",
Long: `remove given scenario(s)`,
Example: `cscli scenarios remove crowdsec/xxx crowdsec/xyz`,
Aliases: []string{"delete"},
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, forceAction)
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one scenario to remove or '--all'")
}
for _, name := range args {
cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, forceAction)
}
return nil
},
}
cmdScenariosRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdScenariosRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdScenariosRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the scenarios")
return cmdScenariosRemove
}
func NewCmdScenariosUpgrade() *cobra.Command {
var cmdScenariosUpgrade = &cobra.Command{
Use: "upgrade [config]",
Short: "Upgrade given scenario(s)",
Long: `Fetch and Upgrade given scenario(s) from hub`,
Example: `cscli scenarios upgrade crowdsec/xxx crowdsec/xyz`,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
} else {
if len(args) == 0 {
return fmt.Errorf("specify at least one scenario to upgrade or '--all'")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, forceAction)
}
}
return nil
},
}
cmdScenariosUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the scenarios")
cmdScenariosUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
return cmdScenariosUpgrade
}
func NewCmdScenariosInspect() *cobra.Command {
var cmdScenariosInspect = &cobra.Command{
Use: "inspect [config]",
Short: "Inspect given scenario",
Long: `Inspect given scenario`,
Example: `cscli scenarios inspect crowdsec/xxx`,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
InspectItem(args[0], cwhub.SCENARIOS)
},
}
cmdScenariosInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
return cmdScenariosInspect
}
func NewCmdScenariosList() *cobra.Command {
var cmdScenariosList = &cobra.Command{
Use: "list [config]",
Short: "List all scenario(s) or given one",
Long: `List all scenario(s) or given one`,
Example: `cscli scenarios list
cscli scenarios list crowdsecurity/xxx`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all)
},
}
cmdScenariosList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmdScenariosList
}

10
cmd/crowdsec-cli/setup.go
View file

@ -6,11 +6,12 @@ import (
"os"
"os/exec"
goccyyaml "github.com/goccy/go-yaml"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
goccyyaml "github.com/goccy/go-yaml"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/setup"
)
@ -303,7 +304,12 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error {
return fmt.Errorf("while reading file %s: %w", fromFile, err)
}
if err = setup.InstallHubItems(csConfig, input, dryRun); err != nil {
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), log.StandardLogger())
if err != nil {
return err
}
if err = setup.InstallHubItems(hub, input, dryRun); err != nil {
return err
}

308
cmd/crowdsec-cli/simulation.go
View file

@ -13,210 +13,128 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func addToExclusion(name string) error {
csConfig.Cscli.SimulationConfig.Exclusions = append(csConfig.Cscli.SimulationConfig.Exclusions, name)
return nil
type cliSimulation struct {
cfg configGetter
}
func removeFromExclusion(name string) error {
index := indexOf(name, csConfig.Cscli.SimulationConfig.Exclusions)
// Remove element from the slice
csConfig.Cscli.SimulationConfig.Exclusions[index] = csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1] = ""
csConfig.Cscli.SimulationConfig.Exclusions = csConfig.Cscli.SimulationConfig.Exclusions[:len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
return nil
func NewCLISimulation(cfg configGetter) *cliSimulation {
return &cliSimulation{
cfg: cfg,
}
}
func enableGlobalSimulation() error {
csConfig.Cscli.SimulationConfig.Simulation = new(bool)
*csConfig.Cscli.SimulationConfig.Simulation = true
csConfig.Cscli.SimulationConfig.Exclusions = []string{}
if err := dumpSimulationFile(); err != nil {
log.Fatalf("unable to dump simulation file: %s", err)
}
log.Printf("global simulation: enabled")
return nil
}
func dumpSimulationFile() error {
newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
if err != nil {
return fmt.Errorf("unable to marshal simulation configuration: %s", err)
}
err = os.WriteFile(csConfig.ConfigPaths.SimulationFilePath, newConfigSim, 0644)
if err != nil {
return fmt.Errorf("write simulation config in '%s' failed: %s", csConfig.ConfigPaths.SimulationFilePath, err)
}
log.Debugf("updated simulation file %s", csConfig.ConfigPaths.SimulationFilePath)
return nil
}
func disableGlobalSimulation() error {
csConfig.Cscli.SimulationConfig.Simulation = new(bool)
*csConfig.Cscli.SimulationConfig.Simulation = false
csConfig.Cscli.SimulationConfig.Exclusions = []string{}
newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
if err != nil {
return fmt.Errorf("unable to marshal new simulation configuration: %s", err)
}
err = os.WriteFile(csConfig.ConfigPaths.SimulationFilePath, newConfigSim, 0644)
if err != nil {
return fmt.Errorf("unable to write new simulation config in '%s' : %s", csConfig.ConfigPaths.SimulationFilePath, err)
}
log.Printf("global simulation: disabled")
return nil
}
func simulationStatus() error {
if csConfig.Cscli.SimulationConfig == nil {
log.Printf("global simulation: disabled (configuration file is missing)")
return nil
}
if *csConfig.Cscli.SimulationConfig.Simulation {
log.Println("global simulation: enabled")
if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
log.Println("Scenarios not in simulation mode :")
for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
log.Printf(" - %s", scenario)
}
}
} else {
log.Println("global simulation: disabled")
if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
log.Println("Scenarios in simulation mode :")
for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
log.Printf(" - %s", scenario)
}
}
}
return nil
}
func NewSimulationCmds() *cobra.Command {
var cmdSimulation = &cobra.Command{
func (cli *cliSimulation) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "simulation [command]",
Short: "Manage simulation status of scenarios",
Example: `cscli simulation status
cscli simulation enable crowdsecurity/ssh-bf
cscli simulation disable crowdsecurity/ssh-bf`,
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadSimulation(); err != nil {
log.Fatal(err)
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
if err := cli.cfg().LoadSimulation(); err != nil {
return err
}
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before using simulation")
}
if csConfig.Cscli.SimulationConfig == nil {
if cli.cfg().Cscli.SimulationConfig == nil {
return fmt.Errorf("no simulation configured")
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
PersistentPostRun: func(cmd *cobra.Command, _ []string) {
if cmd.Name() != "status" {
log.Infof(ReloadMessage())
}
},
}
cmdSimulation.Flags().SortFlags = false
cmdSimulation.PersistentFlags().SortFlags = false
cmd.Flags().SortFlags = false
cmd.PersistentFlags().SortFlags = false
cmdSimulation.AddCommand(NewSimulationEnableCmd())
cmdSimulation.AddCommand(NewSimulationDisableCmd())
cmdSimulation.AddCommand(NewSimulationStatusCmd())
cmd.AddCommand(cli.NewEnableCmd())
cmd.AddCommand(cli.NewDisableCmd())
cmd.AddCommand(cli.NewStatusCmd())
return cmdSimulation
return cmd
}
func NewSimulationEnableCmd() *cobra.Command {
func (cli *cliSimulation) NewEnableCmd() *cobra.Command {
var forceGlobalSimulation bool
var cmdSimulationEnable = &cobra.Command{
cmd := &cobra.Command{
Use: "enable [scenario] [-global]",
Short: "Enable the simulation, globally or on specified scenarios",
Example: `cscli simulation enable`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if err := require.Hub(csConfig); err != nil {
log.Fatal(err)
RunE: func(cmd *cobra.Command, args []string) error {
hub, err := require.Hub(cli.cfg(), nil, nil)
if err != nil {
return err
}
if len(args) > 0 {
for _, scenario := range args {
var item = cwhub.GetItem(cwhub.SCENARIOS, scenario)
var item = hub.GetItem(cwhub.SCENARIOS, scenario)
if item == nil {
log.Errorf("'%s' doesn't exist or is not a scenario", scenario)
continue
}
if !item.Installed {
if !item.State.Installed {
log.Warningf("'%s' isn't enabled", scenario)
}
isExcluded := slices.Contains(csConfig.Cscli.SimulationConfig.Exclusions, scenario)
if *csConfig.Cscli.SimulationConfig.Simulation && !isExcluded {
isExcluded := slices.Contains(cli.cfg().Cscli.SimulationConfig.Exclusions, scenario)
if *cli.cfg().Cscli.SimulationConfig.Simulation && !isExcluded {
log.Warning("global simulation is already enabled")
continue
}
if !*csConfig.Cscli.SimulationConfig.Simulation && isExcluded {
if !*cli.cfg().Cscli.SimulationConfig.Simulation && isExcluded {
log.Warningf("simulation for '%s' already enabled", scenario)
continue
}
if *csConfig.Cscli.SimulationConfig.Simulation && isExcluded {
if err := removeFromExclusion(scenario); err != nil {
log.Fatal(err)
}
if *cli.cfg().Cscli.SimulationConfig.Simulation && isExcluded {
cli.removeFromExclusion(scenario)
log.Printf("simulation enabled for '%s'", scenario)
continue
}
if err := addToExclusion(scenario); err != nil {
log.Fatal(err)
}
cli.addToExclusion(scenario)
log.Printf("simulation mode for '%s' enabled", scenario)
}
if err := dumpSimulationFile(); err != nil {
log.Fatalf("simulation enable: %s", err)
if err := cli.dumpSimulationFile(); err != nil {
return fmt.Errorf("simulation enable: %s", err)
}
} else if forceGlobalSimulation {
if err := enableGlobalSimulation(); err != nil {
log.Fatalf("unable to enable global simulation mode : %s", err)
if err := cli.enableGlobalSimulation(); err != nil {
return fmt.Errorf("unable to enable global simulation mode: %s", err)
}
} else {
printHelp(cmd)
}
return nil
},
}
cmdSimulationEnable.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Enable global simulation (reverse mode)")
cmd.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Enable global simulation (reverse mode)")
return cmdSimulationEnable
return cmd
}
func NewSimulationDisableCmd() *cobra.Command {
func (cli *cliSimulation) NewDisableCmd() *cobra.Command {
var forceGlobalSimulation bool
var cmdSimulationDisable = &cobra.Command{
cmd := &cobra.Command{
Use: "disable [scenario]",
Short: "Disable the simulation mode. Disable only specified scenarios",
Example: `cscli simulation disable`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) > 0 {
for _, scenario := range args {
isExcluded := slices.Contains(csConfig.Cscli.SimulationConfig.Exclusions, scenario)
if !*csConfig.Cscli.SimulationConfig.Simulation && !isExcluded {
isExcluded := slices.Contains(cli.cfg().Cscli.SimulationConfig.Exclusions, scenario)
if !*cli.cfg().Cscli.SimulationConfig.Simulation && !isExcluded {
log.Warningf("%s isn't in simulation mode", scenario)
continue
}
if !*csConfig.Cscli.SimulationConfig.Simulation && isExcluded {
if err := removeFromExclusion(scenario); err != nil {
log.Fatal(err)
}
if !*cli.cfg().Cscli.SimulationConfig.Simulation && isExcluded {
cli.removeFromExclusion(scenario)
log.Printf("simulation mode for '%s' disabled", scenario)
continue
}
@ -224,42 +142,138 @@ func NewSimulationDisableCmd() *cobra.Command {
log.Warningf("simulation mode is enabled but is already disable for '%s'", scenario)
continue
}
if err := addToExclusion(scenario); err != nil {
log.Fatal(err)
}
cli.addToExclusion(scenario)
log.Printf("simulation mode for '%s' disabled", scenario)
}
if err := dumpSimulationFile(); err != nil {
log.Fatalf("simulation disable: %s", err)
if err := cli.dumpSimulationFile(); err != nil {
return fmt.Errorf("simulation disable: %s", err)
}
} else if forceGlobalSimulation {
if err := disableGlobalSimulation(); err != nil {
log.Fatalf("unable to disable global simulation mode : %s", err)
if err := cli.disableGlobalSimulation(); err != nil {
return fmt.Errorf("unable to disable global simulation mode: %s", err)
}
} else {
printHelp(cmd)
}
return nil
},
}
cmdSimulationDisable.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Disable global simulation (reverse mode)")
cmd.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Disable global simulation (reverse mode)")
return cmdSimulationDisable
return cmd
}
func NewSimulationStatusCmd() *cobra.Command {
var cmdSimulationStatus = &cobra.Command{
func (cli *cliSimulation) NewStatusCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "status",
Short: "Show simulation mode status",
Example: `cscli simulation status`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if err := simulationStatus(); err != nil {
log.Fatal(err)
}
Run: func(_ *cobra.Command, _ []string) {
cli.status()
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
},
}
return cmdSimulationStatus
return cmd
}
func (cli *cliSimulation) addToExclusion(name string) {
cfg := cli.cfg()
cfg.Cscli.SimulationConfig.Exclusions = append(cfg.Cscli.SimulationConfig.Exclusions, name)
}
func (cli *cliSimulation) removeFromExclusion(name string) {
cfg := cli.cfg()
index := slices.Index(cfg.Cscli.SimulationConfig.Exclusions, name)
// Remove element from the slice
cfg.Cscli.SimulationConfig.Exclusions[index] = cfg.Cscli.SimulationConfig.Exclusions[len(cfg.Cscli.SimulationConfig.Exclusions)-1]
cfg.Cscli.SimulationConfig.Exclusions[len(cfg.Cscli.SimulationConfig.Exclusions)-1] = ""
cfg.Cscli.SimulationConfig.Exclusions = cfg.Cscli.SimulationConfig.Exclusions[:len(cfg.Cscli.SimulationConfig.Exclusions)-1]
}
func (cli *cliSimulation) enableGlobalSimulation() error {
cfg := cli.cfg()
cfg.Cscli.SimulationConfig.Simulation = new(bool)
*cfg.Cscli.SimulationConfig.Simulation = true
cfg.Cscli.SimulationConfig.Exclusions = []string{}
if err := cli.dumpSimulationFile(); err != nil {
return fmt.Errorf("unable to dump simulation file: %s", err)
}
log.Printf("global simulation: enabled")
return nil
}
func (cli *cliSimulation) dumpSimulationFile() error {
cfg := cli.cfg()
newConfigSim, err := yaml.Marshal(cfg.Cscli.SimulationConfig)
if err != nil {
return fmt.Errorf("unable to marshal simulation configuration: %s", err)
}
err = os.WriteFile(cfg.ConfigPaths.SimulationFilePath, newConfigSim, 0o644)
if err != nil {
return fmt.Errorf("write simulation config in '%s' failed: %s", cfg.ConfigPaths.SimulationFilePath, err)
}
log.Debugf("updated simulation file %s", cfg.ConfigPaths.SimulationFilePath)
return nil
}
func (cli *cliSimulation) disableGlobalSimulation() error {
cfg := cli.cfg()
cfg.Cscli.SimulationConfig.Simulation = new(bool)
*cfg.Cscli.SimulationConfig.Simulation = false
cfg.Cscli.SimulationConfig.Exclusions = []string{}
newConfigSim, err := yaml.Marshal(cfg.Cscli.SimulationConfig)
if err != nil {
return fmt.Errorf("unable to marshal new simulation configuration: %s", err)
}
err = os.WriteFile(cfg.ConfigPaths.SimulationFilePath, newConfigSim, 0o644)
if err != nil {
return fmt.Errorf("unable to write new simulation config in '%s': %s", cfg.ConfigPaths.SimulationFilePath, err)
}
log.Printf("global simulation: disabled")
return nil
}
func (cli *cliSimulation) status() {
cfg := cli.cfg()
if cfg.Cscli.SimulationConfig == nil {
log.Printf("global simulation: disabled (configuration file is missing)")
return
}
if *cfg.Cscli.SimulationConfig.Simulation {
log.Println("global simulation: enabled")
if len(cfg.Cscli.SimulationConfig.Exclusions) > 0 {
log.Println("Scenarios not in simulation mode :")
for _, scenario := range cfg.Cscli.SimulationConfig.Exclusions {
log.Printf(" - %s", scenario)
}
}
} else {
log.Println("global simulation: disabled")
if len(cfg.Cscli.SimulationConfig.Exclusions) > 0 {
log.Println("Scenarios in simulation mode :")
for _, scenario := range cfg.Cscli.SimulationConfig.Exclusions {
log.Printf(" - %s", scenario)
}
}
}
}

127
cmd/crowdsec-cli/support.go
View file

@ -37,6 +37,7 @@ const (
SUPPORT_OS_INFO_PATH = "osinfo.txt"
SUPPORT_PARSERS_PATH = "hub/parsers.txt"
SUPPORT_SCENARIOS_PATH = "hub/scenarios.txt"
SUPPORT_CONTEXTS_PATH = "hub/scenarios.txt"
SUPPORT_COLLECTIONS_PATH = "hub/collections.txt"
SUPPORT_POSTOVERFLOWS_PATH = "hub/postoverflows.txt"
SUPPORT_BOUNCERS_PATH = "lapi/bouncers.txt"
@ -58,10 +59,6 @@ func stripAnsiString(str string) string {
func collectMetrics() ([]byte, []byte, error) {
log.Info("Collecting prometheus metrics")
err := csConfig.LoadPrometheus()
if err != nil {
return nil, nil, err
}
if csConfig.Cscli.PrometheusUrl == "" {
log.Warn("No Prometheus URL configured, metrics will not be collected")
@ -69,19 +66,25 @@ func collectMetrics() ([]byte, []byte, error) {
}
humanMetrics := bytes.NewBuffer(nil)
err = FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl+"/metrics", "human")
if err != nil {
return nil, nil, fmt.Errorf("could not fetch promtheus metrics: %s", err)
ms := NewMetricStore()
if err := ms.Fetch(csConfig.Cscli.PrometheusUrl); err != nil {
return nil, nil, fmt.Errorf("could not fetch prometheus metrics: %s", err)
}
req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl+"/metrics", nil)
if err := ms.Format(humanMetrics, nil, "human", false); err != nil {
return nil, nil, err
}
req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl, nil)
if err != nil {
return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err)
}
client := &http.Client{}
resp, err := client.Do(req)
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, nil, fmt.Errorf("could not get metrics from prometheus endpoint: %s", err)
}
@ -103,17 +106,20 @@ func collectVersion() []byte {
func collectFeatures() []byte {
log.Info("Collecting feature flags")
enabledFeatures := fflag.Crowdsec.GetEnabledFeatures()
w := bytes.NewBuffer(nil)
for _, k := range enabledFeatures {
fmt.Fprintf(w, "%s\n", k)
}
return w.Bytes()
}
func collectOSInfo() ([]byte, error) {
log.Info("Collecting OS info")
info, err := osinfo.GetOSInfo()
if err != nil {
@ -132,42 +138,65 @@ func collectOSInfo() ([]byte, error) {
return w.Bytes(), nil
}
func collectHubItems(itemType string) []byte {
func collectHubItems(hub *cwhub.Hub, itemType string) []byte {
var err error
out := bytes.NewBuffer(nil)
log.Infof("Collecting %s list", itemType)
ListItems(out, []string{itemType}, []string{}, false, true, all)
items := make(map[string][]*cwhub.Item)
if items[itemType], err = selectItems(hub, itemType, nil, true); err != nil {
log.Warnf("could not collect %s list: %s", itemType, err)
}
if err := listItems(out, []string{itemType}, items, false); err != nil {
log.Warnf("could not collect %s list: %s", itemType, err)
}
return out.Bytes()
}
func collectBouncers(dbClient *database.Client) ([]byte, error) {
out := bytes.NewBuffer(nil)
err := getBouncers(out, dbClient)
bouncers, err := dbClient.ListBouncers()
if err != nil {
return nil, err
return nil, fmt.Errorf("unable to list bouncers: %s", err)
}
getBouncersTable(out, bouncers)
return out.Bytes(), nil
}
func collectAgents(dbClient *database.Client) ([]byte, error) {
out := bytes.NewBuffer(nil)
err := getAgents(out, dbClient)
machines, err := dbClient.ListMachines()
if err != nil {
return nil, err
return nil, fmt.Errorf("unable to list machines: %s", err)
}
getAgentsTable(out, machines)
return out.Bytes(), nil
}
func collectAPIStatus(login string, password string, endpoint string, prefix string) []byte {
func collectAPIStatus(login string, password string, endpoint string, prefix string, hub *cwhub.Hub) []byte {
if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil {
return []byte("No agent credentials found, are we LAPI ?")
}
pwd := strfmt.Password(password)
apiurl, err := url.Parse(endpoint)
pwd := strfmt.Password(password)
apiurl, err := url.Parse(endpoint)
if err != nil {
return []byte(fmt.Sprintf("cannot parse API URL: %s", err))
}
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return []byte(fmt.Sprintf("could not collect scenarios: %s", err))
}
@ -179,6 +208,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
if err != nil {
return []byte(fmt.Sprintf("could not init client: %s", err))
}
t := models.WatcherAuthRequest{
MachineID: &login,
Password: &pwd,
@ -195,6 +225,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
func collectCrowdsecConfig() []byte {
log.Info("Collecting crowdsec config")
config, err := os.ReadFile(*csConfig.FilePath)
if err != nil {
return []byte(fmt.Sprintf("could not read config file: %s", err))
@ -207,15 +238,18 @@ func collectCrowdsecConfig() []byte {
func collectCrowdsecProfile() []byte {
log.Info("Collecting crowdsec profile")
config, err := os.ReadFile(csConfig.API.Server.ProfilesPath)
if err != nil {
return []byte(fmt.Sprintf("could not read profile file: %s", err))
}
return config
}
func collectAcquisitionConfig() map[string][]byte {
log.Info("Collecting acquisition config")
ret := make(map[string][]byte)
for _, filename := range csConfig.Crowdsec.AcquisitionFiles {
@ -230,8 +264,14 @@ func collectAcquisitionConfig() map[string][]byte {
return ret
}
func NewSupportCmd() *cobra.Command {
var cmdSupport = &cobra.Command{
type cliSupport struct{}
func NewCLISupport() *cliSupport {
return &cliSupport{}
}
func (cli cliSupport) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "support [action]",
Short: "Provide commands to help during support",
Args: cobra.MinimumNArgs(1),
@ -241,9 +281,15 @@ func NewSupportCmd() *cobra.Command {
},
}
cmd.AddCommand(cli.NewDumpCmd())
return cmd
}
func (cli cliSupport) NewDumpCmd() *cobra.Command {
var outFile string
cmdDump := &cobra.Command{
cmd := &cobra.Command{
Use: "dump",
Short: "Dump all your configuration to a zip file for easier support",
Long: `Dump the following informations:
@ -253,6 +299,7 @@ func NewSupportCmd() *cobra.Command {
- Installed parsers list
- Installed scenarios list
- Installed postoverflows list
- Installed context list
- Bouncers list
- Machines list
- CAPI status
@ -264,7 +311,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
`,
Args: cobra.NoArgs,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
Run: func(_ *cobra.Command, _ []string) {
var err error
var skipHub, skipDB, skipCAPI, skipLAPI, skipAgent bool
infos := map[string][]byte{
@ -284,23 +331,25 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_AGENTS_PATH] = []byte(err.Error())
}
if err := csConfig.LoadAPIServer(); err != nil {
if err = csConfig.LoadAPIServer(true); err != nil {
log.Warnf("could not load LAPI, skipping CAPI check")
skipLAPI = true
infos[SUPPORT_CAPI_STATUS_PATH] = []byte(err.Error())
}
if err := csConfig.LoadCrowdsec(); err != nil {
if err = csConfig.LoadCrowdsec(); err != nil {
log.Warnf("could not load agent config, skipping crowdsec config check")
skipAgent = true
}
if err := require.Hub(csConfig); err != nil {
hub, err := require.Hub(csConfig, nil, nil)
if err != nil {
log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
skipHub = true
infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())
infos[SUPPORT_SCENARIOS_PATH] = []byte(err.Error())
infos[SUPPORT_POSTOVERFLOWS_PATH] = []byte(err.Error())
infos[SUPPORT_CONTEXTS_PATH] = []byte(err.Error())
infos[SUPPORT_COLLECTIONS_PATH] = []byte(err.Error())
}
@ -333,10 +382,11 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_CROWDSEC_CONFIG_PATH] = collectCrowdsecConfig()
if !skipHub {
infos[SUPPORT_PARSERS_PATH] = collectHubItems(cwhub.PARSERS)
infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(cwhub.SCENARIOS)
infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.PARSERS_OVFLW)
infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(cwhub.COLLECTIONS)
infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS)
infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS)
infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
infos[SUPPORT_CONTEXTS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS)
}
if !skipDB {
@ -358,7 +408,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_CAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Server.OnlineClient.Credentials.Login,
csConfig.API.Server.OnlineClient.Credentials.Password,
csConfig.API.Server.OnlineClient.Credentials.URL,
CAPIURLPrefix)
CAPIURLPrefix,
hub)
}
if !skipLAPI {
@ -366,12 +417,12 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_LAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Client.Credentials.Login,
csConfig.API.Client.Credentials.Password,
csConfig.API.Client.Credentials.URL,
LAPIURLPrefix)
LAPIURLPrefix,
hub)
infos[SUPPORT_CROWDSEC_PROFILE_PATH] = collectCrowdsecProfile()
}
if !skipAgent {
acquis := collectAcquisitionConfig()
for filename, content := range acquis {
@ -397,7 +448,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
log.Fatalf("could not finalize zip file: %s", err)
}
err = os.WriteFile(outFile, w.Bytes(), 0600)
err = os.WriteFile(outFile, w.Bytes(), 0o600)
if err != nil {
log.Fatalf("could not write zip file to %s: %s", outFile, err)
}
@ -405,8 +456,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
log.Infof("Written zip file to %s", outFile)
},
}
cmdDump.Flags().StringVarP(&outFile, "outFile", "f", "", "File to dump the information to")
cmdSupport.AddCommand(cmdDump)
return cmdSupport
cmd.Flags().StringVarP(&outFile, "outFile", "f", "", "File to dump the information to")
return cmd
}

669
cmd/crowdsec-cli/utils.go
View file

@ -1,264 +1,23 @@
package main
import (
"encoding/csv"
"encoding/json"
"fmt"
"io"
"math"
"net"
"net/http"
"os"
"slices"
"strconv"
"strings"
"time"
"github.com/fatih/color"
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/prom2json"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/agext/levenshtein"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
const MaxDistance = 7
func printHelp(cmd *cobra.Command) {
err := cmd.Help()
if err != nil {
if err := cmd.Help(); err != nil {
log.Fatalf("unable to print help(): %s", err)
}
}
func indexOf(s string, slice []string) int {
for i, elem := range slice {
if s == elem {
return i
}
}
return -1
}
func LoadHub() error {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if csConfig.Hub == nil {
return fmt.Errorf("unable to load hub")
}
if err := cwhub.SetHubBranch(); err != nil {
log.Warningf("unable to set hub branch (%s), default to master", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
return fmt.Errorf("Failed to get Hub index : '%w'. Run 'sudo cscli hub update' to get the hub index", err)
}
return nil
}
func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
errMsg := ""
if score < MaxDistance {
errMsg = fmt.Sprintf("unable to find %s '%s', did you mean %s ?", itemType, baseItem, suggestItem)
} else {
errMsg = fmt.Sprintf("unable to find %s '%s'", itemType, baseItem)
}
if ignoreErr {
log.Error(errMsg)
} else {
log.Fatalf(errMsg)
}
}
func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
allItems := make([]string, 0)
nearestScore := 100
nearestItem := &cwhub.Item{}
hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
for _, item := range hubItems {
allItems = append(allItems, item.Name)
}
for _, s := range allItems {
d := levenshtein.Distance(itemName, s, nil)
if d < nearestScore {
nearestScore = d
nearestItem = cwhub.GetItem(itemType, s)
}
}
return nearestItem, nearestScore
}
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := LoadHub(); err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
for _, item := range hubItems {
if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
comp = append(comp, item.Name)
}
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := LoadHub(); err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
items, err := cwhub.GetInstalledItemsAsString(itemType)
if err != nil {
cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
if toComplete != "" {
for _, item := range items {
if strings.Contains(item, toComplete) {
comp = append(comp, item)
}
}
} else {
comp = items
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}
func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) {
var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus)
for _, itemType := range itemTypes {
itemName := ""
if len(args) == 1 {
itemName = args[0]
}
hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all)
}
if csConfig.Cscli.Output == "human" {
for _, itemType := range itemTypes {
var statuses []cwhub.ItemHubStatus
var ok bool
if statuses, ok = hubStatusByItemType[itemType]; !ok {
log.Errorf("unknown item type: %s", itemType)
continue
}
listHubItemTable(out, "\n"+strings.ToUpper(itemType), statuses)
}
} else if csConfig.Cscli.Output == "json" {
x, err := json.MarshalIndent(hubStatusByItemType, "", " ")
if err != nil {
log.Fatalf("failed to unmarshal")
}
out.Write(x)
} else if csConfig.Cscli.Output == "raw" {
csvwriter := csv.NewWriter(out)
if showHeader {
header := []string{"name", "status", "version", "description"}
if showType {
header = append(header, "type")
}
err := csvwriter.Write(header)
if err != nil {
log.Fatalf("failed to write header: %s", err)
}
}
for _, itemType := range itemTypes {
var statuses []cwhub.ItemHubStatus
var ok bool
if statuses, ok = hubStatusByItemType[itemType]; !ok {
log.Errorf("unknown item type: %s", itemType)
continue
}
for _, status := range statuses {
if status.LocalVersion == "" {
status.LocalVersion = "n/a"
}
row := []string{
status.Name,
status.Status,
status.LocalVersion,
status.Description,
}
if showType {
row = append(row, itemType)
}
err := csvwriter.Write(row)
if err != nil {
log.Fatalf("failed to write raw output : %s", err)
}
}
}
csvwriter.Flush()
}
}
func InspectItem(name string, objecitemType string) {
hubItem := cwhub.GetItem(objecitemType, name)
if hubItem == nil {
log.Fatalf("unable to retrieve item.")
}
var b []byte
var err error
switch csConfig.Cscli.Output {
case "human", "raw":
b, err = yaml.Marshal(*hubItem)
if err != nil {
log.Fatalf("unable to marshal item : %s", err)
}
case "json":
b, err = json.MarshalIndent(*hubItem, "", " ")
if err != nil {
log.Fatalf("unable to marshal item : %s", err)
}
}
fmt.Printf("%s", string(b))
if csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
return
}
if prometheusURL == "" {
//This is technically wrong to do this, as the prometheus section contains a listen address, not an URL to query prometheus
//But for ease of use, we will use the listen address as the prometheus URL because it will be 127.0.0.1 in the default case
listenAddr := csConfig.Prometheus.ListenAddr
if listenAddr == "" {
listenAddr = "127.0.0.1"
}
listenPort := csConfig.Prometheus.ListenPort
if listenPort == 0 {
listenPort = 6060
}
prometheusURL = fmt.Sprintf("http://%s:%d/metrics", listenAddr, listenPort)
log.Debugf("No prometheus URL provided using: %s", prometheusURL)
}
fmt.Printf("\nCurrent metrics : \n")
ShowMetrics(hubItem)
}
func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
/*if a range is provided, change the scope*/
if *ipRange != "" {
_, _, err := net.ParseCIDR(*ipRange)
@ -266,6 +25,7 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
return fmt.Errorf("%s isn't a valid range", *ipRange)
}
}
if *ip != "" {
ipRepr := net.ParseIP(*ip)
if ipRepr == nil {
@ -273,7 +33,7 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
}
}
//avoid confusion on scope (ip vs Ip and range vs Range)
// avoid confusion on scope (ip vs Ip and range vs Range)
switch strings.ToLower(*scope) {
case "ip":
*scope = types.Ip
@ -284,432 +44,10 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
case "as":
*scope = types.AS
}
return nil
}
func ShowMetrics(hubItem *cwhub.Item) {
switch hubItem.Type {
case cwhub.PARSERS:
metrics := GetParserMetric(prometheusURL, hubItem.Name)
parserMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.SCENARIOS:
metrics := GetScenarioMetric(prometheusURL, hubItem.Name)
scenarioMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.COLLECTIONS:
for _, item := range hubItem.Parsers {
metrics := GetParserMetric(prometheusURL, item)
parserMetricsTable(color.Output, item, metrics)
}
for _, item := range hubItem.Scenarios {
metrics := GetScenarioMetric(prometheusURL, item)
scenarioMetricsTable(color.Output, item, metrics)
}
for _, item := range hubItem.Collections {
hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
if hubItem == nil {
log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
}
ShowMetrics(hubItem)
}
default:
log.Errorf("item of type '%s' is unknown", hubItem.Type)
}
}
// GetParserMetric is a complete rip from prom2json
func GetParserMetric(url string, itemName string) map[string]map[string]int {
stats := make(map[string]map[string]int)
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
source, ok := metric.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v", metric.Labels)
} else {
if srctype, ok := metric.Labels["type"]; ok {
source = srctype + ":" + source
}
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_reader_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
stats[source]["parsed"] = 0
stats[source]["reads"] = 0
stats[source]["unparsed"] = 0
stats[source]["hits"] = 0
}
stats[source]["reads"] += ival
case "cs_parser_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_parser_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
case "cs_node_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["hits"] += ival
case "cs_node_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_node_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
default:
continue
}
}
}
return stats
}
func GetScenarioMetric(url string, itemName string) map[string]int {
stats := make(map[string]int)
stats["instantiation"] = 0
stats["curr_count"] = 0
stats["overflow"] = 0
stats["pour"] = 0
stats["underflow"] = 0
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_bucket_created_total":
stats["instantiation"] += ival
case "cs_buckets":
stats["curr_count"] += ival
case "cs_bucket_overflowed_total":
stats["overflow"] += ival
case "cs_bucket_poured_total":
stats["pour"] += ival
case "cs_bucket_underflowed_total":
stats["underflow"] += ival
default:
continue
}
}
}
return stats
}
// it's a rip of the cli version, but in silent-mode
func silenceInstallItem(name string, obtype string) (string, error) {
var item = cwhub.GetItem(obtype, name)
if item == nil {
return "", fmt.Errorf("error retrieving item")
}
it := *item
if downloadOnly && it.Downloaded && it.UpToDate {
return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
}
it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction, false)
if err != nil {
return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
}
if err := cwhub.AddItem(obtype, it); err != nil {
return "", err
}
if downloadOnly {
return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
}
it, err = cwhub.EnableItem(csConfig.Hub, it)
if err != nil {
return "", fmt.Errorf("error while enabling %s : %v", it.Name, err)
}
if err := cwhub.AddItem(obtype, it); err != nil {
return "", err
}
return fmt.Sprintf("Enabled %s", it.Name), nil
}
func GetPrometheusMetric(url string) []*prom2json.Family {
mfChan := make(chan *dto.MetricFamily, 1024)
// Start with the DefaultTransport for sane defaults.
transport := http.DefaultTransport.(*http.Transport).Clone()
// Conservatively disable HTTP keep-alives as this program will only
// ever need a single HTTP request.
transport.DisableKeepAlives = true
// Timeout early if the server doesn't even return the headers.
transport.ResponseHeaderTimeout = time.Minute
go func() {
defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
err := prom2json.FetchMetricFamilies(url, mfChan, transport)
if err != nil {
log.Fatalf("failed to fetch prometheus metrics : %v", err)
}
}()
result := []*prom2json.Family{}
for mf := range mfChan {
result = append(result, prom2json.NewFamily(mf))
}
log.Debugf("Finished reading prometheus output, %d entries", len(result))
return result
}
func RestoreHub(dirPath string) error {
var err error
if err := csConfig.LoadHub(); err != nil {
return err
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
for _, itype := range cwhub.ItemTypes {
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
if _, err = os.Stat(itemDirectory); err != nil {
log.Infof("no %s in backup", itype)
continue
}
/*restore the upstream items*/
upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
file, err := os.ReadFile(upstreamListFN)
if err != nil {
return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
}
var upstreamList []string
err = json.Unmarshal(file, &upstreamList)
if err != nil {
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
}
for _, toinstall := range upstreamList {
label, err := silenceInstallItem(toinstall, itype)
if err != nil {
log.Errorf("Error while installing %s : %s", toinstall, err)
} else if label != "" {
log.Infof("Installed %s : %s", toinstall, label)
} else {
log.Printf("Installed %s : ok", toinstall)
}
}
/*restore the local and tainted items*/
files, err := os.ReadDir(itemDirectory)
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
}
for _, file := range files {
//this was the upstream data
if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
continue
}
if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
//we expect a stage here
if !file.IsDir() {
continue
}
stage := file.Name()
stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
}
/*find items*/
ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
}
//finally copy item
for _, tfile := range ifiles {
log.Infof("Going to restore local/tainted [%s]", tfile.Name())
sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
} else {
log.Infof("Going to restore local/tainted [%s]", file.Name())
sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
}
}
return nil
}
func BackupHub(dirPath string) error {
var err error
var itemDirectory string
var upstreamParsers []string
for _, itemType := range cwhub.ItemTypes {
clog := log.WithFields(log.Fields{
"type": itemType,
})
itemMap := cwhub.GetItemMap(itemType)
if itemMap == nil {
clog.Infof("No %s to backup.", itemType)
continue
}
itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
}
upstreamParsers = []string{}
for k, v := range itemMap {
clog = clog.WithFields(log.Fields{
"file": v.Name,
})
if !v.Installed { //only backup installed ones
clog.Debugf("[%s] : not installed", k)
continue
}
//for the local/tainted ones, we backup the full file
if v.Tainted || v.Local || !v.UpToDate {
//we need to backup stages for parsers
if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
}
}
clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
if err = CopyFile(v.LocalPath, tfile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
}
clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
continue
}
clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
upstreamParsers = append(upstreamParsers, v.Name)
}
//write the upstream items
upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
if err != nil {
return fmt.Errorf("failed marshaling upstream parsers : %s", err)
}
err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
if err != nil {
return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
}
clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
}
return nil
}
type unit struct {
value int64
symbol string
}
var ranges = []unit{
{value: 1e18, symbol: "E"},
{value: 1e15, symbol: "P"},
{value: 1e12, symbol: "T"},
{value: 1e9, symbol: "G"},
{value: 1e6, symbol: "M"},
{value: 1e3, symbol: "k"},
{value: 1, symbol: ""},
}
func formatNumber(num int) string {
goodUnit := unit{}
for _, u := range ranges {
if int64(num) >= u.value {
goodUnit = u
break
}
}
if goodUnit.value == 1 {
return fmt.Sprintf("%d%s", num, goodUnit.symbol)
}
res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
}
func getDBClient() (*database.Client, error) {
var err error
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
return nil, err
}
ret, err := database.NewClient(csConfig.DbConfig)
if err != nil {
return nil, err
}
return ret, nil
}
func removeFromSlice(val string, slice []string) []string {
var i int
var value string
@ -731,5 +69,4 @@ func removeFromSlice(val string, slice []string) []string {
}
return slice
}

45
cmd/crowdsec-cli/utils_table.go
View file

@ -3,6 +3,7 @@ package main
import (
"fmt"
"io"
"strconv"
"github.com/aquasecurity/table"
"github.com/enescakir/emoji"
@ -10,19 +11,33 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatus) {
func listHubItemTable(out io.Writer, title string, items []*cwhub.Item) {
t := newLightTable(out)
t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
for _, status := range statuses {
t.AddRow(status.Name, status.UTF8_Status, status.LocalVersion, status.LocalPath)
for _, item := range items {
status := fmt.Sprintf("%v %s", item.State.Emoji(), item.State.Text())
t.AddRow(item.Name, status, item.State.LocalVersion, item.State.LocalPath)
}
renderTableTitle(out, title)
t.Render()
}
func appsecMetricsTable(out io.Writer, itemName string, metrics map[string]int) {
t := newTable(out)
t.SetHeaders("Inband Hits", "Outband Hits")
t.AddRow(
strconv.Itoa(metrics["inband_hits"]),
strconv.Itoa(metrics["outband_hits"]),
)
renderTableTitle(out, fmt.Sprintf("\n - (AppSec Rule) %s:", itemName))
t.Render()
}
func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int) {
if metrics["instantiation"] == 0 {
return
@ -31,11 +46,11 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
t.SetHeaders("Current Count", "Overflows", "Instantiated", "Poured", "Expired")
t.AddRow(
fmt.Sprintf("%d", metrics["curr_count"]),
fmt.Sprintf("%d", metrics["overflow"]),
fmt.Sprintf("%d", metrics["instantiation"]),
fmt.Sprintf("%d", metrics["pour"]),
fmt.Sprintf("%d", metrics["underflow"]),
strconv.Itoa(metrics["curr_count"]),
strconv.Itoa(metrics["overflow"]),
strconv.Itoa(metrics["instantiation"]),
strconv.Itoa(metrics["pour"]),
strconv.Itoa(metrics["underflow"]),
)
renderTableTitle(out, fmt.Sprintf("\n - (Scenario) %s:", itemName))
@ -43,23 +58,25 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
}
func parserMetricsTable(out io.Writer, itemName string, metrics map[string]map[string]int) {
skip := true
t := newTable(out)
t.SetHeaders("Parsers", "Hits", "Parsed", "Unparsed")
// don't show table if no hits
showTable := false
for source, stats := range metrics {
if stats["hits"] > 0 {
t.AddRow(
source,
fmt.Sprintf("%d", stats["hits"]),
fmt.Sprintf("%d", stats["parsed"]),
fmt.Sprintf("%d", stats["unparsed"]),
strconv.Itoa(stats["hits"]),
strconv.Itoa(stats["parsed"]),
strconv.Itoa(stats["unparsed"]),
)
skip = false
showTable = true
}
}
if !skip {
if showTable {
renderTableTitle(out, fmt.Sprintf("\n - (Parser) %s:", itemName))
t.Render()
}

27
cmd/crowdsec-cli/version.go Normal file
View file

@ -0,0 +1,27 @@
package main
import (
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
)
type cliVersion struct{}
func NewCLIVersion() *cliVersion {
return &cliVersion{}
}
func (cli cliVersion) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "version",
Short: "Display version",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
Run: func(_ *cobra.Command, _ []string) {
cwversion.Show()
},
}
return cmd
}

26
cmd/crowdsec/crowdsec.go
View file

@ -13,6 +13,8 @@ import (
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/pkg/acquisition"
"github.com/crowdsecurity/crowdsec/pkg/appsec"
"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
@ -20,31 +22,35 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/types"
)
func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) {
var err error
// Populate cwhub package tools
if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
return nil, fmt.Errorf("while loading hub index: %w", err)
if err = alertcontext.LoadConsoleContext(cConfig, hub); err != nil {
return nil, fmt.Errorf("while loading context: %w", err)
}
// Start loading configs
csParsers := parser.NewParsers()
csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
return nil, fmt.Errorf("while loading parsers: %w", err)
}
if err := LoadBuckets(cConfig); err != nil {
if err := LoadBuckets(cConfig, hub); err != nil {
return nil, fmt.Errorf("while loading scenarios: %w", err)
}
if err := appsec.LoadAppsecRules(hub); err != nil {
return nil, fmt.Errorf("while loading appsec rules: %w", err)
}
if err := LoadAcquisition(cConfig); err != nil {
return nil, fmt.Errorf("while loading acquisition config: %w", err)
}
return csParsers, nil
}
func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.Hub) error {
inputEventChan = make(chan types.Event)
inputLineChan = make(chan types.Event)
@ -99,7 +105,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ {
outputsTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/runOutput")
if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials); err != nil {
if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials, hub); err != nil {
log.Fatalf("starting outputs error : %s", err)
return err
}
@ -131,7 +137,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
return nil
}
func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady chan bool) {
func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub.Hub, agentReady chan bool) {
crowdsecTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/serveCrowdsec")
go func() {
@ -139,7 +145,7 @@ func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady
// this logs every time, even at config reload
log.Debugf("running agent after %s ms", time.Since(crowdsecT0))
agentReady <- true
if err := runCrowdsec(cConfig, parsers); err != nil {
if err := runCrowdsec(cConfig, parsers, hub); err != nil {
log.Fatalf("unable to start crowdsec routines: %s", err)
}
}()

43
cmd/crowdsec/hook.go Normal file
View file

@ -0,0 +1,43 @@
package main
import (
"io"
"os"
log "github.com/sirupsen/logrus"
)
type ConditionalHook struct {
Writer io.Writer
LogLevels []log.Level
Enabled bool
}
func (hook *ConditionalHook) Fire(entry *log.Entry) error {
if hook.Enabled {
line, err := entry.String()
if err != nil {
return err
}
_, err = hook.Writer.Write([]byte(line))
return err
}
return nil
}
func (hook *ConditionalHook) Levels() []log.Level {
return hook.LogLevels
}
// The primal logging hook is set up before parsing config.yaml.
// Once config.yaml is parsed, the primal hook is disabled if the
// configured logger is writing to stderr. Otherwise it's used to
// report fatal errors and panics to stderr in addition to the log file.
var primalHook = &ConditionalHook{
Writer: os.Stderr,
LogLevels: []log.Level{log.FatalLevel, log.PanicLevel},
Enabled: true,
}

62
cmd/crowdsec/main.go
View file

@ -6,6 +6,7 @@ import (
_ "net/http/pprof"
"os"
"runtime"
"runtime/pprof"
"strings"
"time"
@ -71,24 +72,27 @@ type Flags struct {
DisableCAPI bool
Transform string
OrderEvent bool
CpuProfile string
}
type labelsMap map[string]string
func LoadBuckets(cConfig *csconfig.Config) error {
func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error {
var (
err error
files []string
)
for _, hubScenarioItem := range cwhub.GetItemMap(cwhub.SCENARIOS) {
if hubScenarioItem.Installed {
files = append(files, hubScenarioItem.LocalPath)
for _, hubScenarioItem := range hub.GetItemMap(cwhub.SCENARIOS) {
if hubScenarioItem.State.Installed {
files = append(files, hubScenarioItem.State.LocalPath)
}
}
buckets = leakybucket.NewBuckets()
log.Infof("Loading %d scenario files", len(files))
holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, files, &bucketsTomb, buckets, flags.OrderEvent)
holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, hub, files, &bucketsTomb, buckets, flags.OrderEvent)
if err != nil {
return fmt.Errorf("scenario loading failed: %v", err)
@ -99,6 +103,7 @@ func LoadBuckets(cConfig *csconfig.Config) error {
holders[holderIndex].Profiling = true
}
}
return nil
}
@ -143,8 +148,10 @@ func (l labelsMap) Set(label string) error {
if len(split) != 2 {
return fmt.Errorf("invalid format for label '%s', must be key:value", pair)
}
l[split[0]] = split[1]
}
return nil
}
@ -168,10 +175,13 @@ func (f *Flags) Parse() {
flag.BoolVar(&f.DisableAPI, "no-api", false, "disable local API")
flag.BoolVar(&f.DisableCAPI, "no-capi", false, "disable communication with Central API")
flag.BoolVar(&f.OrderEvent, "order-event", false, "enforce event ordering with significant performance cost")
if runtime.GOOS == "windows" {
flag.StringVar(&f.WinSvc, "winsvc", "", "Windows service Action: Install, Remove etc..")
}
flag.StringVar(&dumpFolder, "dump-data", "", "dump parsers/buckets raw outputs")
flag.StringVar(&f.CpuProfile, "cpu-profile", "", "write cpu profile to file")
flag.Parse()
}
@ -205,6 +215,7 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
// avoid returning a new ptr to the same value
return curLevelPtr
}
return &ret
}
@ -212,11 +223,7 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*csconfig.Config, error) {
cConfig, _, err := csconfig.NewConfig(configFile, disableAgent, disableAPI, quiet)
if err != nil {
return nil, err
}
if (cConfig.Common == nil || *cConfig.Common == csconfig.CommonCfg{}) {
return nil, fmt.Errorf("unable to load configuration: common section is empty")
return nil, fmt.Errorf("while loading configuration file: %w", err)
}
cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags)
@ -228,11 +235,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
dumpStates = true
}
// Configuration paths are dependency to load crowdsec configuration
if err := cConfig.LoadConfigurationPaths(); err != nil {
return nil, err
}
if flags.SingleFileType != "" && flags.OneShotDSN != "" {
// if we're in time-machine mode, we don't want to log to file
cConfig.Common.LogMedia = "stdout"
@ -247,6 +249,8 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
return nil, err
}
primalHook.Enabled = (cConfig.Common.LogMedia != "stdout")
if err := csconfig.LoadFeatureFlagsFile(configFile, log.StandardLogger()); err != nil {
return nil, err
}
@ -258,7 +262,7 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
}
if !cConfig.DisableAPI {
if err := cConfig.LoadAPIServer(); err != nil {
if err := cConfig.LoadAPIServer(false); err != nil {
return nil, err
}
}
@ -271,10 +275,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
return nil, errors.New("You must run at least the API Server or crowdsec")
}
if flags.TestMode && !cConfig.DisableAgent {
cConfig.Crowdsec.LintOnly = true
}
if flags.OneShotDSN != "" && flags.SingleFileType == "" {
return nil, errors.New("-dsn requires a -type argument")
}
@ -295,6 +295,7 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
if cConfig.DisableAPI {
cConfig.Common.Daemonize = false
}
log.Infof("single file mode : log_media=%s daemonize=%t", cConfig.Common.LogMedia, cConfig.Common.Daemonize)
}
@ -304,6 +305,7 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
if cConfig.Common.Daemonize && runtime.GOOS == "windows" {
log.Debug("Daemonization is not supported on Windows, disabling")
cConfig.Common.Daemonize = false
}
@ -321,6 +323,8 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
var crowdsecT0 time.Time
func main() {
log.AddHook(primalHook)
if err := fflag.RegisterAllFeatures(); err != nil {
log.Fatalf("failed to register features: %s", err)
}
@ -351,9 +355,25 @@ func main() {
os.Exit(0)
}
if flags.CpuProfile != "" {
f, err := os.Create(flags.CpuProfile)
if err != nil {
log.Fatalf("could not create CPU profile: %s", err)
}
log.Infof("CPU profile will be written to %s", flags.CpuProfile)
if err := pprof.StartCPUProfile(f); err != nil {
f.Close()
log.Fatalf("could not start CPU profile: %s", err)
}
defer f.Close()
defer pprof.StopCPUProfile()
}
err := StartRunSvc()
if err != nil {
log.Fatal(err)
pprof.StopCPUProfile()
log.Fatal(err) //nolint:gocritic // Disable warning for the defer pprof.StopCPUProfile() call
}
os.Exit(0)
}

16
cmd/crowdsec/metrics.go
View file

@ -151,14 +151,6 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
if !config.Enabled {
return
}
if config.ListenAddr == "" {
log.Warning("prometheus is enabled, but the listen address is empty, using '127.0.0.1'")
config.ListenAddr = "127.0.0.1"
}
if config.ListenPort == 0 {
log.Warning("prometheus is enabled, but the listen port is empty, using '6060'")
config.ListenPort = 6060
}
// Registering prometheus
// If in aggregated mode, do not register events associated with a source, to keep the cardinality low
@ -169,7 +161,8 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow,
v1.LapiRouteHits,
leaky.BucketsCurrentCount,
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics)
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, parser.NodesWlHitsOk, parser.NodesWlHits,
)
} else {
log.Infof("Loading prometheus collectors")
prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo,
@ -177,8 +170,9 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
globalCsInfo, globalParsingHistogram, globalPourHistogram,
v1.LapiRouteHits, v1.LapiMachineHits, v1.LapiBouncerHits, v1.LapiNilDecisions, v1.LapiNonNilDecisions, v1.LapiResponseTime,
leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, leaky.BucketsCurrentCount,
globalActiveDecisions, globalAlerts,
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics)
globalActiveDecisions, globalAlerts, parser.NodesWlHitsOk, parser.NodesWlHits,
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics,
)
}
}

60
cmd/crowdsec/output.go
View file

@ -62,7 +62,8 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error
var bucketOverflows []types.Event
func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets,
postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node, apiConfig csconfig.ApiCredentialsCfg) error {
postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node,
apiConfig csconfig.ApiCredentialsCfg, hub *cwhub.Hub) error {
var err error
ticker := time.NewTicker(1 * time.Second)
@ -70,11 +71,20 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
var cache []types.RuntimeAlert
var cacheMutex sync.Mutex
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("loading list of installed hub scenarios: %w", err)
}
appsecRules, err := hub.GetInstalledItemNames(cwhub.APPSEC_RULES)
if err != nil {
return fmt.Errorf("loading list of installed hub appsec rules: %w", err)
}
installedScenariosAndAppsecRules := make([]string, 0, len(scenarios)+len(appsecRules))
installedScenariosAndAppsecRules = append(installedScenariosAndAppsecRules, scenarios...)
installedScenariosAndAppsecRules = append(installedScenariosAndAppsecRules, appsecRules...)
apiURL, err := url.Parse(apiConfig.URL)
if err != nil {
return fmt.Errorf("parsing api url ('%s'): %w", apiConfig.URL, err)
@ -86,14 +96,27 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
password := strfmt.Password(apiConfig.Password)
Client, err := apiclient.NewClient(&apiclient.Config{
MachineID: apiConfig.Login,
Password: password,
Scenarios: scenarios,
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
PapiURL: papiURL,
VersionPrefix: "v1",
UpdateScenario: func() ([]string, error) {return cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)},
MachineID: apiConfig.Login,
Password: password,
Scenarios: installedScenariosAndAppsecRules,
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
PapiURL: papiURL,
VersionPrefix: "v1",
UpdateScenario: func() ([]string, error) {
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return nil, err
}
appsecRules, err := hub.GetInstalledItemNames(cwhub.APPSEC_RULES)
if err != nil {
return nil, err
}
ret := make([]string, 0, len(scenarios)+len(appsecRules))
ret = append(ret, scenarios...)
ret = append(ret, appsecRules...)
return ret, nil
},
})
if err != nil {
return fmt.Errorf("new client api: %w", err)
@ -101,7 +124,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
authResp, _, err := Client.Auth.AuthenticateWatcher(context.Background(), models.WatcherAuthRequest{
MachineID: &apiConfig.Login,
Password: &password,
Scenarios: scenarios,
Scenarios: installedScenariosAndAppsecRules,
})
if err != nil {
return fmt.Errorf("authenticate watcher (%s): %w", apiConfig.Login, err)
@ -145,13 +168,6 @@ LOOP:
}
break LOOP
case event := <-overflow:
//if the Alert is nil, it's to signal bucket is ready for GC, don't track this
if dumpStates && event.Overflow.Alert != nil {
if bucketOverflows == nil {
bucketOverflows = make([]types.Event, 0)
}
bucketOverflows = append(bucketOverflows, event)
}
/*if alert is empty and mapKey is present, the overflow is just to cleanup bucket*/
if event.Overflow.Alert == nil && event.Overflow.Mapkey != "" {
buckets.Bucket_map.Delete(event.Overflow.Mapkey)
@ -163,6 +179,14 @@ LOOP:
return fmt.Errorf("postoverflow failed : %s", err)
}
log.Printf("%s", *event.Overflow.Alert.Message)
//if the Alert is nil, it's to signal bucket is ready for GC, don't track this
//dump after postoveflow processing to avoid missing whitelist info
if dumpStates && event.Overflow.Alert != nil {
if bucketOverflows == nil {
bucketOverflows = make([]types.Event, 0)
}
bucketOverflows = append(bucketOverflows, event)
}
if event.Overflow.Whitelisted {
log.Printf("[%s] is whitelisted, skip.", *event.Overflow.Alert.Message)
continue

7
cmd/crowdsec/parse.go
View file

@ -22,6 +22,13 @@ LOOP:
if !event.Process {
continue
}
/*Application security engine is going to generate 2 events:
- one that is treated as a log and can go to scenarios
- another one that will go directly to LAPI*/
if event.Type == types.APPSEC {
outputEventChan <- event
continue
}
if event.Line.Module == "" {
log.Errorf("empty event.Line.Module field, the acquisition module must set it ! : %+v", event.Line)
continue

22
cmd/crowdsec/run_in_svc.go
View file

@ -1,14 +1,12 @@
//go:build linux || freebsd || netbsd || openbsd || solaris || !windows
// +build linux freebsd netbsd openbsd solaris !windows
//go:build !windows
package main
import (
"fmt"
"os"
"runtime/pprof"
log "github.com/sirupsen/logrus"
"github.com/sirupsen/logrus/hooks/writer"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/go-cs-lib/version"
@ -25,15 +23,9 @@ func StartRunSvc() error {
defer trace.CatchPanic("crowdsec/StartRunSvc")
// Set a default logger with level=fatal on stderr,
// in addition to the one we configure afterwards
log.AddHook(&writer.Hook{
Writer: os.Stderr,
LogLevels: []log.Level{
log.PanicLevel,
log.FatalLevel,
},
})
//Always try to stop CPU profiling to avoid passing flags around
//It's a noop if profiling is not enabled
defer pprof.StopCPUProfile()
if cConfig, err = LoadConfig(flags.ConfigFile, flags.DisableAgent, flags.DisableAPI, false); err != nil {
return err
@ -47,6 +39,7 @@ func StartRunSvc() error {
// Enable profiling early
if cConfig.Prometheus != nil {
var dbClient *database.Client
var err error
if cConfig.DbConfig != nil {
@ -56,8 +49,11 @@ func StartRunSvc() error {
return fmt.Errorf("unable to create database client: %s", err)
}
}
registerPrometheus(cConfig.Prometheus)
go servePrometheus(cConfig.Prometheus, dbClient, apiReady, agentReady)
}
return Serve(cConfig, apiReady, agentReady)
}

5
cmd/crowdsec/run_in_svc_windows.go
View file

@ -2,6 +2,7 @@ package main
import (
"fmt"
"runtime/pprof"
log "github.com/sirupsen/logrus"
"golang.org/x/sys/windows/svc"
@ -19,6 +20,10 @@ func StartRunSvc() error {
defer trace.CatchPanic("crowdsec/StartRunSvc")
//Always try to stop CPU profiling to avoid passing flags around
//It's a noop if profiling is not enabled
defer pprof.StopCPUProfile()
isRunninginService, err := svc.IsWindowsService()
if err != nil {
return fmt.Errorf("failed to determine if we are running in windows service mode: %w", err)

24
cmd/crowdsec/serve.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"os"
"os/signal"
"runtime/pprof"
"syscall"
"time"
@ -14,6 +15,7 @@ import (
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
@ -76,7 +78,12 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
}
if !cConfig.DisableAgent {
csParsers, err := initCrowdsec(cConfig)
hub, err := cwhub.NewHub(cConfig.Hub, nil, false, log.StandardLogger())
if err != nil {
return nil, fmt.Errorf("while loading hub index: %w", err)
}
csParsers, err := initCrowdsec(cConfig, hub)
if err != nil {
return nil, fmt.Errorf("unable to init crowdsec: %w", err)
}
@ -93,7 +100,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
}
agentReady := make(chan bool, 1)
serveCrowdsec(csParsers, cConfig, agentReady)
serveCrowdsec(csParsers, cConfig, hub, agentReady)
}
log.Printf("Reload is finished")
@ -239,6 +246,10 @@ func HandleSignals(cConfig *csconfig.Config) error {
exitChan := make(chan error)
//Always try to stop CPU profiling to avoid passing flags around
//It's a noop if profiling is not enabled
defer pprof.StopCPUProfile()
go func() {
defer trace.CatchPanic("crowdsec/HandleSignals")
Loop:
@ -342,14 +353,19 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
}
if !cConfig.DisableAgent {
csParsers, err := initCrowdsec(cConfig)
hub, err := cwhub.NewHub(cConfig.Hub, nil, false, log.StandardLogger())
if err != nil {
return fmt.Errorf("while loading hub index: %w", err)
}
csParsers, err := initCrowdsec(cConfig, hub)
if err != nil {
return fmt.Errorf("crowdsec init: %w", err)
}
// if it's just linting, we're done
if !flags.TestMode {
serveCrowdsec(csParsers, cConfig, agentReady)
serveCrowdsec(csParsers, cConfig, hub, agentReady)
}
} else {
agentReady <- true

6
cmd/crowdsec/win_service.go
View file

@ -3,7 +3,6 @@
// license that can be found in the LICENSE file.
//go:build windows
// +build windows
package main
@ -24,7 +23,7 @@ type crowdsec_winservice struct {
config *csconfig.Config
}
func (m *crowdsec_winservice) Execute(args []string, r <-chan svc.ChangeRequest, changes chan<- svc.Status) (ssec bool, errno uint32) {
func (m *crowdsec_winservice) Execute(args []string, r <-chan svc.ChangeRequest, changes chan<- svc.Status) (bool, uint32) {
const cmdsAccepted = svc.AcceptStop | svc.AcceptShutdown
changes <- svc.Status{State: svc.StartPending}
tick := time.Tick(500 * time.Millisecond)
@ -60,7 +59,8 @@ func (m *crowdsec_winservice) Execute(args []string, r <-chan svc.ChangeRequest,
if err != nil {
log.Fatal(err)
}
return
return false, 0
}
func runService(name string) error {

1
cmd/crowdsec/win_service_install.go
View file

@ -3,7 +3,6 @@
// license that can be found in the LICENSE file.
//go:build windows
// +build windows
package main

2
cmd/crowdsec/win_service_manage.go
View file

@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build windows
//go:build windows
package main

80
cmd/notification-http/main.go
View file

@ -4,6 +4,7 @@ import (
"bytes"
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"io"
"net/http"
@ -22,6 +23,10 @@ type PluginConfig struct {
SkipTLSVerification bool `yaml:"skip_tls_verification"`
Method string `yaml:"method"`
LogLevel *string `yaml:"log_level"`
Client *http.Client `yaml:"-"`
CertPath string `yaml:"cert_path"`
KeyPath string `yaml:"key_path"`
CAPath string `yaml:"ca_cert_path"`
}
type HTTPPlugin struct {
@ -35,6 +40,64 @@ var logger hclog.Logger = hclog.New(&hclog.LoggerOptions{
JSONFormat: true,
})
func getCertPool(caPath string) (*x509.CertPool, error) {
cp, err := x509.SystemCertPool()
if err != nil {
return nil, fmt.Errorf("unable to load system CA certificates: %w", err)
}
if cp == nil {
cp = x509.NewCertPool()
}
if caPath == "" {
return cp, nil
}
logger.Info(fmt.Sprintf("Using CA cert '%s'", caPath))
caCert, err := os.ReadFile(caPath)
if err != nil {
return nil, fmt.Errorf("unable to load CA certificate '%s': %w", caPath, err)
}
cp.AppendCertsFromPEM(caCert)
return cp, nil
}
func getTLSClient(tlsVerify bool, caPath, certPath, keyPath string) (*http.Client, error) {
var client *http.Client
caCertPool, err := getCertPool(caPath)
if err != nil {
return nil, err
}
tlsConfig := &tls.Config{
RootCAs: caCertPool,
InsecureSkipVerify: tlsVerify,
}
if certPath != "" && keyPath != "" {
logger.Info(fmt.Sprintf("Using client certificate '%s' and key '%s'", certPath, keyPath))
cert, err := tls.LoadX509KeyPair(certPath, keyPath)
if err != nil {
return nil, fmt.Errorf("unable to load client certificate '%s' and key '%s': %w", certPath, keyPath, err)
}
tlsConfig.Certificates = []tls.Certificate{cert}
}
client = &http.Client{
Transport: &http.Transport{
TLSClientConfig: tlsConfig,
},
}
return client, err
}
func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notification) (*protobufs.Empty, error) {
if _, ok := s.PluginConfigByName[notification.Name]; !ok {
return nil, fmt.Errorf("invalid plugin config name %s", notification.Name)
@ -46,25 +109,17 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
}
logger.Info(fmt.Sprintf("received signal for %s config", notification.Name))
client := http.Client{}
if cfg.SkipTLSVerification {
client.Transport = &http.Transport{
TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
}
}
request, err := http.NewRequest(cfg.Method, cfg.URL, bytes.NewReader([]byte(notification.Text)))
if err != nil {
return nil, err
}
for headerName, headerValue := range cfg.Headers {
logger.Debug(fmt.Sprintf("adding header %s: %s", headerName, headerValue))
request.Header.Add(headerName, headerValue)
}
logger.Debug(fmt.Sprintf("making HTTP %s call to %s with body %s", cfg.Method, cfg.URL, notification.Text))
resp, err := client.Do(request)
resp, err := cfg.Client.Do(request.WithContext(ctx))
if err != nil {
logger.Error(fmt.Sprintf("Failed to make HTTP request : %s", err))
return nil, err
@ -89,6 +144,13 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
func (s *HTTPPlugin) Configure(ctx context.Context, config *protobufs.Config) (*protobufs.Empty, error) {
d := PluginConfig{}
err := yaml.Unmarshal(config.Config, &d)
if err != nil {
return nil, err
}
d.Client, err = getTLSClient(d.SkipTLSVerification, d.CAPath, d.CertPath, d.KeyPath)
if err != nil {
return nil, err
}
s.PluginConfigByName[d.Name] = d
logger.Debug(fmt.Sprintf("HTTP plugin '%s' use URL '%s'", d.Name, d.URL))
return &protobufs.Empty{}, err

2
cmd/notification-sentinel/main.go
View file

@ -90,7 +90,7 @@ func (s *SentinelPlugin) Notify(ctx context.Context, notification *protobufs.Not
req.Header.Set("x-ms-date", now)
client := &http.Client{}
resp, err := client.Do(req)
resp, err := client.Do(req.WithContext(ctx))
if err != nil {
logger.Error("failed to send request", "error", err)
return &protobufs.Empty{}, err

3
cmd/notification-slack/main.go
View file

@ -38,10 +38,9 @@ func (n *Notify) Notify(ctx context.Context, notification *protobufs.Notificatio
if cfg.LogLevel != nil && *cfg.LogLevel != "" {
logger.SetLevel(hclog.LevelFromString(*cfg.LogLevel))
}
logger.Info(fmt.Sprintf("found notify signal for %s config", notification.Name))
logger.Debug(fmt.Sprintf("posting to %s webhook, message %s", cfg.Webhook, notification.Text))
err := slack.PostWebhook(n.ConfigByName[notification.Name].Webhook, &slack.WebhookMessage{
err := slack.PostWebhookContext(ctx, n.ConfigByName[notification.Name].Webhook, &slack.WebhookMessage{
Text: notification.Text,
})
if err != nil {

2
cmd/notification-splunk/main.go
View file

@ -65,7 +65,7 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio
req.Header.Add("Authorization", fmt.Sprintf("Splunk %s", cfg.Token))
logger.Debug(fmt.Sprintf("posting event %s to %s", string(data), req.URL))
resp, err := s.Client.Do(req)
resp, err := s.Client.Do(req.WithContext(ctx))
if err != nil {
return &protobufs.Empty{}, err
}

4
config/acquis_win.yaml
View file

@ -10,7 +10,7 @@ labels:
---
##Firewall
filenames:
- C:\Windows\System32\LogFiles\Firewall\pfirewall.log
- C:\Windows\System32\LogFiles\Firewall\*.log
labels:
type: windows-firewall
---
@ -28,4 +28,4 @@ use_time_machine: true
filenames:
- C:\inetpub\logs\LogFiles\*\*.log
labels:
type: iis
type: iis

1
config/config.yaml
View file

@ -6,7 +6,6 @@ common:
log_max_size: 20
compress_logs: true
log_max_files: 10
working_dir: .
config_paths:
config_dir: /etc/crowdsec/
data_dir: /var/lib/crowdsec/data/

1
config/config_win.yaml
View file

@ -3,7 +3,6 @@ common:
log_media: file
log_level: info
log_dir: C:\ProgramData\CrowdSec\log\
working_dir: .
config_paths:
config_dir: C:\ProgramData\CrowdSec\config\
data_dir: C:\ProgramData\CrowdSec\data\

1
config/config_win_no_lapi.yaml
View file

@ -3,7 +3,6 @@ common:
log_media: file
log_level: info
log_dir: C:\ProgramData\CrowdSec\log\
working_dir: .
config_paths:
config_dir: C:\ProgramData\CrowdSec\config\
data_dir: C:\ProgramData\CrowdSec\data\

2
config/dev.yaml
View file

@ -2,7 +2,6 @@ common:
daemonize: true
log_media: stdout
log_level: info
working_dir: .
config_paths:
config_dir: ./config
data_dir: ./data/
@ -34,6 +33,7 @@ api:
client:
credentials_path: ./config/local_api_credentials.yaml
server:
console_path: ./config/console.yaml
#insecure_skip_verify: true
listen_uri: 127.0.0.1:8081
profiles_path: ./config/profiles.yaml

1
config/user.yaml
View file

@ -3,7 +3,6 @@ common:
log_media: stdout
log_level: info
log_dir: /var/log/
working_dir: .
config_paths:
config_dir: /etc/crowdsec/
data_dir: /var/lib/crowdsec/data

Some files were not shown because too many files have changed in this diff Show more