Compare commits

..

1 commit
master ... ci

Author SHA1 Message Date
Marco Mariani
7b4978f4bc Revert "build static by default" 2023-04-05 16:37:39 +02:00
735 changed files with 29308 additions and 46931 deletions

View file

@ -3,4 +3,3 @@
#.git
/tests
/crowdsec-v*

10
.github/codecov.yml vendored
View file

@ -1,10 +0,0 @@
# we measure coverage but don't enforce it
# https://docs.codecov.com/docs/codecov-yaml
coverage:
status:
patch:
default:
target: 0%
project:
default:
target: 0%

View file

@ -42,7 +42,7 @@ issue:
3. Check [Releases](https://github.com/crowdsecurity/crowdsec/releases/latest) to make sure your agent is on the latest version.
- prefix: kind
list: ['feature', 'bug', 'packaging', 'enhancement', 'refactoring']
list: ['feature', 'bug', 'packaging', 'enhancement']
multiple: false
author_association:
author: true
@ -54,7 +54,6 @@ issue:
@$AUTHOR: There are no 'kind' label on this issue. You need a 'kind' label to start the triage process.
* `/kind feature`
* `/kind enhancement`
* `/kind refactoring`
* `/kind bug`
* `/kind packaging`
@ -66,13 +65,12 @@ pull_request:
labels:
- prefix: kind
multiple: false
list: [ 'feature', 'enhancement', 'fix', 'chore', 'dependencies', 'refactoring']
list: [ 'feature', 'enhancement', 'fix', 'chore', 'dependencies']
needs:
comment: |
@$AUTHOR: There are no 'kind' label on this PR. You need a 'kind' label to generate the release automatically.
* `/kind feature`
* `/kind enhancement`
* `/kind refactoring`
* `/kind fix`
* `/kind chore`
* `/kind dependencies`
@ -83,7 +81,7 @@ pull_request:
failure: Missing kind label to generate release automatically.
- prefix: area
list: [ "agent", "local-api", "cscli", "security", "configuration", "appsec"]
list: [ "agent", "local-api", "cscli", "security", "configuration"]
multiple: true
needs:
comment: |
@ -91,7 +89,6 @@ pull_request:
* `/area agent`
* `/area local-api`
* `/area cscli`
* `/area appsec`
* `/area security`
* `/area configuration`
@ -101,4 +98,4 @@ pull_request:
author_association:
collaborator: true
member: true
owner: true
owner: true

View file

@ -1,4 +1,4 @@
name: (sub) Bats / Hub
name: Hub tests
on:
workflow_call:
@ -8,13 +8,16 @@ on:
GIST_BADGES_ID:
required: true
env:
PREFIX_TEST_NAMES_WITH_FILE: true
jobs:
build:
strategy:
matrix:
test-file: ["hub-1.bats", "hub-2.bats", "hub-3.bats"]
go-version: ["1.20.1"]
name: "Functional tests"
name: "Build + tests"
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
@ -24,36 +27,49 @@ jobs:
sudo chmod +w /etc/machine-id
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: "Check out CrowdSec repository"
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true
- name: "Set up Go"
uses: actions/setup-go@v5
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: "1.22.2"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
go install github.com/mikefarah/yq/v4@latest
go install github.com/cloudflare/cfssl/cmd/cfssl@master
go install github.com/cloudflare/cfssl/cmd/cfssljson@master
- name: "Build crowdsec and fixture"
run: make bats-clean bats-build bats-fixture BUILD_STATIC=1
run: make bats-clean bats-build bats-fixture
- name: "Run hub tests"
run: |
./test/bin/generate-hub-tests
./test/run-tests ./test/dyn-bats/${{ matrix.test-file }} --formatter $(pwd)/test/lib/color-formatter
run: make bats-test-hub
- name: "Collect hub coverage"
run: ./test/bin/collect-hub-coverage >> $GITHUB_ENV
- name: "Create Parsers badge"
uses: schneegans/dynamic-badges-action@v1.7.0
uses: schneegans/dynamic-badges-action@v1.6.0
if: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'crowdsecurity' }}
with:
auth: ${{ secrets.GIST_BADGES_SECRET }}
@ -64,7 +80,7 @@ jobs:
color: ${{ env.SCENARIO_BADGE_COLOR }}
- name: "Create Scenarios badge"
uses: schneegans/dynamic-badges-action@v1.7.0
uses: schneegans/dynamic-badges-action@v1.6.0
if: ${{ github.ref == 'refs/heads/master' && github.repository_owner == 'crowdsecurity' }}
with:
auth: ${{ secrets.GIST_BADGES_SECRET }}

View file

@ -1,4 +1,4 @@
name: (sub) Bats / MySQL
name: Functional tests (MySQL)
on:
workflow_call:
@ -7,9 +7,16 @@ on:
required: true
type: string
env:
PREFIX_TEST_NAMES_WITH_FILE: true
jobs:
build:
name: "Functional tests"
strategy:
matrix:
go-version: ["1.20.1"]
name: "Build + tests"
runs-on: ubuntu-latest
timeout-minutes: 30
services:
@ -27,26 +34,41 @@ jobs:
sudo chmod +w /etc/machine-id
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: "Check out CrowdSec repository"
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true
- name: "Set up Go"
uses: actions/setup-go@v5
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: "1.22.2"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
go install github.com/mikefarah/yq/v4@latest
go install github.com/cloudflare/cfssl/cmd/cfssl@master
go install github.com/cloudflare/cfssl/cmd/cfssljson@master
- name: "Build crowdsec and fixture"
run: |
make clean bats-build bats-fixture BUILD_STATIC=1
make clean bats-build bats-fixture
env:
DB_BACKEND: mysql
MYSQL_HOST: 127.0.0.1
@ -55,7 +77,7 @@ jobs:
MYSQL_USER: root
- name: "Run tests"
run: ./test/run-tests ./test/bats --formatter $(pwd)/test/lib/color-formatter
run: make bats-test
env:
DB_BACKEND: mysql
MYSQL_HOST: 127.0.0.1

View file

@ -1,16 +1,23 @@
name: (sub) Bats / Postgres
name: Functional tests (Postgres)
on:
workflow_call:
env:
PREFIX_TEST_NAMES_WITH_FILE: true
jobs:
build:
name: "Functional tests"
strategy:
matrix:
go-version: ["1.20.1"]
name: "Build + tests"
runs-on: ubuntu-latest
timeout-minutes: 30
services:
database:
image: postgres:16
image: postgres:14
env:
POSTGRES_PASSWORD: "secret"
ports:
@ -23,39 +30,46 @@ jobs:
steps:
- name: "Install pg_dump v16"
# we can remove this when it's released on ubuntu-latest
run: |
sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /etc/apt/trusted.gpg.d/pgdg.asc &>/dev/null
sudo apt update
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install postgresql-client-16
- name: "Force machineid"
run: |
sudo chmod +w /etc/machine-id
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: "Check out CrowdSec repository"
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true
- name: "Set up Go"
uses: actions/setup-go@v5
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: "1.22.2"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
go install github.com/mikefarah/yq/v4@latest
go install github.com/cloudflare/cfssl/cmd/cfssl@master
go install github.com/cloudflare/cfssl/cmd/cfssljson@master
- name: "Build crowdsec and fixture (DB_BACKEND: pgx)"
run: |
make clean bats-build bats-fixture BUILD_STATIC=1
make clean bats-build bats-fixture
env:
DB_BACKEND: pgx
PGHOST: 127.0.0.1
@ -64,7 +78,7 @@ jobs:
PGUSER: postgres
- name: "Run tests (DB_BACKEND: pgx)"
run: ./test/run-tests ./test/bats --formatter $(pwd)/test/lib/color-formatter
run: make bats-test
env:
DB_BACKEND: pgx
PGHOST: 127.0.0.1
@ -72,6 +86,24 @@ jobs:
PGPASSWORD: "secret"
PGUSER: postgres
# - name: "Build crowdsec and fixture (DB_BACKEND: postgres)"
# run: make clean bats-build bats-fixture
# env:
# DB_BACKEND: postgres
# PGHOST: 127.0.0.1
# PGPORT: 5432
# PGPASSWORD: "secret"
# PGUSER: postgres
#
# - name: "Run tests (DB_BACKEND: postgres)"
# run: make bats-test
# env:
# DB_BACKEND: postgres
# PGHOST: 127.0.0.1
# PGPORT: 5432
# PGPASSWORD: "secret"
# PGUSER: postgres
- name: "Show stack traces"
run: for file in $(find /tmp/crowdsec-crash.*.txt); do echo ">>>>> $file"; cat $file; echo; done
if: ${{ always() }}

View file

@ -1,14 +1,19 @@
name: (sub) Bats / sqlite + coverage
name: Functional tests (sqlite)
on:
workflow_call:
env:
PREFIX_TEST_NAMES_WITH_FILE: true
TEST_COVERAGE: true
jobs:
build:
name: "Functional tests"
strategy:
matrix:
go-version: ["1.20.1"]
name: "Build + tests"
runs-on: ubuntu-latest
timeout-minutes: 20
@ -19,29 +24,44 @@ jobs:
sudo chmod +w /etc/machine-id
echo githubciXXXXXXXXXXXXXXXXXXXXXXXX | sudo tee /etc/machine-id
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: "Check out CrowdSec repository"
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: true
- name: "Set up Go"
uses: actions/setup-go@v5
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: "1.22.2"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: "Install bats dependencies"
env:
GOBIN: /usr/local/bin
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq libre2-dev
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential daemonize jq netcat-openbsd
go install github.com/mikefarah/yq/v4@latest
go install github.com/cloudflare/cfssl/cmd/cfssl@master
go install github.com/cloudflare/cfssl/cmd/cfssljson@master
- name: "Build crowdsec and fixture"
run: |
make clean bats-build bats-fixture BUILD_STATIC=1
make clean bats-build bats-fixture
- name: "Run tests"
run: ./test/run-tests ./test/bats --formatter $(pwd)/test/lib/color-formatter
run: make bats-test
- name: "Collect coverage data"
run: |
@ -52,6 +72,7 @@ jobs:
-e '/plugins/notifications' \
-e '/pkg/protobufs' \
-e '/pkg/cwversions' \
-e '/pkg/cstest' \
-e '/pkg/models' \
< coverage-bats-raw.out \
> coverage-bats.out
@ -77,8 +98,7 @@ jobs:
if: ${{ always() }}
- name: Upload crowdsec coverage to codecov
uses: codecov/codecov-action@v4
uses: codecov/codecov-action@v3
with:
files: ./coverage-bats.out
flags: bats
token: ${{ secrets.CODECOV_TOKEN }}

View file

@ -31,7 +31,7 @@ jobs:
# Jobs for Postgres (and sometimes MySQL) can have failing tests on GitHub
# CI, but they pass when run on devs' machines or in the release checks. We
# disable them here by default. Remove if...false to enable them.
# disable them here by default. Remove the if..false to enable them.
mariadb:
uses: ./.github/workflows/bats-mysql.yml

View file

@ -1,35 +0,0 @@
# https://docs.github.com/en/actions/using-workflows/caching-dependencies-to-speed-up-workflows#managing-caches
name: cleanup caches by a branch
on:
pull_request:
types:
- closed
jobs:
cleanup:
runs-on: ubuntu-latest
steps:
- name: Check out code
uses: actions/checkout@v4
- name: Cleanup
run: |
gh extension install actions/gh-actions-cache
REPO=${{ github.repository }}
BRANCH="refs/pull/${{ github.event.pull_request.number }}/merge"
echo "Fetching list of cache key"
cacheKeysForPR=$(gh actions-cache list -R $REPO -B $BRANCH | cut -f 1 )
## Setting this to not fail the workflow while deleting cache keys.
set +e
echo "Deleting caches..."
for cacheKey in $cacheKeysForPR
do
gh actions-cache delete $cacheKey -R $REPO -B $BRANCH --confirm
done
echo "Done"
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

View file

@ -21,26 +21,42 @@ on:
jobs:
build:
strategy:
matrix:
go-version: ["1.20.1"]
name: Build
runs-on: windows-2019
steps:
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Check out code into the Go module directory
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: false
- name: "Set up Go"
uses: actions/setup-go@v5
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: "1.22.2"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: Build
run: make windows_installer BUILD_RE2_WASM=1
run: make windows_installer
- name: Upload MSI
uses: actions/upload-artifact@v4
uses: actions/upload-artifact@v3
with:
path: crowdsec*msi
name: crowdsec.msi

View file

@ -12,7 +12,7 @@ jobs:
runs-on: ubuntu-latest
steps:
# Drafts your next Release notes as Pull Requests are merged into "master"
- uses: release-drafter/release-drafter@v6
- uses: release-drafter/release-drafter@v5
with:
config-name: release-drafter.yml
# (Optional) specify config name to use, relative to .github/. Default: release-drafter.yml

View file

@ -44,20 +44,11 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
# required to pick up tags for BUILD_VERSION
fetch-depth: 0
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: "1.22.2"
cache-dependency-path: "**/go.sum"
uses: actions/checkout@v3
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v3
uses: github/codeql-action/init@v2
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
@ -67,8 +58,8 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
# - name: Autobuild
# uses: github/codeql-action/autobuild@v3
- name: Autobuild
uses: github/codeql-action/autobuild@v2
# Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
@ -77,8 +68,9 @@ jobs:
# and modify them (or add more) to build your code if your project
# uses a compiled language
- run: |
make clean build BUILD_RE2_WASM=1
#- run: |
# make bootstrap
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v3
uses: github/codeql-action/analyze@v2

View file

@ -15,42 +15,71 @@ on:
- 'README.md'
jobs:
test_flavor:
strategy:
# we could test all the flavors in a single pytest job,
# but let's split them (and the image build) in multiple runners for performance
matrix:
# can be slim, full or debian (no debian slim).
flavor: ["slim", "debian"]
test_docker_image:
runs-on: ubuntu-latest
timeout-minutes: 30
steps:
- name: Check out the repo
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
uses: docker/setup-buildx-action@v2
with:
config: .github/buildkit.toml
- name: "Build image"
uses: docker/build-push-action@v5
# - name: "Build flavor: full"
# uses: docker/build-push-action@v4
# with:
# context: .
# file: ./Dockerfile
# tags: crowdsecurity/crowdsec:test
# target: full
# platforms: linux/amd64
# load: true
# cache-from: type=gha
# cache-to: type=gha,mode=min
- name: "Build flavor: slim"
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile${{ matrix.flavor == 'debian' && '.debian' || '' }}
tags: crowdsecurity/crowdsec:test${{ matrix.flavor == 'full' && '' || '-' }}${{ matrix.flavor == 'full' && '' || matrix.flavor }}
target: ${{ matrix.flavor == 'debian' && 'full' || matrix.flavor }}
file: ./Dockerfile
tags: crowdsecurity/crowdsec:test-slim
target: slim
platforms: linux/amd64
load: true
cache-from: type=gha
cache-to: type=gha,mode=min
- name: "Build flavor: full"
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile
tags: crowdsecurity/crowdsec:test
target: full
platforms: linux/amd64
load: true
cache-from: type=gha
cache-to: type=gha,mode=min
- name: "Build flavor: full (debian)"
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile.debian
tags: crowdsecurity/crowdsec:test-debian
target: full
platforms: linux/amd64
load: true
cache-from: type=gha
cache-to: type=gha,mode=min
- name: "Setup Python"
uses: actions/setup-python@v5
uses: actions/setup-python@v4
with:
python-version: "3.x"
@ -59,15 +88,15 @@ jobs:
cd docker/test
python -m pip install --upgrade pipenv wheel
#- name: "Cache virtualenvs"
# id: cache-pipenv
# uses: actions/cache@v4
# with:
# path: ~/.local/share/virtualenvs
# key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
- name: "Cache virtualenvs"
id: cache-pipenv
uses: actions/cache@v3
with:
path: ~/.local/share/virtualenvs
key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }}
- name: "Install dependencies"
#if: steps.cache-pipenv.outputs.cache-hit != 'true'
if: steps.cache-pipenv.outputs.cache-hit != 'true'
run: |
cd docker/test
pipenv install --deploy
@ -78,10 +107,9 @@ jobs:
- name: "Run tests"
env:
CROWDSEC_TEST_VERSION: test
CROWDSEC_TEST_FLAVORS: ${{ matrix.flavor }}
CROWDSEC_TEST_FLAVORS: slim,debian
CROWDSEC_TEST_NETWORK: net-test
CROWDSEC_TEST_TIMEOUT: 90
# running serially to reduce test flakiness
run: |
cd docker/test
pipenv run pytest -n 1 --durations=0 --color=yes
pipenv run pytest -n 2 --durations=0 --color=yes

View file

@ -20,25 +20,41 @@ env:
jobs:
build:
strategy:
matrix:
go-version: ["1.20.1"]
name: "Build + tests"
runs-on: windows-2022
steps:
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Check out CrowdSec repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: false
- name: "Set up Go"
uses: actions/setup-go@v5
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: "1.22.2"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: Build
run: |
make build BUILD_RE2_WASM=1
make build
- name: Run tests
run: |
@ -48,16 +64,15 @@ jobs:
cat out.txt | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
- name: Upload unit coverage to Codecov
uses: codecov/codecov-action@v4
uses: codecov/codecov-action@v3
with:
files: coverage.out
flags: unit-windows
token: ${{ secrets.CODECOV_TOKEN }}
- name: golangci-lint
uses: golangci/golangci-lint-action@v4
uses: golangci/golangci-lint-action@v3
with:
version: v1.57
version: v1.51
args: --issues-exit-code=1 --timeout 10m
only-new-issues: false
# the cache is already managed above, enabling it here

View file

@ -24,18 +24,23 @@ env:
RICHGO_FORCE_COLOR: 1
AWS_HOST: localstack
# these are to mimic aws config
AWS_ACCESS_KEY_ID: test
AWS_SECRET_ACCESS_KEY: test
AWS_ACCESS_KEY_ID: AKIAIOSFODNN7EXAMPLE
AWS_SECRET_ACCESS_KEY: wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
AWS_REGION: us-east-1
KINESIS_INITIALIZE_STREAMS: "stream-1-shard:1,stream-2-shards:2"
CROWDSEC_FEATURE_DISABLE_HTTP_RETRY_BACKOFF: true
jobs:
build:
strategy:
matrix:
go-version: ["1.20.1"]
name: "Build + tests"
runs-on: ubuntu-latest
services:
localstack:
image: localstack/localstack:3.0
image: localstack/localstack:1.3.0
ports:
- 4566:4566 # Localstack exposes all services on the same port
env:
@ -44,7 +49,7 @@ jobs:
KINESIS_ERROR_PROBABILITY: ""
DOCKER_HOST: unix:///var/run/docker.sock
KINESIS_INITIALIZE_STREAMS: ${{ env.KINESIS_INITIALIZE_STREAMS }}
LOCALSTACK_HOST: ${{ env.AWS_HOST }} # Required so that resource urls are provided properly
HOSTNAME_EXTERNAL: ${{ env.AWS_HOST }} # Required so that resource urls are provided properly
# e.g sqs url will get localhost if we don't set this env to map our service
options: >-
--name=localstack
@ -53,7 +58,7 @@ jobs:
--health-timeout=5s
--health-retries=3
zoo1:
image: confluentinc/cp-zookeeper:7.4.3
image: confluentinc/cp-zookeeper:7.3.0
ports:
- "2181:2181"
env:
@ -103,62 +108,55 @@ jobs:
--health-timeout 10s
--health-retries 5
loki:
image: grafana/loki:2.9.1
ports:
- "3100:3100"
options: >-
--name=loki1
--health-cmd "wget -q -O - http://localhost:3100/ready | grep 'ready'"
--health-interval 30s
--health-timeout 10s
--health-retries 5
--health-start-period 30s
steps:
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Check out CrowdSec repository
uses: actions/checkout@v4
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: false
- name: "Set up Go"
uses: actions/setup-go@v5
- name: Cache Go modules
uses: actions/cache@v3
with:
go-version: "1.22.2"
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: Create localstack streams
- name: Build and run tests
run: |
aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-1-shard --shard-count 1
aws --endpoint-url=http://127.0.0.1:4566 --region us-east-1 kinesis create-stream --stream-name stream-2-shards --shard-count 2
- name: Build and run tests, static
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev
go install github.com/ory/go-acc@v0.2.8
go install github.com/kyoh86/richgo@v0.3.10
set -o pipefail
make build BUILD_STATIC=1
make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
make build
make go-acc | richgo testfilter
- name: Run tests again, dynamic
- name: Build and run tests (static)
run: |
make clean build
set -o pipefail
make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
make clean build BUILD_STATIC=yes
make test \
| richgo testfilter
- name: Upload unit coverage to Codecov
uses: codecov/codecov-action@v4
uses: codecov/codecov-action@v3
with:
files: coverage.out
flags: unit-linux
token: ${{ secrets.CODECOV_TOKEN }}
- name: golangci-lint
uses: golangci/golangci-lint-action@v4
uses: golangci/golangci-lint-action@v3
with:
version: v1.57
version: v1.51
args: --issues-exit-code=1 --timeout 10m
only-new-issues: false
# the cache is already managed above, enabling it here

View file

@ -23,7 +23,7 @@ jobs:
runs-on: ubuntu-latest
steps:
# Semantic versioning, lock to different version: v2, v2.0 or a commit hash.
- uses: BirthdayResearch/oss-governance-bot@v4
- uses: BirthdayResearch/oss-governance-bot@v3
with:
# You can use a PAT to post a comment/label/status so that it shows up as a user instead of github-actions
github-token: ${{secrets.GITHUB_TOKEN}} # optional, default to '${{ github.token }}'

View file

@ -1,47 +0,0 @@
name: (push-master) Publish latest Docker images
on:
push:
branches: [ master ]
paths:
- 'pkg/**'
- 'cmd/**'
- 'mk/**'
- 'docker/docker_start.sh'
- 'docker/config.yaml'
- '.github/workflows/publish-docker-master.yml'
- '.github/workflows/publish-docker.yml'
- 'Dockerfile'
- 'Dockerfile.debian'
- 'go.mod'
- 'go.sum'
- 'Makefile'
jobs:
dev-alpine:
uses: ./.github/workflows/publish-docker.yml
with:
platform: linux/amd64
crowdsec_version: ""
image_version: dev
latest: false
push: true
slim: false
debian: false
secrets:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
dev-debian:
uses: ./.github/workflows/publish-docker.yml
with:
platform: linux/amd64
crowdsec_version: ""
image_version: dev
latest: false
push: true
slim: false
debian: true
secrets:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}

View file

@ -1,48 +0,0 @@
name: (manual) Publish Docker images
on:
workflow_dispatch:
inputs:
image_version:
description: Docker Image version (base tag, i.e. v1.6.0-2)
required: true
crowdsec_version:
description: Crowdsec version (BUILD_VERSION)
required: true
latest:
description: Overwrite latest (and slim) tags?
default: false
required: true
push:
description: Really push?
default: false
required: true
jobs:
alpine:
uses: ./.github/workflows/publish-docker.yml
secrets:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
with:
image_version: ${{ github.event.inputs.image_version }}
crowdsec_version: ${{ github.event.inputs.crowdsec_version }}
latest: ${{ github.event.inputs.latest == 'true' }}
push: ${{ github.event.inputs.push == 'true' }}
slim: true
debian: false
platform: "linux/amd64,linux/386,linux/arm64,linux/arm/v7,linux/arm/v6"
debian:
uses: ./.github/workflows/publish-docker.yml
secrets:
DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
with:
image_version: ${{ github.event.inputs.image_version }}
crowdsec_version: ${{ github.event.inputs.crowdsec_version }}
latest: ${{ github.event.inputs.latest == 'true' }}
push: ${{ github.event.inputs.push == 'true' }}
slim: false
debian: true
platform: "linux/amd64,linux/386,linux/arm64"

View file

@ -1,125 +0,0 @@
name: (sub) Publish Docker images
on:
workflow_call:
secrets:
DOCKER_USERNAME:
required: true
DOCKER_PASSWORD:
required: true
inputs:
platform:
required: true
type: string
image_version:
required: true
type: string
crowdsec_version:
required: true
type: string
latest:
required: true
type: boolean
push:
required: true
type: boolean
slim:
required: true
type: boolean
debian:
required: true
type: boolean
jobs:
push_to_registry:
name: Push Docker image to registries
runs-on: ubuntu-latest
steps:
- name: Check out the repo
uses: actions/checkout@v4
with:
fetch-depth: 0
- name: Set up QEMU
uses: docker/setup-qemu-action@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
config: .github/buildkit.toml
- name: Login to DockerHub
uses: docker/login-action@v3
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Prepare (slim)
if: ${{ inputs.slim }}
id: slim
run: |
DOCKERHUB_IMAGE=${{ secrets.DOCKER_USERNAME }}/crowdsec
GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
VERSION=${{ inputs.image_version }}
DEBIAN=${{ inputs.debian && '-debian' || '' }}
TAGS="${DOCKERHUB_IMAGE}:${VERSION}-slim${DEBIAN},${GHCR_IMAGE}:${VERSION}-slim${DEBIAN}"
if [[ ${{ inputs.latest }} == true ]]; then
TAGS=$TAGS,${DOCKERHUB_IMAGE}:slim${DEBIAN},${GHCR_IMAGE}:slim${DEBIAN}
fi
echo "tags=${TAGS}" >> $GITHUB_OUTPUT
echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
- name: Prepare (full)
id: full
run: |
DOCKERHUB_IMAGE=${{ secrets.DOCKER_USERNAME }}/crowdsec
GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
VERSION=${{ inputs.image_version }}
DEBIAN=${{ inputs.debian && '-debian' || '' }}
TAGS="${DOCKERHUB_IMAGE}:${VERSION}${DEBIAN},${GHCR_IMAGE}:${VERSION}${DEBIAN}"
if [[ ${{ inputs.latest }} == true ]]; then
TAGS=$TAGS,${DOCKERHUB_IMAGE}:latest${DEBIAN},${GHCR_IMAGE}:latest${DEBIAN}
fi
echo "tags=${TAGS}" >> $GITHUB_OUTPUT
echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
- name: Build and push image (slim)
if: ${{ inputs.slim }}
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile${{ inputs.debian && '.debian' || '' }}
push: ${{ inputs.push }}
tags: ${{ steps.slim.outputs.tags }}
target: slim
platforms: ${{ inputs.platform }}
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.slim.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}
build-args: |
BUILD_VERSION=${{ inputs.crowdsec_version }}
- name: Build and push image (full)
uses: docker/build-push-action@v5
with:
context: .
file: ./Dockerfile${{ inputs.debian && '.debian' || '' }}
push: ${{ inputs.push }}
tags: ${{ steps.full.outputs.tags }}
target: full
platforms: ${{ inputs.platform }}
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.full.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}
build-args: |
BUILD_VERSION=${{ inputs.crowdsec_version }}

View file

@ -1,40 +0,0 @@
# .github/workflows/build-docker-image.yml
name: Release
on:
release:
types:
- prereleased
permissions:
# Use write for: hub release edit
contents: write
jobs:
build:
name: Build and upload binary package
runs-on: ubuntu-latest
steps:
- name: Check out code into the Go module directory
uses: actions/checkout@v4
with:
fetch-depth: 0
submodules: false
- name: "Set up Go"
uses: actions/setup-go@v5
with:
go-version: "1.22.2"
- name: Build the binaries
run: |
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install build-essential libre2-dev
make vendor release BUILD_STATIC=1
- name: Upload to release
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
tag_name="${GITHUB_REF##*/}"
gh release upload "$tag_name" crowdsec-release.tgz vendor.tgz *-vendor.tar.xz

View file

@ -0,0 +1,70 @@
name: Publish Docker image on Push to Master
on:
push:
branches: [ master ]
paths:
- 'pkg/**'
- 'cmd/**'
- 'plugins/**'
- 'docker/docker_start.sh'
- 'docker/config.yaml'
- '.github/workflows/publish_docker-image_on_master.yml'
- 'Dockerfile'
- 'go.mod'
- 'go.sum'
- 'Makefile'
jobs:
push_to_registry:
name: Push Docker image to Docker Hub
runs-on: ubuntu-latest
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Prepare
id: prep
run: |
DOCKER_IMAGE=crowdsecurity/crowdsec
GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
VERSION=dev
TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}"
echo "tags=${TAGS}" >> $GITHUB_OUTPUT
echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
config: .github/buildkit.toml
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push full image
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.prep.outputs.tags }}
platforms: linux/amd64
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.prep.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}
cache-from: type=gha
cache-to: type=gha,mode=min

View file

@ -0,0 +1,94 @@
# .github/workflows/build-docker-image.yml
name: build
on:
release:
types:
- prereleased
jobs:
build:
strategy:
matrix:
go-version: ["1.20.1"]
name: Build and upload binary package
runs-on: ubuntu-latest
steps:
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
- name: Check out code into the Go module directory
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: false
- name: Cache Go modules
uses: actions/cache@v3
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: Build the binaries
run: make release
- name: Upload to release
uses: JasonEtco/upload-to-release@master
with:
args: crowdsec-release.tgz application/x-gzip
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
build_static:
strategy:
matrix:
go-version: ["1.20.1"]
name: Build and upload binary package
runs-on: ubuntu-latest
steps:
- name: "Set up Go ${{ matrix.go-version }}"
uses: actions/setup-go@v3
with:
go-version: ${{ matrix.go-version }}
id: go
- name: Check out code into the Go module directory
uses: actions/checkout@v3
with:
fetch-depth: 0
submodules: false
- name: Cache Go modules
uses: actions/cache@v3
with:
path: |
~/go/pkg/mod
~/.cache/go-build
~/Library/Caches/go-build
%LocalAppData%\go-build
key: ${{ runner.os }}-${{ matrix.go-version }}-go-${{ hashFiles('**/go.sum') }}
restore-keys: |
${{ runner.os }}-${{ matrix.go-version }}-go-
- name: Build the binaries
run: |
make release BUILD_STATIC=yes
mv crowdsec-release.tgz crowdsec-release-static.tgz
- name: Upload to release
uses: JasonEtco/upload-to-release@master
with:
args: crowdsec-release-static.tgz application/x-gzip
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

View file

@ -0,0 +1,61 @@
name: Publish Docker Debian image
on:
release:
types:
- released
- prereleased
workflow_dispatch:
jobs:
push_to_registry:
name: Push Docker debian image to Docker Hub
runs-on: ubuntu-latest
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Prepare
id: prep
run: |
DOCKER_IMAGE=crowdsecurity/crowdsec
VERSION=bullseye
if [[ $GITHUB_REF == refs/tags/* ]]; then
VERSION=${GITHUB_REF#refs/tags/}
elif [[ $GITHUB_REF == refs/heads/* ]]; then
VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -E 's#/+#-#g')
elif [[ $GITHUB_REF == refs/pull/* ]]; then
VERSION=pr-${{ github.event.number }}
fi
TAGS="${DOCKER_IMAGE}:${VERSION}-debian"
if [[ ${{ github.event.action }} == released ]]; then
TAGS=$TAGS,${DOCKER_IMAGE}:latest-debian
fi
echo "version=${VERSION}" >> $GITHUB_OUTPUT
echo "tags=${TAGS}" >> $GITHUB_OUTPUT
echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
config: .github/buildkit.toml
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Build and push
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile.debian
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.prep.outputs.tags }}
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6,linux/386
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.prep.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}

View file

@ -0,0 +1,86 @@
name: Publish Docker image
on:
release:
types:
- released
- prereleased
jobs:
push_to_registry:
name: Push Docker image to Docker Hub
runs-on: ubuntu-latest
steps:
- name: Check out the repo
uses: actions/checkout@v3
with:
fetch-depth: 0
- name: Prepare
id: prep
run: |
DOCKER_IMAGE=crowdsecurity/crowdsec
GHCR_IMAGE=ghcr.io/${{ github.repository_owner }}/crowdsec
VERSION=edge
if [[ $GITHUB_REF == refs/tags/* ]]; then
VERSION=${GITHUB_REF#refs/tags/}
elif [[ $GITHUB_REF == refs/heads/* ]]; then
VERSION=$(echo ${GITHUB_REF#refs/heads/} | sed -E 's#/+#-#g')
elif [[ $GITHUB_REF == refs/pull/* ]]; then
VERSION=pr-${{ github.event.number }}
fi
TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}"
TAGS_SLIM="${DOCKER_IMAGE}:${VERSION}-slim"
if [[ ${{ github.event.action }} == released ]]; then
TAGS=$TAGS,${DOCKER_IMAGE}:latest,${GHCR_IMAGE}:latest
TAGS_SLIM=$TAGS_SLIM,${DOCKER_IMAGE}:slim
fi
echo "version=${VERSION}" >> $GITHUB_OUTPUT
echo "tags=${TAGS}" >> $GITHUB_OUTPUT
echo "tags_slim=${TAGS_SLIM}" >> $GITHUB_OUTPUT
echo "created=$(date -u +'%Y-%m-%dT%H:%M:%SZ')" >> $GITHUB_OUTPUT
- name: Set up QEMU
uses: docker/setup-qemu-action@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
with:
config: .github/buildkit.toml
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ secrets.DOCKER_USERNAME }}
password: ${{ secrets.DOCKER_PASSWORD }}
- name: Login to GitHub Container Registry
uses: docker/login-action@v2
with:
registry: ghcr.io
username: ${{ github.repository_owner }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push slim image
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.prep.outputs.tags_slim }}
target: slim
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6,linux/386
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.prep.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}
- name: Build and push full image
uses: docker/build-push-action@v4
with:
context: .
file: ./Dockerfile
push: ${{ github.event_name != 'pull_request' }}
tags: ${{ steps.prep.outputs.tags }}
platforms: linux/amd64,linux/arm64,linux/arm/v7,linux/arm/v6,linux/386
labels: |
org.opencontainers.image.source=${{ github.event.repository.html_url }}
org.opencontainers.image.created=${{ steps.prep.outputs.created }}
org.opencontainers.image.revision=${{ github.sha }}

View file

@ -1,4 +1,4 @@
name: (push-master) Update Docker Hub README
name: Update Docker Hub README
on:
push:
@ -13,7 +13,7 @@ jobs:
steps:
-
name: Check out the repo
uses: actions/checkout@v4
uses: actions/checkout@v3
if: ${{ github.repository_owner == 'crowdsecurity' }}
-
name: Update docker hub README

26
.gitignore vendored
View file

@ -6,25 +6,12 @@
*.dylib
*~
.pc
# IDEs
.vscode
.idea
# If vendor is included, allow prebuilt (wasm?) libraries.
!vendor/**/*.so
# Test binaries, built with `go test -c`
*.test
*.cover
# Test dependencies
test/tools/*
# VMs used for dev/test
.vagrant
# Test binaries, built from *_test.go
pkg/csplugin/tests/cs_plugin_test*
@ -36,21 +23,24 @@ test/coverage/*
*.swp
*.swo
# Dependencies are not vendored by default, but a tarball is created by "make vendor"
# and provided in the release. Used by gentoo, etc.
vendor/
vendor.tgz
# Dependency directories (remove the comment below to include it)
# vendor/
# crowdsec binaries
cmd/crowdsec-cli/cscli
cmd/crowdsec/crowdsec
cmd/notification-*/notification-*
plugins/notifications/http/notification-http
plugins/notifications/slack/notification-slack
plugins/notifications/splunk/notification-splunk
plugins/notifications/email/notification-email
plugins/notifications/dummy/notification-dummy
# Test cache (downloaded files)
.cache
# Release stuff
crowdsec-v*
pkg/cwhub/hubdir/.index.json
msi
*.msi
**/*.nupkg

View file

@ -1,66 +1,38 @@
# https://github.com/golangci/golangci-lint/blob/master/.golangci.reference.yml
run:
skip-dirs:
- pkg/time/rate
skip-files:
- pkg/database/ent/generate.go
- pkg/yamlpatch/merge.go
- pkg/yamlpatch/merge_test.go
linters-settings:
cyclop:
# lower this after refactoring
max-complexity: 48
gci:
sections:
- standard
- default
- prefix(github.com/crowdsecurity)
- prefix(github.com/crowdsecurity/crowdsec)
gomoddirectives:
replace-allow-list:
- golang.org/x/time/rate
gocognit:
# lower this after refactoring
min-complexity: 145
gocyclo:
# lower this after refactoring
min-complexity: 48
min-complexity: 30
funlen:
# Checks the number of lines in a function.
# If lower than 0, disable the check.
# Default: 60
# lower this after refactoring
lines: 437
lines: -1
# Checks the number of statements in a function.
# If lower than 0, disable the check.
# Default: 40
# lower this after refactoring
statements: 122
statements: -1
govet:
enable-all: true
disable:
- reflectvaluecompare
- fieldalignment
check-shadowing: true
lll:
# lower this after refactoring
line-length: 2607
maintidx:
# raise this after refactoring
under: 11
line-length: 140
misspell:
locale: US
nestif:
# lower this after refactoring
min-complexity: 28
nlreturn:
block-size: 5
nolintlint:
allow-leading-space: true # don't require machine-readable nolint directives (i.e. with no leading space)
allow-unused: false # report any unused nolint directives
require-explanation: false # don't require an explanation for nolint directives
require-specific: false # don't require nolint directives to be specific about which linter is being skipped
@ -68,183 +40,104 @@ linters-settings:
interfacebloat:
max: 12
depguard:
rules:
wrap:
deny:
- pkg: "github.com/pkg/errors"
desc: "errors.Wrap() is deprecated in favor of fmt.Errorf()"
files:
- "!**/pkg/database/*.go"
- "!**/pkg/exprhelpers/*.go"
- "!**/pkg/acquisition/modules/appsec/appsec.go"
- "!**/pkg/acquisition/modules/loki/internal/lokiclient/loki_client.go"
- "!**/pkg/apiserver/controllers/v1/errors.go"
yaml:
files:
- "!**/pkg/acquisition/acquisition.go"
- "!**/pkg/acquisition/acquisition_test.go"
- "!**/pkg/acquisition/modules/appsec/appsec.go"
- "!**/pkg/acquisition/modules/cloudwatch/cloudwatch.go"
- "!**/pkg/acquisition/modules/docker/docker.go"
- "!**/pkg/acquisition/modules/file/file.go"
- "!**/pkg/acquisition/modules/journalctl/journalctl.go"
- "!**/pkg/acquisition/modules/kafka/kafka.go"
- "!**/pkg/acquisition/modules/kinesis/kinesis.go"
- "!**/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go"
- "!**/pkg/acquisition/modules/loki/loki.go"
- "!**/pkg/acquisition/modules/loki/timestamp_test.go"
- "!**/pkg/acquisition/modules/s3/s3.go"
- "!**/pkg/acquisition/modules/syslog/syslog.go"
- "!**/pkg/acquisition/modules/wineventlog/wineventlog_windows.go"
- "!**/pkg/appsec/appsec.go"
- "!**/pkg/appsec/loader.go"
- "!**/pkg/csplugin/broker.go"
- "!**/pkg/csplugin/broker_test.go"
- "!**/pkg/dumps/bucket_dump.go"
- "!**/pkg/dumps/parser_dump.go"
- "!**/pkg/hubtest/coverage.go"
- "!**/pkg/hubtest/hubtest_item.go"
- "!**/pkg/hubtest/parser_assert.go"
- "!**/pkg/hubtest/scenario_assert.go"
- "!**/pkg/leakybucket/buckets_test.go"
- "!**/pkg/leakybucket/manager_load.go"
- "!**/pkg/metabase/metabase.go"
- "!**/pkg/parser/node.go"
- "!**/pkg/parser/node_test.go"
- "!**/pkg/parser/parsing_test.go"
- "!**/pkg/parser/stage.go"
deny:
- pkg: "gopkg.in/yaml.v2"
desc: "yaml.v2 is deprecated for new code in favor of yaml.v3"
wsl:
# Allow blocks to end with comments
allow-trailing-comment: true
linters:
enable-all: true
disable:
#
# DEPRECATED by golangi-lint
#
- deadcode
- exhaustivestruct
- golint
- ifshort
- interfacer
- maligned
- nosnakecase
- scopelint
- structcheck
- varcheck
#
# Disabled until fixed for go 1.22
#
- copyloopvar # copyloopvar is a linter detects places where loop variables are copied
- intrange # intrange is a linter to find places where for loops could make use of an integer range.
- deadcode # The owner seems to have abandoned the linter. Replaced by unused.
- exhaustivestruct # The owner seems to have abandoned the linter. Replaced by exhaustruct.
- golint # Golint differs from gofmt. Gofmt reformats Go source code, whereas golint prints out style mistakes
- ifshort # Checks that your code uses short syntax for if-statements whenever possible
- interfacer # Linter that suggests narrower interface types
- maligned # Tool to detect Go structs that would take less memory if their fields were sorted
- nosnakecase # nosnakecase is a linter that detects snake case of variable naming and function name.
- scopelint # Scopelint checks for unpinned variables in go programs
- structcheck # The owner seems to have abandoned the linter. Replaced by unused.
- varcheck # The owner seems to have abandoned the linter. Replaced by unused.
#
# Enabled
#
# - asasalint # check for pass []any as any in variadic func(...any)
# - asciicheck # checks that all code identifiers does not have non-ASCII symbols in the name
# - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers
# - bidichk # Checks for dangerous unicode character sequences
# - bodyclose # checks whether HTTP response body is closed successfully
# - cyclop # checks function and package cyclomatic complexity
# - decorder # check declaration order and count of types, constants, variables and functions
# - depguard # Go linter that checks if package imports are in a list of acceptable packages
# - dupword # checks for duplicate words in the source code
# - durationcheck # check for two durations multiplied together
# - errcheck # errcheck is a program for checking for unchecked errors in Go code. These unchecked errors can be critical bugs in some cases
# - errorlint # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13.
# - execinquery # execinquery is a linter about query string checker in Query function which reads your Go src files and warning it finds
# - errcheck # Errcheck is a program for checking for unchecked errors in go programs. These unchecked errors can be critical bugs in some cases
# - exportloopref # checks for pointers to enclosing loop variables
# - funlen # Tool for detection of long functions
# - ginkgolinter # enforces standards of using ginkgo and gomega
# - gocheckcompilerdirectives # Checks that go compiler directive comments (//go:) are valid.
# - gochecknoinits # Checks that no init functions are present in Go code
# - gochecksumtype # Run exhaustiveness checks on Go "sum types"
# - gocognit # Computes and checks the cognitive complexity of functions
# - gocritic # Provides diagnostics that check for bugs, performance and style issues.
# - gocyclo # Computes and checks the cyclomatic complexity of functions
# - goheader # Checks is file header matches to pattern
# - gomoddirectives # Manage the use of 'replace', 'retract', and 'excludes' directives in go.mod.
# - gomodguard # Allow and block list linter for direct Go module dependencies. This is different from depguard where there are different block types for example version constraints and module recommendations.
# - goprintffuncname # Checks that printf-like functions are named with `f` at the end
# - gosimple # (megacheck): Linter for Go source code that specializes in simplifying code
# - gosmopolitan # Report certain i18n/l10n anti-patterns in your Go codebase
# - govet # (vet, vetshadow): Vet examines Go source code and reports suspicious constructs. It is roughly the same as 'go vet' and uses its passes.
# - grouper # Analyze expression groups.
# - gosimple # (megacheck): Linter for Go source code that specializes in simplifying a code
# - govet # (vet, vetshadow): Vet examines Go source code and reports suspicious constructs, such as Printf calls whose arguments do not align with the format string
# - grouper # An analyzer to analyze expression groups.
# - importas # Enforces consistent import aliases
# - ineffassign # Detects when assignments to existing variables are not used
# - interfacebloat # A linter that checks the number of methods inside an interface.
# - lll # Reports long lines
# - loggercheck # (logrlint): Checks key value pairs for common logger libraries (kitlog,klog,logr,zap).
# - logrlint # Check logr arguments.
# - maintidx # maintidx measures the maintainability index of each function.
# - makezero # Finds slice declarations with non-zero initial length
# - mirror # reports wrong mirror patterns of bytes/strings usage
# - misspell # Finds commonly misspelled English words
# - nakedret # Checks that functions with naked returns are not longer than a maximum size (can be zero).
# - nestif # Reports deeply nested if statements
# - misspell # Finds commonly misspelled English words in comments
# - nilerr # Finds the code that returns nil even if it checks that the error is not nil.
# - nolintlint # Reports ill-formed or insufficient nolint directives
# - nonamedreturns # Reports all named returns
# - nosprintfhostport # Checks for misuse of Sprintf to construct a host with port in a URL.
# - perfsprint # Checks that fmt.Sprintf can be replaced with a faster alternative.
# - predeclared # find code that shadows one of Go's predeclared identifiers
# - reassign # Checks that package variables are not reassigned
# - rowserrcheck # checks whether Rows.Err of rows is checked successfully
# - sloglint # ensure consistent code style when using log/slog
# - spancheck # Checks for mistakes with OpenTelemetry/Census spans.
# - sqlclosecheck # Checks that sql.Rows, sql.Stmt, sqlx.NamedStmt, pgx.Query are closed.
# - staticcheck # (megacheck): It's a set of rules from staticcheck. It's not the same thing as the staticcheck binary. The author of staticcheck doesn't support or approve the use of staticcheck as a library inside golangci-lint.
# - tenv # tenv is analyzer that detects using os.Setenv instead of t.Setenv since Go1.17
# - rowserrcheck # checks whether Err of rows is checked successfully
# - sqlclosecheck # Checks that sql.Rows and sql.Stmt are closed.
# - staticcheck # (megacheck): Staticcheck is a go vet on steroids, applying a ton of static analysis checks
# - testableexamples # linter checks if examples are testable (have an expected output)
# - testifylint # Checks usage of github.com/stretchr/testify.
# - tenv # tenv is analyzer that detects using os.Setenv instead of t.Setenv since Go1.17
# - tparallel # tparallel detects inappropriate usage of t.Parallel() method in your Go test codes
# - typecheck # Like the front-end of a Go compiler, parses and type-checks Go code
# - unconvert # Remove unnecessary type conversions
# - unused # (megacheck): Checks Go code for unused constants, variables, functions and types
# - usestdlibvars # A linter that detect the possibility to use variables/constants from the Go standard library.
# - wastedassign # Finds wasted assignment statements
# - zerologlint # Detects the wrong usage of `zerolog` that a user forgets to dispatch with `Send` or `Msg`
#
# Recommended? (easy)
#
- dogsled # Checks assignments with too many blank identifiers (e.g. x, _, _, _, := f())
- errchkjson # Checks types passed to the json encoding functions. Reports unsupported types and reports occations, where the check for the returned error can be omitted.
- errchkjson # Checks types passed to the json encoding functions. Reports unsupported types and optionally reports occations, where the check for the returned error can be omitted.
- errorlint # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13.
- exhaustive # check exhaustiveness of enum switch statements
- gci # Gci control golang package import order and make it always deterministic.
- godot # Check if comments end in a period
- gofmt # Gofmt checks whether code was gofmt-ed. By default this tool runs with -s option to check for code simplification
- goimports # Check import statements are formatted according to the 'goimport' command. Reformat imports in autofix mode.
- goimports # In addition to fixing imports, goimports also formats your code in the same style as gofmt.
- gosec # (gas): Inspects source code for security problems
- inamedparam # reports interfaces with unnamed method parameters
- lll # Reports long lines
- musttag # enforce field tags in (un)marshaled structs
- nakedret # Finds naked returns in functions greater than a specified function length
- nonamedreturns # Reports all named returns
- nosprintfhostport # Checks for misuse of Sprintf to construct a host with port in a URL.
- promlinter # Check Prometheus metrics naming via promlint
- protogetter # Reports direct reads from proto message fields when getters should be used
- revive # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint.
- tagalign # check that struct tags are well aligned
- thelper # thelper detects tests helpers which is not start with t.Helper() method.
- thelper # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers
- wastedassign # wastedassign finds wasted assignment statements.
- wrapcheck # Checks that errors returned from external packages are wrapped
#
# Recommended? (requires some work)
#
- bodyclose # checks whether HTTP response body is closed successfully
- containedctx # containedctx is a linter that detects struct contained context.Context field
- contextcheck # check whether the function uses a non-inherited context
- contextcheck # check the function whether use a non-inherited context
- errname # Checks that sentinel errors are prefixed with the `Err` and error types are suffixed with the `Error`.
- gomnd # An analyzer to detect magic numbers.
- ireturn # Accept Interfaces, Return Concrete Types
- nilnil # Checks that there is no simultaneous return of `nil` error and an invalid value.
- noctx # Finds sending http request without context.Context
- noctx # noctx finds sending http request without context.Context
- unparam # Reports unused function parameters
#
@ -253,25 +146,31 @@ linters:
- gofumpt # Gofumpt checks whether code was gofumpt-ed.
- nlreturn # nlreturn checks for a new line before return and branch statements to increase code clarity
- whitespace # Whitespace is a linter that checks for unnecessary newlines at the start and end of functions, if, for, etc.
- wsl # add or remove empty lines
- whitespace # Tool for detection of leading and trailing whitespace
- wsl # Whitespace Linter - Forces you to use empty lines!
#
# Well intended, but not ready for this
#
- cyclop # checks function and package cyclomatic complexity
- dupl # Tool for code clone detection
- forcetypeassert # finds forced type assertions
- gocognit # Computes and checks the cognitive complexity of functions
- gocyclo # Computes and checks the cyclomatic complexity of functions
- godox # Tool for detection of FIXME, TODO and other comment keywords
- goerr113 # Go linter to check the errors handling expressions
- paralleltest # Detects missing usage of t.Parallel() method in your Go test
- goerr113 # Golang linter to check the errors handling expressions
- maintidx # maintidx measures the maintainability index of each function.
- nestif # Reports deeply nested if statements
- paralleltest # paralleltest detects missing usage of t.Parallel() method in your Go test
- testpackage # linter that makes you use a separate _test package
#
# Too strict / too many false positives (for now?)
#
- execinquery # execinquery is a linter about query string checker in Query function which reads your Go src files and warning it finds
- exhaustruct # Checks if all structure fields are initialized
- forbidigo # Forbids identifiers
- gochecknoglobals # Check that no global variables exist.
- gochecknoglobals # check that no global variables exist
- goconst # Finds repeated strings that could be replaced by a constant
- stylecheck # Stylecheck is a replacement for golint
- tagliatelle # Checks the struct tags.
@ -288,30 +187,29 @@ issues:
# “Look, thats why theres rules, understand? So that you think before you
# break em.” ― Terry Pratchett
exclude-dirs:
- pkg/time/rate
exclude-files:
- pkg/yamlpatch/merge.go
- pkg/yamlpatch/merge_test.go
exclude-generated-strict: true
max-issues-per-linter: 0
max-same-issues: 0
max-same-issues: 10
exclude-rules:
# Won't fix:
- path: go.mod
text: "replacement are not allowed: golang.org/x/time/rate"
# `err` is often shadowed, we may continue to do it
- linters:
- govet
text: "shadow: declaration of \"err\" shadows declaration"
#
# errcheck
#
- linters:
- errcheck
text: "Error return value of `.*` is not checked"
#
# gocritic
#
- linters:
- gocritic
text: "ifElseChain: rewrite if-else to switch statement"
@ -328,55 +226,6 @@ issues:
- gocritic
text: "commentFormatting: put a space between `//` and comment text"
# Will fix, trivial - just beware of merge conflicts
- linters:
- perfsprint
text: "fmt.Sprintf can be replaced .*"
- linters:
- perfsprint
text: "fmt.Errorf can be replaced with errors.New"
#
# Will fix, easy but some neurons required
#
- linters:
- errorlint
text: "non-wrapping format verb for fmt.Errorf. Use `%w` to format errors"
- linters:
- errorlint
text: "type assertion on error will fail on wrapped errors. Use errors.As to check for specific errors"
- linters:
- errorlint
text: "type switch on error will fail on wrapped errors. Use errors.As to check for specific errors"
- linters:
- errorlint
text: "type assertion on error will fail on wrapped errors. Use errors.Is to check for specific errors"
- linters:
- errorlint
text: "comparing with .* will fail on wrapped errors. Use errors.Is to check for a specific error"
- linters:
- errorlint
text: "switch on an error will fail on wrapped errors. Use errors.Is to check for specific errors"
- linters:
- nosprintfhostport
text: "host:port in url should be constructed with net.JoinHostPort and not directly with fmt.Sprintf"
# https://github.com/timakin/bodyclose
- linters:
- bodyclose
text: "response body must be closed"
# named/naked returns are evil, with a single exception
# https://go.dev/wiki/CodeReviewComments#named-result-parameters
- linters:
- nonamedreturns
text: "named return .* with type .* found"
- staticcheck
text: "x509.ParseCRL has been deprecated since Go 1.19: Use ParseRevocationList instead"

View file

@ -1,63 +1,59 @@
# vim: set ft=dockerfile:
FROM golang:1.22.2-alpine3.18 AS build
ARG GOVERSION=1.20.1
ARG BUILD_VERSION
FROM golang:${GOVERSION}-alpine AS build
WORKDIR /go/src/crowdsec
# We like to choose the release of re2 to use, and Alpine does not ship a static version anyway.
ENV RE2_VERSION=2023-03-01
ENV BUILD_VERSION=${BUILD_VERSION}
# wizard.sh requires GNU coreutils
RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold coreutils pkgconfig && \
wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \
tar -xzf ${RE2_VERSION}.tar.gz && \
cd re2-${RE2_VERSION} && \
make install && \
echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
go install github.com/mikefarah/yq/v4@v4.43.1
COPY . .
RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
# wizard.sh requires GNU coreutils
RUN apk add --no-cache git gcc libc-dev make bash gettext binutils-gold coreutils && \
echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
make clean release DOCKER_BUILD=1 && \
cd crowdsec-v* && \
./wizard.sh --docker-mode && \
cd - >/dev/null && \
cscli hub update && \
cscli collections install crowdsecurity/linux && \
cscli parsers install crowdsecurity/whitelists
cscli parsers install crowdsecurity/whitelists && \
go install github.com/mikefarah/yq/v4@v4.31.2
# In case we need to remove agents here..
# cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete
FROM alpine:latest as slim
RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata bash rsync && \
RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata bash && \
mkdir -p /staging/etc/crowdsec && \
mkdir -p /staging/etc/crowdsec/acquis.d && \
mkdir -p /staging/var/lib/crowdsec && \
mkdir -p /var/lib/crowdsec/data
COPY --from=build /go/bin/yq /usr/local/bin/crowdsec /usr/local/bin/cscli /usr/local/bin/
COPY --from=build /go/bin/yq /usr/local/bin/yq
COPY --from=build /etc/crowdsec /staging/etc/crowdsec
COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec
RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml
ENTRYPOINT /bin/bash /docker_start.sh
ENTRYPOINT /bin/bash docker_start.sh
FROM slim as full
FROM slim as plugins
# Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
# The files are here for reference, as users will need to mount a new version to be actually able to use notifications
COPY --from=build \
/go/src/crowdsec/cmd/notification-email/email.yaml \
/go/src/crowdsec/cmd/notification-http/http.yaml \
/go/src/crowdsec/cmd/notification-slack/slack.yaml \
/go/src/crowdsec/cmd/notification-splunk/splunk.yaml \
/go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml \
/staging/etc/crowdsec/notifications/
COPY --from=build /go/src/crowdsec/plugins/notifications/email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
FROM slim as geoip
COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec
FROM plugins as full
COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec

View file

@ -1,42 +1,32 @@
# vim: set ft=dockerfile:
FROM golang:1.22.2-bookworm AS build
ARG GOVERSION=1.20.1
ARG BUILD_VERSION
FROM golang:${GOVERSION}-bullseye AS build
WORKDIR /go/src/crowdsec
COPY . .
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_NOWARNINGS="yes"
# We like to choose the release of re2 to use, the debian version is usually older.
ENV RE2_VERSION=2023-03-01
ENV BUILD_VERSION=${BUILD_VERSION}
# wizard.sh requires GNU coreutils
RUN apt-get update && \
apt-get install -y -q git gcc libc-dev make bash gettext binutils-gold coreutils tzdata && \
wget https://github.com/google/re2/archive/refs/tags/${RE2_VERSION}.tar.gz && \
tar -xzf ${RE2_VERSION}.tar.gz && \
cd re2-${RE2_VERSION} && \
make && \
make install && \
echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \
go install github.com/mikefarah/yq/v4@v4.43.1
COPY . .
RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
make clean release DOCKER_BUILD=1 && \
cd crowdsec-v* && \
./wizard.sh --docker-mode && \
cd - >/dev/null && \
cscli hub update && \
cscli collections install crowdsecurity/linux && \
cscli parsers install crowdsecurity/whitelists
cscli parsers install crowdsecurity/whitelists && \
go install github.com/mikefarah/yq/v4@v4.31.2
# In case we need to remove agents here..
# cscli machines list -o json | yq '.[].machineId' | xargs -r cscli machines delete
FROM debian:bookworm-slim as slim
FROM debian:bullseye-slim as slim
ENV DEBIAN_FRONTEND=noninteractive
ENV DEBCONF_NOWARNINGS="yes"
@ -48,15 +38,16 @@ RUN apt-get update && \
iproute2 \
ca-certificates \
bash \
tzdata \
rsync && \
tzdata && \
mkdir -p /staging/etc/crowdsec && \
mkdir -p /staging/etc/crowdsec/acquis.d && \
mkdir -p /staging/var/lib/crowdsec && \
mkdir -p /var/lib/crowdsec/data
COPY --from=build /go/bin/yq /usr/local/bin/crowdsec /usr/local/bin/cscli /usr/local/bin/
COPY --from=build /go/bin/yq /usr/local/bin/yq
COPY --from=build /etc/crowdsec /staging/etc/crowdsec
COPY --from=build /usr/local/bin/crowdsec /usr/local/bin/crowdsec
COPY --from=build /usr/local/bin/cscli /usr/local/bin/cscli
COPY --from=build /go/src/crowdsec/docker/docker_start.sh /
COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml
RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml && \
@ -68,14 +59,10 @@ FROM slim as plugins
# Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
# The files are here for reference, as users will need to mount a new version to be actually able to use notifications
COPY --from=build \
/go/src/crowdsec/cmd/notification-email/email.yaml \
/go/src/crowdsec/cmd/notification-http/http.yaml \
/go/src/crowdsec/cmd/notification-slack/slack.yaml \
/go/src/crowdsec/cmd/notification-splunk/splunk.yaml \
/go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml \
/staging/etc/crowdsec/notifications/
COPY --from=build /go/src/crowdsec/plugins/notifications/email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
FROM slim as geoip

290
Makefile
View file

@ -1,79 +1,36 @@
include mk/platform.mk
include mk/gmsl
# By default, this build requires the C++ re2 library to be installed.
#
# Debian/Ubuntu: apt install libre2-dev
# Fedora/CentOS: dnf install re2-devel
# FreeBSD: pkg install re2
# Alpine: apk add re2-dev
# Windows: choco install re2
# MacOS: brew install re2
# To build without re2, run "make BUILD_RE2_WASM=1"
# The WASM version is slower and introduces a short delay when starting a process
# (including cscli) so it is not recommended for production use.
BUILD_RE2_WASM ?= 0
# To build static binaries, run "make BUILD_STATIC=1".
# On some platforms, this requires additional packages
# (e.g. glibc-static and libstdc++-static on fedora, centos.. which are on the powertools/crb repository).
# If the static build fails at the link stage, it might be because the static library is not provided
# for your distribution (look for libre2.a). See the Dockerfile for an example of how to build it.
BUILD_STATIC ?= 0
# List of plugins to build
PLUGINS ?= $(patsubst ./cmd/notification-%,%,$(wildcard ./cmd/notification-*))
# Can be overriden, if you can deal with the consequences
BUILD_REQUIRE_GO_MAJOR ?= 1
BUILD_REQUIRE_GO_MINOR ?= 21
#--------------------------------------
GO = go
GOTEST = $(GO) test
BUILD_CODENAME ?= alphaga
CROWDSEC_FOLDER = ./cmd/crowdsec
CSCLI_FOLDER = ./cmd/crowdsec-cli/
PLUGINS_DIR_PREFIX = ./cmd/notification-
HTTP_PLUGIN_FOLDER = ./plugins/notifications/http
SLACK_PLUGIN_FOLDER = ./plugins/notifications/slack
SPLUNK_PLUGIN_FOLDER = ./plugins/notifications/splunk
EMAIL_PLUGIN_FOLDER = ./plugins/notifications/email
DUMMY_PLUGIN_FOLDER = ./plugins/notifications/dummy
HTTP_PLUGIN_BIN = notification-http$(EXT)
SLACK_PLUGIN_BIN = notification-slack$(EXT)
SPLUNK_PLUGIN_BIN = notification-splunk$(EXT)
EMAIL_PLUGIN_BIN = notification-email$(EXT)
DUMMY_PLUGIN_BIN= notification-dummy$(EXT)
HTTP_PLUGIN_CONFIG = http.yaml
SLACK_PLUGIN_CONFIG = slack.yaml
SPLUNK_PLUGIN_CONFIG = splunk.yaml
EMAIL_PLUGIN_CONFIG = email.yaml
CROWDSEC_BIN = crowdsec$(EXT)
CSCLI_BIN = cscli$(EXT)
# semver comparison to select the hub branch requires the version to start with "v"
ifneq ($(call substr,$(BUILD_VERSION),1,1),v)
$(error BUILD_VERSION "$(BUILD_VERSION)" should start with "v")
endif
# Directory for the release files
RELDIR = crowdsec-$(BUILD_VERSION)
BUILD_CMD = build
GO_MODULE_NAME = github.com/crowdsecurity/crowdsec
# Check if a given value is considered truthy and returns "0" or "1".
# A truthy value is one of the following: "1", "yes", or "true", case-insensitive.
#
# Usage:
# ifeq ($(call bool,$(FOO)),1)
# $(info Let's foo)
# endif
bool = $(if $(filter $(call lc, $1),1 yes true),1,0)
#--------------------------------------
#
# Define MAKE_FLAGS and LD_OPTS for the sub-makefiles in cmd/
#
MAKE_FLAGS = --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
LD_OPTS_VARS= \
-X 'github.com/crowdsecurity/go-cs-lib/version.Version=$(BUILD_VERSION)' \
-X 'github.com/crowdsecurity/go-cs-lib/version.BuildDate=$(BUILD_TIMESTAMP)' \
-X 'github.com/crowdsecurity/go-cs-lib/version.Tag=$(BUILD_TAG)' \
-X '$(GO_MODULE_NAME)/pkg/cwversion.Version=$(BUILD_VERSION)' \
-X '$(GO_MODULE_NAME)/pkg/cwversion.BuildDate=$(BUILD_TIMESTAMP)' \
-X '$(GO_MODULE_NAME)/pkg/cwversion.Codename=$(BUILD_CODENAME)' \
-X '$(GO_MODULE_NAME)/pkg/cwversion.Tag=$(BUILD_TAG)' \
-X '$(GO_MODULE_NAME)/pkg/csconfig.defaultConfigDir=$(DEFAULT_CONFIGDIR)' \
-X '$(GO_MODULE_NAME)/pkg/csconfig.defaultDataDir=$(DEFAULT_DATADIR)'
@ -81,195 +38,129 @@ ifneq (,$(DOCKER_BUILD))
LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.System=docker'
endif
GO_TAGS := netgo,osusergo,sqlite_omit_load_extension
# this will be used by Go in the make target, some distributions require it
export PKG_CONFIG_PATH:=/usr/local/lib/pkgconfig:$(PKG_CONFIG_PATH)
ifeq ($(call bool,$(BUILD_RE2_WASM)),0)
ifeq ($(PKG_CONFIG),)
$(error "pkg-config is not available. Please install pkg-config.")
endif
ifeq ($(RE2_CHECK),)
RE2_FAIL := "libre2-dev is not installed, please install it or set BUILD_RE2_WASM=1 to use the WebAssembly version"
ifdef BUILD_STATIC
export LD_OPTS=-ldflags "-s -w $(LD_OPTS_VARS) -extldflags '-static'" -tags netgo,osusergo,sqlite_omit_load_extension
else
# += adds a space that we don't want
GO_TAGS := $(GO_TAGS),re2_cgo
LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.Libre2=C++'
endif
export LD_OPTS=-ldflags "-s -w $(LD_OPTS_VARS)"
endif
# Build static to avoid the runtime dependency on libre2.so
ifeq ($(call bool,$(BUILD_STATIC)),1)
BUILD_TYPE = static
EXTLDFLAGS := -extldflags '-static'
else
BUILD_TYPE = dynamic
EXTLDFLAGS :=
endif
LD_OPTS += -trimpath
# Build with debug symbols, and disable optimizations + inlining, to use Delve
ifeq ($(call bool,$(DEBUG)),1)
STRIP_SYMBOLS :=
DISABLE_OPTIMIZATION := -gcflags "-N -l"
else
STRIP_SYMBOLS := -s -w
DISABLE_OPTIMIZATION :=
endif
export LD_OPTS=-ldflags "$(STRIP_SYMBOLS) $(EXTLDFLAGS) $(LD_OPTS_VARS)" \
-trimpath -tags $(GO_TAGS) $(DISABLE_OPTIMIZATION)
ifeq ($(call bool,$(TEST_COVERAGE)),1)
ifneq (,$(TEST_COVERAGE))
LD_OPTS += -cover
endif
#--------------------------------------
GOCMD = go
GOTEST = $(GOCMD) test
RELDIR = crowdsec-$(BUILD_VERSION)
.PHONY: build
build: pre-build goversion crowdsec cscli plugins ## Build crowdsec, cscli and plugins
.PHONY: pre-build
pre-build: ## Sanity checks and build information
$(info Building $(BUILD_VERSION) ($(BUILD_TAG)) $(BUILD_TYPE) for $(GOOS)/$(GOARCH))
ifneq (,$(RE2_FAIL))
$(error $(RE2_FAIL))
endif
ifneq (,$(RE2_CHECK))
$(info Using C++ regexp library)
else
$(info Fallback to WebAssembly regexp library. To use the C++ version, make sure you have installed libre2-dev and pkg-config.)
endif
ifeq ($(call bool,$(DEBUG)),1)
$(info Building with debug symbols and disabled optimizations)
endif
ifeq ($(call bool,$(TEST_COVERAGE)),1)
$(info Test coverage collection enabled)
endif
# intentional, empty line
$(info )
build: goversion crowdsec cscli plugins
.PHONY: all
all: clean test build ## Clean, test and build (requires localstack)
all: clean test build
.PHONY: plugins
plugins: ## Build notification plugins
@$(foreach plugin,$(PLUGINS), \
$(MAKE) -C $(PLUGINS_DIR_PREFIX)$(plugin) build $(MAKE_FLAGS); \
)
# same as "$(MAKE) -f debian/rules clean" but without the dependency on debhelper
.PHONY: clean-debian
clean-debian:
@$(RM) -r debian/crowdsec
@$(RM) -r debian/crowdsec
@$(RM) -r debian/files
@$(RM) -r debian/.debhelper
@$(RM) -r debian/*.substvars
@$(RM) -r debian/*-stamp
.PHONY: clean-rpm
clean-rpm:
@$(RM) -r rpm/BUILD
@$(RM) -r rpm/BUILDROOT
@$(RM) -r rpm/RPMS
@$(RM) -r rpm/SOURCES/*.tar.gz
@$(RM) -r rpm/SRPMS
plugins: http-plugin slack-plugin splunk-plugin email-plugin dummy-plugin
.PHONY: clean
clean: clean-debian clean-rpm testclean ## Remove build artifacts
@$(MAKE) -C $(CROWDSEC_FOLDER) clean $(MAKE_FLAGS)
@$(MAKE) -C $(CSCLI_FOLDER) clean $(MAKE_FLAGS)
clean: testclean
@$(MAKE) -C $(CROWDSEC_FOLDER) clean --no-print-directory RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
@$(MAKE) -C $(CSCLI_FOLDER) clean --no-print-directory RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
@$(RM) $(CROWDSEC_BIN) $(WIN_IGNORE_ERR)
@$(RM) $(CSCLI_BIN) $(WIN_IGNORE_ERR)
@$(RM) *.log $(WIN_IGNORE_ERR)
@$(RM) crowdsec-release.tgz $(WIN_IGNORE_ERR)
@$(foreach plugin,$(PLUGINS), \
$(MAKE) -C $(PLUGINS_DIR_PREFIX)$(plugin) clean $(MAKE_FLAGS); \
)
@$(RM) crowdsec-release-static.tgz $(WIN_IGNORE_ERR)
@$(RM) $(HTTP_PLUGIN_FOLDER)/$(HTTP_PLUGIN_BIN) $(WIN_IGNORE_ERR)
@$(RM) $(SLACK_PLUGIN_FOLDER)/$(SLACK_PLUGIN_BIN) $(WIN_IGNORE_ERR)
@$(RM) $(SPLUNK_PLUGIN_FOLDER)/$(SPLUNK_PLUGIN_BIN) $(WIN_IGNORE_ERR)
@$(RM) $(EMAIL_PLUGIN_FOLDER)/$(EMAIL_PLUGIN_BIN) $(WIN_IGNORE_ERR)
@$(RM) $(DUMMY_PLUGIN_FOLDER)/$(DUMMY_PLUGIN_BIN) $(WIN_IGNORE_ERR)
.PHONY: cscli
cscli: goversion ## Build cscli
@$(MAKE) -C $(CSCLI_FOLDER) build $(MAKE_FLAGS)
.PHONY: crowdsec
crowdsec: goversion ## Build crowdsec
@$(MAKE) -C $(CROWDSEC_FOLDER) build $(MAKE_FLAGS)
cscli: goversion
@$(MAKE) -C $(CSCLI_FOLDER) build --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
.PHONY: generate
generate: ## Generate code for the database and APIs
$(GO) generate ./pkg/database/ent
$(GO) generate ./pkg/models
crowdsec: goversion
@$(MAKE) -C $(CROWDSEC_FOLDER) build --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
http-plugin: goversion
@$(MAKE) -C $(HTTP_PLUGIN_FOLDER) build --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
slack-plugin: goversion
@$(MAKE) -C $(SLACK_PLUGIN_FOLDER) build --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
splunk-plugin: goversion
@$(MAKE) -C $(SPLUNK_PLUGIN_FOLDER) build --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
email-plugin: goversion
@$(MAKE) -C $(EMAIL_PLUGIN_FOLDER) build --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
dummy-plugin: goversion
$(MAKE) -C $(DUMMY_PLUGIN_FOLDER) build --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
.PHONY: testclean
testclean: bats-clean ## Remove test artifacts
testclean: bats-clean
@$(RM) pkg/apiserver/ent $(WIN_IGNORE_ERR)
@$(RM) pkg/cwhub/hubdir $(WIN_IGNORE_ERR)
@$(RM) pkg/cwhub/install $(WIN_IGNORE_ERR)
@$(RM) pkg/types/example.txt $(WIN_IGNORE_ERR)
# for the tests with localstack
export AWS_ENDPOINT_FORCE=http://localhost:4566
export AWS_ACCESS_KEY_ID=test
export AWS_SECRET_ACCESS_KEY=test
export AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
export AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
testenv:
@echo 'NOTE: You need Docker, docker-compose and run "make localstack" in a separate shell ("make localstack-stop" to terminate it)'
.PHONY: test
test: testenv goversion ## Run unit tests with localstack
test: testenv goversion
$(GOTEST) $(LD_OPTS) ./...
.PHONY: go-acc
go-acc: testenv goversion ## Run unit tests with localstack + coverage
go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS)
go-acc: testenv goversion
go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS) | \
sed 's/ *coverage:.*of statements in.*//'
# mock AWS services
.PHONY: localstack
localstack: ## Run localstack containers (required for unit testing)
localstack:
docker-compose -f test/localstack/docker-compose.yml up
.PHONY: localstack-stop
localstack-stop: ## Stop localstack containers
localstack-stop:
docker-compose -f test/localstack/docker-compose.yml down
# build vendor.tgz to be distributed with the release
.PHONY: vendor
vendor: vendor-remove ## CI only - vendor dependencies and archive them for packaging
$(GO) mod vendor
tar czf vendor.tgz vendor
tar --create --auto-compress --file=$(RELDIR)-vendor.tar.xz vendor
# remove vendor directories and vendor.tgz
.PHONY: vendor-remove
vendor-remove: ## Remove vendor dependencies and archives
$(RM) vendor vendor.tgz *-vendor.tar.xz
.PHONY: package
package:
package-common:
@echo "Building Release to dir $(RELDIR)"
@$(MKDIR) $(RELDIR)/cmd/crowdsec
@$(MKDIR) $(RELDIR)/cmd/crowdsec-cli
@$(MKDIR) $(RELDIR)/$(subst ./,,$(HTTP_PLUGIN_FOLDER))
@$(MKDIR) $(RELDIR)/$(subst ./,,$(SLACK_PLUGIN_FOLDER))
@$(MKDIR) $(RELDIR)/$(subst ./,,$(SPLUNK_PLUGIN_FOLDER))
@$(MKDIR) $(RELDIR)/$(subst ./,,$(EMAIL_PLUGIN_FOLDER))
@$(CP) $(CROWDSEC_FOLDER)/$(CROWDSEC_BIN) $(RELDIR)/cmd/crowdsec
@$(CP) $(CSCLI_FOLDER)/$(CSCLI_BIN) $(RELDIR)/cmd/crowdsec-cli
@$(foreach plugin,$(PLUGINS), \
$(MKDIR) $(RELDIR)/$(PLUGINS_DIR_PREFIX)$(plugin); \
$(CP) $(PLUGINS_DIR_PREFIX)$(plugin)/notification-$(plugin)$(EXT) $(RELDIR)/$(PLUGINS_DIR_PREFIX)$(plugin); \
$(CP) $(PLUGINS_DIR_PREFIX)$(plugin)/$(plugin).yaml $(RELDIR)/$(PLUGINS_DIR_PREFIX)$(plugin)/; \
)
@$(CP) $(HTTP_PLUGIN_FOLDER)/$(HTTP_PLUGIN_BIN) $(RELDIR)/$(subst ./,,$(HTTP_PLUGIN_FOLDER))
@$(CP) $(SLACK_PLUGIN_FOLDER)/$(SLACK_PLUGIN_BIN) $(RELDIR)/$(subst ./,,$(SLACK_PLUGIN_FOLDER))
@$(CP) $(SPLUNK_PLUGIN_FOLDER)/$(SPLUNK_PLUGIN_BIN) $(RELDIR)/$(subst ./,,$(SPLUNK_PLUGIN_FOLDER))
@$(CP) $(EMAIL_PLUGIN_FOLDER)/$(EMAIL_PLUGIN_BIN) $(RELDIR)/$(subst ./,,$(EMAIL_PLUGIN_FOLDER))
@$(CP) $(HTTP_PLUGIN_FOLDER)/$(HTTP_PLUGIN_CONFIG) $(RELDIR)/$(subst ./,,$(HTTP_PLUGIN_FOLDER))
@$(CP) $(SLACK_PLUGIN_FOLDER)/$(SLACK_PLUGIN_CONFIG) $(RELDIR)/$(subst ./,,$(SLACK_PLUGIN_FOLDER))
@$(CP) $(SPLUNK_PLUGIN_FOLDER)/$(SPLUNK_PLUGIN_CONFIG) $(RELDIR)/$(subst ./,,$(SPLUNK_PLUGIN_FOLDER))
@$(CP) $(EMAIL_PLUGIN_FOLDER)/$(EMAIL_PLUGIN_CONFIG) $(RELDIR)/$(subst ./,,$(EMAIL_PLUGIN_FOLDER))
@$(CPR) ./config $(RELDIR)
@$(CP) wizard.sh $(RELDIR)
@$(CP) scripts/test_env.sh $(RELDIR)
@$(CP) scripts/test_env.ps1 $(RELDIR)
.PHONY: package
package: package-common
@tar cvzf crowdsec-release.tgz $(RELDIR)
.PHONY: check_release
@ -281,14 +172,14 @@ else
endif
.PHONY: release
release: check_release build package ## Build a release tarball
release: check_release build package
.PHONY: windows_installer
windows_installer: build ## Windows - build the installer
windows_installer: build
@.\make_installer.ps1 -version $(BUILD_VERSION)
.PHONY: chocolatey
chocolatey: windows_installer ## Windows - build the chocolatey package
chocolatey: windows_installer
@.\make_chocolatey.ps1 -version $(BUILD_VERSION)
# Include test/bats.mk only if it exists
@ -301,4 +192,3 @@ include test/bats.mk
endif
include mk/goversion.mk
include mk/help.mk

View file

@ -15,13 +15,19 @@ pool:
stages:
- stage: Build
jobs:
- job: Build
- job:
displayName: "Build"
steps:
- task: GoTool@0
displayName: "Install Go"
- task: DotNetCoreCLI@2
displayName: "Install SignClient"
inputs:
version: '1.22.2'
command: 'custom'
custom: 'tool'
arguments: 'install --global SignClient --version 1.3.155'
- task: GoTool@0
displayName: "Install Go 1.20"
inputs:
version: '1.20.1'
- pwsh: |
choco install -y make
@ -32,15 +38,25 @@ stages:
pwsh: true
#we are not calling make windows_installer because we want to sign the binaries before they are added to the MSI
script: |
make build BUILD_RE2_WASM=1
make build
- task: AzureKeyVault@2
inputs:
azureSubscription: 'Azure subscription 1(8a93ab40-7e99-445e-ad47-0f6a3e2ef546)'
KeyVaultName: 'CodeSigningSecrets'
SecretsFilter: 'CodeSigningUser,CodeSigningPassword'
RunAsPreJob: false
- task: DownloadSecureFile@1
inputs:
secureFile: appsettings.json
- pwsh: |
SignClient.exe Sign --name "crowdsec-binaries" `
--input "**/*.exe" --config (Join-Path -Path $(Agent.TempDirectory) -ChildPath "appsettings.json") `
--user $(CodeSigningUser) --secret '$(CodeSigningPassword)'
displayName: "Sign Crowdsec binaries + plugins"
- pwsh: |
$build_version=$env:BUILD_SOURCEBRANCHNAME
#Override the version if it's set in the pipeline
if ( ${env:USERBUILDVERSION} -ne "")
{
$build_version = ${env:USERBUILDVERSION}
}
if ($build_version.StartsWith("v"))
{
$build_version = $build_version.Substring(1)
@ -53,112 +69,35 @@ stages:
displayName: GetCrowdsecVersion
name: GetCrowdsecVersion
- pwsh: |
Get-ChildItem -Path .\cmd -Directory | ForEach-Object {
$dirName = $_.Name
Get-ChildItem -Path .\cmd\$dirName -File -Filter '*.exe' | ForEach-Object {
$fileName = $_.Name
$destDir = Join-Path $(Build.ArtifactStagingDirectory) cmd\$dirName
New-Item -ItemType Directory -Path $destDir -Force
Copy-Item -Path .\cmd\$dirName\$fileName -Destination $destDir
}
}
displayName: "Copy binaries to staging directory"
- task: PublishPipelineArtifact@1
inputs:
targetPath: '$(Build.ArtifactStagingDirectory)'
artifact: 'unsigned_binaries'
displayName: "Upload binaries artifact"
- stage: Sign
dependsOn: Build
variables:
- group: 'FOSS Build Variables'
- name: BuildVersion
value: $[ stageDependencies.Build.Build.outputs['GetCrowdsecVersion.BuildVersion'] ]
condition: succeeded()
jobs:
- job: Sign
displayName: "Sign"
steps:
- download: current
artifact: unsigned_binaries
displayName: "Download binaries artifact"
- task: CopyFiles@2
inputs:
SourceFolder: '$(Pipeline.Workspace)/unsigned_binaries'
TargetFolder: '$(Build.SourcesDirectory)'
displayName: "Copy binaries to workspace"
- task: DotNetCoreCLI@2
displayName: "Install SignTool tool"
inputs:
command: 'custom'
custom: 'tool'
arguments: install --global sign --version 0.9.0-beta.23127.3
- task: AzureKeyVault@2
displayName: "Get signing parameters"
inputs:
azureSubscription: "Azure subscription"
KeyVaultName: "$(KeyVaultName)"
SecretsFilter: "TenantId,ClientId,ClientSecret,Certificate,KeyVaultUrl"
- pwsh: |
sign code azure-key-vault `
"**/*.exe" `
--base-directory "$(Build.SourcesDirectory)/cmd/" `
--publisher-name "CrowdSec" `
--description "CrowdSec" `
--description-url "https://github.com/crowdsecurity/crowdsec" `
--azure-key-vault-tenant-id "$(TenantId)" `
--azure-key-vault-client-id "$(ClientId)" `
--azure-key-vault-client-secret "$(ClientSecret)" `
--azure-key-vault-certificate "$(Certificate)" `
--azure-key-vault-url "$(KeyVaultUrl)"
displayName: "Sign crowdsec binaries"
- pwsh: |
.\make_installer.ps1 -version '$(BuildVersion)'
.\make_installer.ps1 -version '$(GetCrowdsecVersion.BuildVersion)'
displayName: "Build Crowdsec MSI"
name: BuildMSI
- pwsh: |
.\make_chocolatey.ps1 -version '$(BuildVersion)'
.\make_chocolatey.ps1 -version '$(GetCrowdsecVersion.BuildVersion)'
displayName: "Build Chocolatey nupkg"
- pwsh: |
sign code azure-key-vault `
"*.msi" `
--base-directory "$(Build.SourcesDirectory)" `
--publisher-name "CrowdSec" `
--description "CrowdSec" `
--description-url "https://github.com/crowdsecurity/crowdsec" `
--azure-key-vault-tenant-id "$(TenantId)" `
--azure-key-vault-client-id "$(ClientId)" `
--azure-key-vault-client-secret "$(ClientSecret)" `
--azure-key-vault-certificate "$(Certificate)" `
--azure-key-vault-url "$(KeyVaultUrl)"
displayName: "Sign MSI package"
- pwsh: |
sign code azure-key-vault `
"*.nupkg" `
--base-directory "$(Build.SourcesDirectory)" `
--publisher-name "CrowdSec" `
--description "CrowdSec" `
--description-url "https://github.com/crowdsecurity/crowdsec" `
--azure-key-vault-tenant-id "$(TenantId)" `
--azure-key-vault-client-id "$(ClientId)" `
--azure-key-vault-client-secret "$(ClientSecret)" `
--azure-key-vault-certificate "$(Certificate)" `
--azure-key-vault-url "$(KeyVaultUrl)"
displayName: "Sign nuget package"
- task: PublishPipelineArtifact@1
SignClient.exe Sign --name "crowdsec-msi" `
--input "*.msi" --config (Join-Path -Path $(Agent.TempDirectory) -ChildPath "appsettings.json") `
--user $(CodeSigningUser) --secret '$(CodeSigningPassword)'
displayName: "Sign Crowdsec MSI"
- task: PublishBuildArtifacts@1
inputs:
targetPath: '$(Build.SourcesDirectory)/crowdsec_$(BuildVersion).msi'
artifact: 'signed_msi_package'
displayName: "Upload signed MSI artifact"
- task: PublishPipelineArtifact@1
PathtoPublish: '$(Build.Repository.LocalPath)\\crowdsec_$(GetCrowdsecVersion.BuildVersion).msi'
ArtifactName: 'crowdsec.msi'
publishLocation: 'Container'
displayName: "Upload MSI artifact"
- task: PublishBuildArtifacts@1
inputs:
targetPath: '$(Build.SourcesDirectory)/crowdsec.$(BuildVersion).nupkg'
artifact: 'signed_nuget_package'
displayName: "Upload signed nuget artifact"
PathtoPublish: '$(Build.Repository.LocalPath)\\windows\\Chocolatey\\crowdsec\\crowdsec.$(GetCrowdsecVersion.BuildVersion).nupkg'
ArtifactName: 'crowdsec.nupkg'
publishLocation: 'Container'
displayName: "Upload nupkg artifact"
- stage: Publish
dependsOn: Sign
dependsOn: Build
jobs:
- deployment: "Publish"
displayName: "Publish to GitHub"
@ -180,7 +119,8 @@ stages:
assetUploadMode: 'replace'
addChangeLog: false
isPreRelease: true #we force prerelease because the pipeline is invoked on tag creation, which happens when we do a prerelease
#the .. is an ugly hack, but I can't find the var that gives D:\a\1 ...
assets: |
$(Pipeline.Workspace)/signed_msi_package/*.msi
$(Pipeline.Workspace)/signed_nuget_package/*.nupkg
$(Build.ArtifactStagingDirectory)\..\crowdsec.msi/*.msi
$(Build.ArtifactStagingDirectory)\..\crowdsec.nupkg/*.nupkg
condition: ne(variables['GetLatestPrelease.LatestPreRelease'], '')

View file

@ -4,8 +4,12 @@ ifeq ($(OS), Windows_NT)
EXT = .exe
endif
GO = go
GOBUILD = $(GO) build
# Go parameters
GOCMD = go
GOBUILD = $(GOCMD) build
GOCLEAN = $(GOCMD) clean
GOTEST = $(GOCMD) test
GOGET = $(GOCMD) get
BINARY_NAME = cscli$(EXT)
PREFIX ?= "/"
@ -15,7 +19,7 @@ BIN_PREFIX = $(PREFIX)"/usr/local/bin/"
all: clean build
build: clean
$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
.PHONY: install
install: install-conf install-bin

View file

@ -4,7 +4,6 @@ import (
"context"
"encoding/csv"
"encoding/json"
"errors"
"fmt"
"net/url"
"os"
@ -12,17 +11,17 @@ import (
"strconv"
"strings"
"text/template"
"time"
"github.com/fatih/color"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
@ -30,46 +29,83 @@ import (
func DecisionsFromAlert(alert *models.Alert) string {
ret := ""
decMap := make(map[string]int)
var decMap = make(map[string]int)
for _, decision := range alert.Decisions {
k := *decision.Type
if *decision.Simulated {
k = fmt.Sprintf("(simul)%s", k)
}
v := decMap[k]
decMap[k] = v + 1
}
for k, v := range decMap {
if len(ret) > 0 {
ret += " "
}
ret += fmt.Sprintf("%s:%d", k, v)
}
return ret
}
func (cli *cliAlerts) alertsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
switch cli.cfg().Cscli.Output {
case "raw":
func DateFromAlert(alert *models.Alert) string {
ts, err := time.Parse(time.RFC3339, alert.CreatedAt)
if err != nil {
log.Infof("while parsing %s with %s : %s", alert.CreatedAt, time.RFC3339, err)
return alert.CreatedAt
}
return ts.Format(time.RFC822)
}
func SourceFromAlert(alert *models.Alert) string {
//more than one item, just number and scope
if len(alert.Decisions) > 1 {
return fmt.Sprintf("%d %ss (%s)", len(alert.Decisions), *alert.Decisions[0].Scope, *alert.Decisions[0].Origin)
}
//fallback on single decision information
if len(alert.Decisions) == 1 {
return fmt.Sprintf("%s:%s", *alert.Decisions[0].Scope, *alert.Decisions[0].Value)
}
//try to compose a human friendly version
if *alert.Source.Value != "" && *alert.Source.Scope != "" {
scope := ""
scope = fmt.Sprintf("%s:%s", *alert.Source.Scope, *alert.Source.Value)
extra := ""
if alert.Source.Cn != "" {
extra = alert.Source.Cn
}
if alert.Source.AsNumber != "" {
extra += fmt.Sprintf("/%s", alert.Source.AsNumber)
}
if alert.Source.AsName != "" {
extra += fmt.Sprintf("/%s", alert.Source.AsName)
}
if extra != "" {
scope += " (" + extra + ")"
}
return scope
}
return ""
}
func AlertsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
if csConfig.Cscli.Output == "raw" {
csvwriter := csv.NewWriter(os.Stdout)
header := []string{"id", "scope", "value", "reason", "country", "as", "decisions", "created_at"}
if printMachine {
header = append(header, "machine")
}
if err := csvwriter.Write(header); err != nil {
err := csvwriter.Write(header)
if err != nil {
return err
}
for _, alertItem := range *alerts {
row := []string{
strconv.FormatInt(alertItem.ID, 10),
fmt.Sprintf("%d", alertItem.ID),
*alertItem.Source.Scope,
*alertItem.Source.Value,
*alertItem.Scenario,
@ -81,32 +117,22 @@ func (cli *cliAlerts) alertsToTable(alerts *models.GetAlertsResponse, printMachi
if printMachine {
row = append(row, alertItem.MachineID)
}
if err := csvwriter.Write(row); err != nil {
err := csvwriter.Write(row)
if err != nil {
return err
}
}
csvwriter.Flush()
case "json":
if *alerts == nil {
// avoid returning "null" in json
// could be cleaner if we used slice of alerts directly
fmt.Println("[]")
return nil
}
} else if csConfig.Cscli.Output == "json" {
x, _ := json.MarshalIndent(alerts, "", " ")
fmt.Print(string(x))
case "human":
fmt.Printf("%s", string(x))
} else if csConfig.Cscli.Output == "human" {
if len(*alerts) == 0 {
fmt.Println("No active alerts")
return nil
}
alertsTable(color.Output, alerts, printMachine)
}
return nil
}
@ -128,109 +154,93 @@ var alertTemplate = `
`
func (cli *cliAlerts) displayOneAlert(alert *models.Alert, withDetail bool) error {
tmpl, err := template.New("alert").Parse(alertTemplate)
if err != nil {
return err
}
if err = tmpl.Execute(os.Stdout, alert); err != nil {
return err
}
alertDecisionsTable(color.Output, alert)
if len(alert.Meta) > 0 {
fmt.Printf("\n - Context :\n")
sort.Slice(alert.Meta, func(i, j int) bool {
return alert.Meta[i].Key < alert.Meta[j].Key
})
table := newTable(color.Output)
table.SetRowLines(false)
table.SetHeaders("Key", "Value")
for _, meta := range alert.Meta {
var valSlice []string
if err := json.Unmarshal([]byte(meta.Value), &valSlice); err != nil {
return fmt.Errorf("unknown context value type '%s': %w", meta.Value, err)
}
for _, value := range valSlice {
table.AddRow(
meta.Key,
value,
)
}
func DisplayOneAlert(alert *models.Alert, withDetail bool) error {
if csConfig.Cscli.Output == "human" {
tmpl, err := template.New("alert").Parse(alertTemplate)
if err != nil {
return err
}
err = tmpl.Execute(os.Stdout, alert)
if err != nil {
return err
}
table.Render()
}
alertDecisionsTable(color.Output, alert)
if withDetail {
fmt.Printf("\n - Events :\n")
if len(alert.Meta) > 0 {
fmt.Printf("\n - Context :\n")
sort.Slice(alert.Meta, func(i, j int) bool {
return alert.Meta[i].Key < alert.Meta[j].Key
})
table := newTable(color.Output)
table.SetRowLines(false)
table.SetHeaders("Key", "Value")
for _, meta := range alert.Meta {
var valSlice []string
if err := json.Unmarshal([]byte(meta.Value), &valSlice); err != nil {
return fmt.Errorf("unknown context value type '%s' : %s", meta.Value, err)
}
for _, value := range valSlice {
table.AddRow(
meta.Key,
value,
)
}
}
table.Render()
}
for _, event := range alert.Events {
alertEventTable(color.Output, event)
if withDetail {
fmt.Printf("\n - Events :\n")
for _, event := range alert.Events {
alertEventTable(color.Output, event)
}
}
}
return nil
}
type cliAlerts struct {
client *apiclient.ApiClient
cfg configGetter
}
func NewCLIAlerts(getconfig configGetter) *cliAlerts {
return &cliAlerts{
cfg: getconfig,
}
}
func (cli *cliAlerts) NewCommand() *cobra.Command {
cmd := &cobra.Command{
func NewAlertsCmd() *cobra.Command {
var cmdAlerts = &cobra.Command{
Use: "alerts [action]",
Short: "Manage alerts",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
Aliases: []string{"alert"},
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := cfg.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
var err error
if err := csConfig.LoadAPIClient(); err != nil {
return errors.Wrap(err, "loading api client")
}
apiURL, err := url.Parse(cfg.API.Client.Credentials.URL)
apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
if err != nil {
return fmt.Errorf("parsing api url %s: %w", apiURL, err)
return errors.Wrapf(err, "parsing api url %s", apiURL)
}
cli.client, err = apiclient.NewClient(&apiclient.Config{
MachineID: cfg.API.Client.Credentials.Login,
Password: strfmt.Password(cfg.API.Client.Credentials.Password),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
Client, err = apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
Password: strfmt.Password(csConfig.API.Client.Credentials.Password),
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
URL: apiURL,
VersionPrefix: "v1",
})
if err != nil {
return fmt.Errorf("new api client: %w", err)
}
if err != nil {
return errors.Wrap(err, "new api client")
}
return nil
},
}
cmd.AddCommand(cli.NewListCmd())
cmd.AddCommand(cli.NewInspectCmd())
cmd.AddCommand(cli.NewFlushCmd())
cmd.AddCommand(cli.NewDeleteCmd())
cmdAlerts.AddCommand(NewAlertsListCmd())
cmdAlerts.AddCommand(NewAlertsInspectCmd())
cmdAlerts.AddCommand(NewAlertsFlushCmd())
cmdAlerts.AddCommand(NewAlertsDeleteCmd())
return cmd
return cmdAlerts
}
func (cli *cliAlerts) NewListCmd() *cobra.Command {
alertListFilter := apiclient.AlertsListOpts{
func NewAlertsListCmd() *cobra.Command {
var alertListFilter = apiclient.AlertsListOpts{
ScopeEquals: new(string),
ValueEquals: new(string),
ScenarioEquals: new(string),
@ -242,24 +252,21 @@ func (cli *cliAlerts) NewListCmd() *cobra.Command {
IncludeCAPI: new(bool),
OriginEquals: new(string),
}
limit := new(int)
var limit = new(int)
contained := new(bool)
var printMachine bool
cmd := &cobra.Command{
var cmdAlertsList = &cobra.Command{
Use: "list [filters]",
Short: "List alerts",
Example: `cscli alerts list
cscli alerts list --ip 1.2.3.4
cscli alerts list --range 1.2.3.0/24
cscli alerts list --origin lists
cscli alerts list -s crowdsecurity/ssh-bf
cscli alerts list --type ban`,
Long: `List alerts with optional filters`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, _ []string) error {
RunE: func(cmd *cobra.Command, args []string) error {
var err error
if err := manageCliDecisionAlerts(alertListFilter.IPEquals, alertListFilter.RangeEquals,
alertListFilter.ScopeEquals, alertListFilter.ValueEquals); err != nil {
printHelp(cmd)
@ -325,56 +332,50 @@ cscli alerts list --type ban`,
alertListFilter.Contains = new(bool)
}
alerts, _, err := cli.client.Alerts.List(context.Background(), alertListFilter)
alerts, _, err := Client.Alerts.List(context.Background(), alertListFilter)
if err != nil {
return fmt.Errorf("unable to list alerts: %w", err)
return fmt.Errorf("unable to list alerts: %v", err)
}
if err = cli.alertsToTable(alerts, printMachine); err != nil {
return fmt.Errorf("unable to list alerts: %w", err)
err = AlertsToTable(alerts, printMachine)
if err != nil {
return fmt.Errorf("unable to list alerts: %v", err)
}
return nil
},
}
cmdAlertsList.Flags().SortFlags = false
cmdAlertsList.Flags().BoolVarP(alertListFilter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
cmdAlertsList.Flags().StringVar(alertListFilter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
cmdAlertsList.Flags().StringVar(alertListFilter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
cmdAlertsList.Flags().StringVarP(alertListFilter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
cmdAlertsList.Flags().StringVarP(alertListFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
cmdAlertsList.Flags().StringVarP(alertListFilter.RangeEquals, "range", "r", "", "restrict to alerts from this range (shorthand for --scope range --value <RANGE/X>)")
cmdAlertsList.Flags().StringVar(alertListFilter.TypeEquals, "type", "", "restrict to alerts with given decision type (ie. ban, captcha)")
cmdAlertsList.Flags().StringVar(alertListFilter.ScopeEquals, "scope", "", "restrict to alerts of this scope (ie. ip,range)")
cmdAlertsList.Flags().StringVarP(alertListFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmdAlertsList.Flags().StringVar(alertListFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmdAlertsList.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
cmdAlertsList.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that sent alerts")
cmdAlertsList.Flags().IntVarP(limit, "limit", "l", 50, "limit size of alerts list table (0 to view all alerts)")
flags := cmd.Flags()
flags.SortFlags = false
flags.BoolVarP(alertListFilter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
flags.StringVar(alertListFilter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
flags.StringVar(alertListFilter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
flags.StringVarP(alertListFilter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
flags.StringVarP(alertListFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
flags.StringVarP(alertListFilter.RangeEquals, "range", "r", "", "restrict to alerts from this range (shorthand for --scope range --value <RANGE/X>)")
flags.StringVar(alertListFilter.TypeEquals, "type", "", "restrict to alerts with given decision type (ie. ban, captcha)")
flags.StringVar(alertListFilter.ScopeEquals, "scope", "", "restrict to alerts of this scope (ie. ip,range)")
flags.StringVarP(alertListFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
flags.StringVar(alertListFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
flags.BoolVar(contained, "contained", false, "query decisions contained by range")
flags.BoolVarP(&printMachine, "machine", "m", false, "print machines that sent alerts")
flags.IntVarP(limit, "limit", "l", 50, "limit size of alerts list table (0 to view all alerts)")
return cmd
return cmdAlertsList
}
func (cli *cliAlerts) NewDeleteCmd() *cobra.Command {
var (
ActiveDecision *bool
AlertDeleteAll bool
delAlertByID string
)
alertDeleteFilter := apiclient.AlertsDeleteOpts{
func NewAlertsDeleteCmd() *cobra.Command {
var ActiveDecision *bool
var AlertDeleteAll bool
var delAlertByID string
contained := new(bool)
var alertDeleteFilter = apiclient.AlertsDeleteOpts{
ScopeEquals: new(string),
ValueEquals: new(string),
ScenarioEquals: new(string),
IPEquals: new(string),
RangeEquals: new(string),
}
contained := new(bool)
cmd := &cobra.Command{
var cmdAlertsDelete = &cobra.Command{
Use: "delete [filters] [--all]",
Short: `Delete alerts
/!\ This command can be use only on the same machine than the local API.`,
@ -384,7 +385,7 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`,
DisableAutoGenTag: true,
Aliases: []string{"remove"},
Args: cobra.ExactArgs(0),
PreRunE: func(cmd *cobra.Command, _ []string) error {
PreRunE: func(cmd *cobra.Command, args []string) error {
if AlertDeleteAll {
return nil
}
@ -392,16 +393,16 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`,
*alertDeleteFilter.ScenarioEquals == "" && *alertDeleteFilter.IPEquals == "" &&
*alertDeleteFilter.RangeEquals == "" && delAlertByID == "" {
_ = cmd.Usage()
return errors.New("at least one filter or --all must be specified")
return fmt.Errorf("at least one filter or --all must be specified")
}
return nil
},
RunE: func(cmd *cobra.Command, _ []string) error {
RunE: func(cmd *cobra.Command, args []string) error {
var err error
if !AlertDeleteAll {
if err = manageCliDecisionAlerts(alertDeleteFilter.IPEquals, alertDeleteFilter.RangeEquals,
if err := manageCliDecisionAlerts(alertDeleteFilter.IPEquals, alertDeleteFilter.RangeEquals,
alertDeleteFilter.ScopeEquals, alertDeleteFilter.ValueEquals); err != nil {
printHelp(cmd)
return err
@ -437,14 +438,14 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`,
var alerts *models.DeleteAlertsResponse
if delAlertByID == "" {
alerts, _, err = cli.client.Alerts.Delete(context.Background(), alertDeleteFilter)
alerts, _, err = Client.Alerts.Delete(context.Background(), alertDeleteFilter)
if err != nil {
return fmt.Errorf("unable to delete alerts: %w", err)
return fmt.Errorf("unable to delete alerts : %v", err)
}
} else {
alerts, _, err = cli.client.Alerts.DeleteOne(context.Background(), delAlertByID)
alerts, _, err = Client.Alerts.DeleteOne(context.Background(), delAlertByID)
if err != nil {
return fmt.Errorf("unable to delete alert: %w", err)
return fmt.Errorf("unable to delete alert: %v", err)
}
}
log.Infof("%s alert(s) deleted", alerts.NbDeleted)
@ -452,99 +453,90 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`,
return nil
},
}
flags := cmd.Flags()
flags.SortFlags = false
flags.StringVar(alertDeleteFilter.ScopeEquals, "scope", "", "the scope (ie. ip,range)")
flags.StringVarP(alertDeleteFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
flags.StringVarP(alertDeleteFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
flags.StringVarP(alertDeleteFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
flags.StringVarP(alertDeleteFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
flags.StringVar(&delAlertByID, "id", "", "alert ID")
flags.BoolVarP(&AlertDeleteAll, "all", "a", false, "delete all alerts")
flags.BoolVar(contained, "contained", false, "query decisions contained by range")
return cmd
cmdAlertsDelete.Flags().SortFlags = false
cmdAlertsDelete.Flags().StringVar(alertDeleteFilter.ScopeEquals, "scope", "", "the scope (ie. ip,range)")
cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmdAlertsDelete.Flags().StringVar(&delAlertByID, "id", "", "alert ID")
cmdAlertsDelete.Flags().BoolVarP(&AlertDeleteAll, "all", "a", false, "delete all alerts")
cmdAlertsDelete.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
return cmdAlertsDelete
}
func (cli *cliAlerts) NewInspectCmd() *cobra.Command {
func NewAlertsInspectCmd() *cobra.Command {
var details bool
cmd := &cobra.Command{
var cmdAlertsInspect = &cobra.Command{
Use: `inspect "alert_id"`,
Short: `Show info about an alert`,
Example: `cscli alerts inspect 123`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
cfg := cli.cfg()
if len(args) == 0 {
printHelp(cmd)
return errors.New("missing alert_id")
return fmt.Errorf("missing alert_id")
}
for _, alertID := range args {
id, err := strconv.Atoi(alertID)
if err != nil {
return fmt.Errorf("bad alert id %s", alertID)
}
alert, _, err := cli.client.Alerts.GetByID(context.Background(), id)
alert, _, err := Client.Alerts.GetByID(context.Background(), id)
if err != nil {
return fmt.Errorf("can't find alert with id %s: %w", alertID, err)
return fmt.Errorf("can't find alert with id %s: %s", alertID, err)
}
switch cfg.Cscli.Output {
switch csConfig.Cscli.Output {
case "human":
if err := cli.displayOneAlert(alert, details); err != nil {
if err := DisplayOneAlert(alert, details); err != nil {
continue
}
case "json":
data, err := json.MarshalIndent(alert, "", " ")
if err != nil {
return fmt.Errorf("unable to marshal alert with id %s: %w", alertID, err)
return fmt.Errorf("unable to marshal alert with id %s: %s", alertID, err)
}
fmt.Printf("%s\n", string(data))
case "raw":
data, err := yaml.Marshal(alert)
if err != nil {
return fmt.Errorf("unable to marshal alert with id %s: %w", alertID, err)
return fmt.Errorf("unable to marshal alert with id %s: %s", alertID, err)
}
fmt.Println(string(data))
fmt.Printf("%s\n", string(data))
}
}
return nil
},
}
cmdAlertsInspect.Flags().SortFlags = false
cmdAlertsInspect.Flags().BoolVarP(&details, "details", "d", false, "show alerts with events")
cmd.Flags().SortFlags = false
cmd.Flags().BoolVarP(&details, "details", "d", false, "show alerts with events")
return cmd
return cmdAlertsInspect
}
func (cli *cliAlerts) NewFlushCmd() *cobra.Command {
var (
maxItems int
maxAge string
)
cmd := &cobra.Command{
func NewAlertsFlushCmd() *cobra.Command {
var maxItems int
var maxAge string
var cmdAlertsFlush = &cobra.Command{
Use: `flush`,
Short: `Flush alerts
/!\ This command can be used only on the same machine than the local API`,
Example: `cscli alerts flush --max-items 1000 --max-age 7d`,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := require.LAPI(cfg); err != nil {
return err
RunE: func(cmd *cobra.Command, args []string) error {
var err error
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
return fmt.Errorf("local API is disabled, please run this command on the local API machine")
}
db, err := database.NewClient(cfg.DbConfig)
dbClient, err = database.NewClient(csConfig.DbConfig)
if err != nil {
return fmt.Errorf("unable to create new database client: %w", err)
return fmt.Errorf("unable to create new database client: %s", err)
}
log.Info("Flushing alerts. !! This may take a long time !!")
err = db.FlushAlerts(maxAge, maxItems)
err = dbClient.FlushAlerts(maxAge, maxItems)
if err != nil {
return fmt.Errorf("unable to flush alerts: %w", err)
return fmt.Errorf("unable to flush alerts: %s", err)
}
log.Info("Alerts flushed")
@ -552,9 +544,9 @@ func (cli *cliAlerts) NewFlushCmd() *cobra.Command {
},
}
cmd.Flags().SortFlags = false
cmd.Flags().IntVar(&maxItems, "max-items", 5000, "Maximum number of alert items to keep in the database")
cmd.Flags().StringVar(&maxAge, "max-age", "7d", "Maximum age of alert items to keep in the database")
cmdAlertsFlush.Flags().SortFlags = false
cmdAlertsFlush.Flags().IntVar(&maxItems, "max-items", 5000, "Maximum number of alert items to keep in the database")
cmdAlertsFlush.Flags().StringVar(&maxAge, "max-age", "7d", "Maximum age of alert items to keep in the database")
return cmd
return cmdAlertsFlush
}

View file

@ -3,318 +3,219 @@ package main
import (
"encoding/csv"
"encoding/json"
"errors"
"fmt"
"os"
"slices"
"io"
"strings"
"time"
"github.com/AlecAivazis/survey/v2"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
func askYesNo(message string, defaultAnswer bool) (bool, error) {
var answer bool
prompt := &survey.Confirm{
Message: message,
Default: defaultAnswer,
func getBouncers(out io.Writer, dbClient *database.Client) error {
bouncers, err := dbClient.ListBouncers()
if err != nil {
return fmt.Errorf("unable to list bouncers: %s", err)
}
if err := survey.AskOne(prompt, &answer); err != nil {
return defaultAnswer, err
}
return answer, nil
}
type cliBouncers struct {
db *database.Client
cfg configGetter
}
func NewCLIBouncers(cfg configGetter) *cliBouncers {
return &cliBouncers{
cfg: cfg,
}
}
func (cli *cliBouncers) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "bouncers [action]",
Short: "Manage bouncers [requires local API]",
Long: `To list/add/delete/prune bouncers.
Note: This command requires database direct access, so is intended to be run on Local API/master.
`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"bouncer"},
DisableAutoGenTag: true,
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
var err error
cfg := cli.cfg()
if err = require.LAPI(cfg); err != nil {
return err
if csConfig.Cscli.Output == "human" {
getBouncersTable(out, bouncers)
} else if csConfig.Cscli.Output == "json" {
enc := json.NewEncoder(out)
enc.SetIndent("", " ")
if err := enc.Encode(bouncers); err != nil {
return fmt.Errorf("failed to unmarshal: %w", err)
}
return nil
} else if csConfig.Cscli.Output == "raw" {
csvwriter := csv.NewWriter(out)
err := csvwriter.Write([]string{"name", "ip", "revoked", "last_pull", "type", "version", "auth_type"})
if err != nil {
return fmt.Errorf("failed to write raw header: %w", err)
}
for _, b := range bouncers {
var revoked string
if !b.Revoked {
revoked = "validated"
} else {
revoked = "pending"
}
cli.db, err = database.NewClient(cfg.DbConfig)
err := csvwriter.Write([]string{b.Name, b.IPAddress, revoked, b.LastPull.Format(time.RFC3339), b.Type, b.Version, b.AuthType})
if err != nil {
return fmt.Errorf("can't connect to the database: %w", err)
return fmt.Errorf("failed to write raw: %w", err)
}
}
csvwriter.Flush()
}
return nil
}
func NewBouncersListCmd() *cobra.Command {
cmdBouncersList := &cobra.Command{
Use: "list",
Short: "List bouncers",
Long: `List bouncers`,
Example: `cscli bouncers list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, arg []string) error {
err := getBouncers(color.Output, dbClient)
if err != nil {
return fmt.Errorf("unable to list bouncers: %s", err)
}
return nil
},
}
cmd.AddCommand(cli.newListCmd())
cmd.AddCommand(cli.newAddCmd())
cmd.AddCommand(cli.newDeleteCmd())
cmd.AddCommand(cli.newPruneCmd())
return cmd
return cmdBouncersList
}
func (cli *cliBouncers) list() error {
out := color.Output
func runBouncersAdd(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
bouncers, err := cli.db.ListBouncers()
keyLength, err := flags.GetInt("length")
if err != nil {
return fmt.Errorf("unable to list bouncers: %w", err)
return err
}
switch cli.cfg().Cscli.Output {
case "human":
getBouncersTable(out, bouncers)
case "json":
enc := json.NewEncoder(out)
enc.SetIndent("", " ")
if err := enc.Encode(bouncers); err != nil {
return fmt.Errorf("failed to marshal: %w", err)
}
return nil
case "raw":
csvwriter := csv.NewWriter(out)
if err := csvwriter.Write([]string{"name", "ip", "revoked", "last_pull", "type", "version", "auth_type"}); err != nil {
return fmt.Errorf("failed to write raw header: %w", err)
}
for _, b := range bouncers {
valid := "validated"
if b.Revoked {
valid = "pending"
}
if err := csvwriter.Write([]string{b.Name, b.IPAddress, valid, b.LastPull.Format(time.RFC3339), b.Type, b.Version, b.AuthType}); err != nil {
return fmt.Errorf("failed to write raw: %w", err)
}
}
csvwriter.Flush()
key, err := flags.GetString("key")
if err != nil {
return err
}
return nil
}
keyName := args[0]
var apiKey string
func (cli *cliBouncers) newListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "list",
Short: "list all bouncers within the database",
Example: `cscli bouncers list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.list()
},
if keyName == "" {
return fmt.Errorf("please provide a name for the api key")
}
return cmd
}
func (cli *cliBouncers) add(bouncerName string, key string) error {
var err error
keyLength := 32
apiKey = key
if key == "" {
key, err = middlewares.GenerateAPIKey(keyLength)
if err != nil {
return fmt.Errorf("unable to generate api key: %w", err)
}
apiKey, err = middlewares.GenerateAPIKey(keyLength)
}
_, err = cli.db.CreateBouncer(bouncerName, "", middlewares.HashSHA512(key), types.ApiKeyAuthType)
if err != nil {
return fmt.Errorf("unable to create bouncer: %w", err)
return fmt.Errorf("unable to generate api key: %s", err)
}
_, err = dbClient.CreateBouncer(keyName, "", middlewares.HashSHA512(apiKey), types.ApiKeyAuthType)
if err != nil {
return fmt.Errorf("unable to create bouncer: %s", err)
}
switch cli.cfg().Cscli.Output {
case "human":
fmt.Printf("API key for '%s':\n\n", bouncerName)
fmt.Printf(" %s\n\n", key)
if csConfig.Cscli.Output == "human" {
fmt.Printf("Api key for '%s':\n\n", keyName)
fmt.Printf(" %s\n\n", apiKey)
fmt.Print("Please keep this key since you will not be able to retrieve it!\n")
case "raw":
fmt.Print(key)
case "json":
j, err := json.Marshal(key)
} else if csConfig.Cscli.Output == "raw" {
fmt.Printf("%s", apiKey)
} else if csConfig.Cscli.Output == "json" {
j, err := json.Marshal(apiKey)
if err != nil {
return errors.New("unable to marshal api key")
return fmt.Errorf("unable to marshal api key")
}
fmt.Print(string(j))
fmt.Printf("%s", string(j))
}
return nil
}
func (cli *cliBouncers) newAddCmd() *cobra.Command {
var key string
cmd := &cobra.Command{
Use: "add MyBouncerName",
Short: "add a single bouncer to the database",
func NewBouncersAddCmd() *cobra.Command {
cmdBouncersAdd := &cobra.Command{
Use: "add MyBouncerName [--length 16]",
Short: "add bouncer",
Long: `add bouncer`,
Example: `cscli bouncers add MyBouncerName
cscli bouncers add MyBouncerName --key <random-key>`,
cscli bouncers add MyBouncerName -l 24
cscli bouncers add MyBouncerName -k <random-key>`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
return cli.add(args[0], key)
},
RunE: runBouncersAdd,
}
flags := cmd.Flags()
flags.StringP("length", "l", "", "length of the api key")
_ = flags.MarkDeprecated("length", "use --key instead")
flags.StringVarP(&key, "key", "k", "", "api key for the bouncer")
flags := cmdBouncersAdd.Flags()
return cmd
flags.IntP("length", "l", 16, "length of the api key")
flags.StringP("key", "k", "", "api key for the bouncer")
return cmdBouncersAdd
}
func (cli *cliBouncers) deleteValid(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
bouncers, err := cli.db.ListBouncers()
if err != nil {
cobra.CompError("unable to list bouncers " + err.Error())
}
ret := []string{}
for _, bouncer := range bouncers {
if strings.Contains(bouncer.Name, toComplete) && !slices.Contains(args, bouncer.Name) {
ret = append(ret, bouncer.Name)
}
}
return ret, cobra.ShellCompDirectiveNoFileComp
}
func (cli *cliBouncers) delete(bouncers []string) error {
for _, bouncerID := range bouncers {
err := cli.db.DeleteBouncer(bouncerID)
func runBouncersDelete(cmd *cobra.Command, args []string) error {
for _, bouncerID := range args {
err := dbClient.DeleteBouncer(bouncerID)
if err != nil {
return fmt.Errorf("unable to delete bouncer '%s': %w", bouncerID, err)
return fmt.Errorf("unable to delete bouncer '%s': %s", bouncerID, err)
}
log.Infof("bouncer '%s' deleted successfully", bouncerID)
}
return nil
}
func (cli *cliBouncers) newDeleteCmd() *cobra.Command {
cmd := &cobra.Command{
func NewBouncersDeleteCmd() *cobra.Command {
cmdBouncersDelete := &cobra.Command{
Use: "delete MyBouncerName",
Short: "delete bouncer(s) from the database",
Short: "delete bouncer",
Args: cobra.MinimumNArgs(1),
Aliases: []string{"remove"},
DisableAutoGenTag: true,
ValidArgsFunction: cli.deleteValid,
RunE: func(_ *cobra.Command, args []string) error {
return cli.delete(args)
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
var err error
dbClient, err = getDBClient()
if err != nil {
cobra.CompError("unable to create new database client: " + err.Error())
return nil, cobra.ShellCompDirectiveNoFileComp
}
bouncers, err := dbClient.ListBouncers()
if err != nil {
cobra.CompError("unable to list bouncers " + err.Error())
}
ret := make([]string, 0)
for _, bouncer := range bouncers {
if strings.Contains(bouncer.Name, toComplete) && !inSlice(bouncer.Name, args) {
ret = append(ret, bouncer.Name)
}
}
return ret, cobra.ShellCompDirectiveNoFileComp
},
RunE: runBouncersDelete,
}
return cmd
return cmdBouncersDelete
}
func (cli *cliBouncers) prune(duration time.Duration, force bool) error {
if duration < 2*time.Minute {
if yes, err := askYesNo(
"The duration you provided is less than 2 minutes. " +
"This may remove active bouncers. Continue?", false); err != nil {
return err
} else if !yes {
fmt.Println("User aborted prune. No changes were made.")
return nil
}
}
bouncers, err := cli.db.QueryBouncersLastPulltimeLT(time.Now().UTC().Add(-duration))
if err != nil {
return fmt.Errorf("unable to query bouncers: %w", err)
}
if len(bouncers) == 0 {
fmt.Println("No bouncers to prune.")
return nil
}
getBouncersTable(color.Output, bouncers)
if !force {
if yes, err := askYesNo(
"You are about to PERMANENTLY remove the above bouncers from the database. " +
"These will NOT be recoverable. Continue?", false); err != nil {
return err
} else if !yes {
fmt.Println("User aborted prune. No changes were made.")
return nil
}
}
deleted, err := cli.db.BulkDeleteBouncers(bouncers)
if err != nil {
return fmt.Errorf("unable to prune bouncers: %w", err)
}
fmt.Fprintf(os.Stderr, "Successfully deleted %d bouncers\n", deleted)
return nil
}
func (cli *cliBouncers) newPruneCmd() *cobra.Command {
var (
duration time.Duration
force bool
)
const defaultDuration = 60 * time.Minute
cmd := &cobra.Command{
Use: "prune",
Short: "prune multiple bouncers from the database",
Args: cobra.NoArgs,
func NewBouncersCmd() *cobra.Command {
var cmdBouncers = &cobra.Command{
Use: "bouncers [action]",
Short: "Manage bouncers [requires local API]",
Long: `To list/add/delete bouncers.
Note: This command requires database direct access, so is intended to be run on Local API/master.
`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"bouncer"},
DisableAutoGenTag: true,
Example: `cscli bouncers prune -d 45m
cscli bouncers prune -d 45m --force`,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.prune(duration, force)
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
var err error
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
return fmt.Errorf("local API is disabled, please run this command on the local API machine")
}
dbClient, err = database.NewClient(csConfig.DbConfig)
if err != nil {
return fmt.Errorf("unable to create new database client: %s", err)
}
return nil
},
}
flags := cmd.Flags()
flags.DurationVarP(&duration, "duration", "d", defaultDuration, "duration of time since last pull")
flags.BoolVar(&force, "force", false, "force prune without asking for confirmation")
cmdBouncers.AddCommand(NewBouncersListCmd())
cmdBouncers.AddCommand(NewBouncersAddCmd())
cmdBouncers.AddCommand(NewBouncersDeleteCmd())
return cmd
return cmdBouncers
}

View file

@ -5,9 +5,9 @@ import (
"time"
"github.com/aquasecurity/table"
"github.com/enescakir/emoji"
"github.com/crowdsecurity/crowdsec/pkg/database/ent"
"github.com/crowdsecurity/crowdsec/pkg/emoji"
)
func getBouncersTable(out io.Writer, bouncers []*ent.Bouncer) {
@ -17,9 +17,11 @@ func getBouncersTable(out io.Writer, bouncers []*ent.Bouncer) {
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
for _, b := range bouncers {
revoked := emoji.CheckMark
if b.Revoked {
revoked = emoji.Prohibited
var revoked string
if !b.Revoked {
revoked = emoji.CheckMark.String()
} else {
revoked = emoji.Prohibited.String()
}
t.AddRow(b.Name, b.IPAddress, revoked, b.LastPull.Format(time.RFC3339), b.Type, b.Version, b.AuthType)

View file

@ -2,221 +2,186 @@ package main
import (
"context"
"errors"
"fmt"
"net/url"
"os"
"github.com/go-openapi/strfmt"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
)
const (
CAPIBaseURL = "https://api.crowdsec.net/"
CAPIURLPrefix = "v3"
)
const CAPIBaseURL string = "https://api.crowdsec.net/"
const CAPIURLPrefix = "v3"
type cliCapi struct {
cfg configGetter
}
func NewCLICapi(cfg configGetter) *cliCapi {
return &cliCapi{
cfg: cfg,
}
}
func (cli *cliCapi) NewCommand() *cobra.Command {
cmd := &cobra.Command{
func NewCapiCmd() *cobra.Command {
var cmdCapi = &cobra.Command{
Use: "capi [action]",
Short: "Manage interaction with Central API (CAPI)",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := require.LAPI(cfg); err != nil {
return err
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
return errors.Wrap(err, "Local API is disabled, please run this command on the local API machine")
}
if err := require.CAPI(cfg); err != nil {
return err
if csConfig.API.Server.OnlineClient == nil {
log.Fatalf("no configuration for Central API in '%s'", *csConfig.FilePath)
}
return nil
},
}
cmd.AddCommand(cli.newRegisterCmd())
cmd.AddCommand(cli.newStatusCmd())
cmdCapi.AddCommand(NewCapiRegisterCmd())
cmdCapi.AddCommand(NewCapiStatusCmd())
return cmd
return cmdCapi
}
func (cli *cliCapi) register(capiUserPrefix string, outputFile string) error {
cfg := cli.cfg()
func NewCapiRegisterCmd() *cobra.Command {
var capiUserPrefix string
var outputFile string
capiUser, err := generateID(capiUserPrefix)
if err != nil {
return fmt.Errorf("unable to generate machine id: %w", err)
}
password := strfmt.Password(generatePassword(passwordLength))
apiurl, err := url.Parse(types.CAPIBaseURL)
if err != nil {
return fmt.Errorf("unable to parse api url %s: %w", types.CAPIBaseURL, err)
}
_, err = apiclient.RegisterClient(&apiclient.Config{
MachineID: capiUser,
Password: password,
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiurl,
VersionPrefix: CAPIURLPrefix,
}, nil)
if err != nil {
return fmt.Errorf("api client register ('%s'): %w", types.CAPIBaseURL, err)
}
log.Infof("Successfully registered to Central API (CAPI)")
var dumpFile string
switch {
case outputFile != "":
dumpFile = outputFile
case cfg.API.Server.OnlineClient.CredentialsFilePath != "":
dumpFile = cfg.API.Server.OnlineClient.CredentialsFilePath
default:
dumpFile = ""
}
apiCfg := csconfig.ApiCredentialsCfg{
Login: capiUser,
Password: password.String(),
URL: types.CAPIBaseURL,
}
apiConfigDump, err := yaml.Marshal(apiCfg)
if err != nil {
return fmt.Errorf("unable to marshal api credentials: %w", err)
}
if dumpFile != "" {
err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
if err != nil {
return fmt.Errorf("write api credentials in '%s' failed: %w", dumpFile, err)
}
log.Infof("Central API credentials written to '%s'", dumpFile)
} else {
fmt.Println(string(apiConfigDump))
}
log.Warning(ReloadMessage())
return nil
}
func (cli *cliCapi) newRegisterCmd() *cobra.Command {
var (
capiUserPrefix string
outputFile string
)
cmd := &cobra.Command{
var cmdCapiRegister = &cobra.Command{
Use: "register",
Short: "Register to Central API (CAPI)",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.register(capiUserPrefix, outputFile)
Run: func(cmd *cobra.Command, args []string) {
var err error
capiUser, err := generateID(capiUserPrefix)
if err != nil {
log.Fatalf("unable to generate machine id: %s", err)
}
password := strfmt.Password(generatePassword(passwordLength))
apiurl, err := url.Parse(types.CAPIBaseURL)
if err != nil {
log.Fatalf("unable to parse api url %s : %s", types.CAPIBaseURL, err)
}
_, err = apiclient.RegisterClient(&apiclient.Config{
MachineID: capiUser,
Password: password,
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
URL: apiurl,
VersionPrefix: CAPIURLPrefix,
}, nil)
if err != nil {
log.Fatalf("api client register ('%s'): %s", types.CAPIBaseURL, err)
}
log.Printf("Successfully registered to Central API (CAPI)")
var dumpFile string
if outputFile != "" {
dumpFile = outputFile
} else if csConfig.API.Server.OnlineClient.CredentialsFilePath != "" {
dumpFile = csConfig.API.Server.OnlineClient.CredentialsFilePath
} else {
dumpFile = ""
}
apiCfg := csconfig.ApiCredentialsCfg{
Login: capiUser,
Password: password.String(),
URL: types.CAPIBaseURL,
}
if fflag.PapiClient.IsEnabled() {
apiCfg.PapiURL = types.PAPIBaseURL
}
apiConfigDump, err := yaml.Marshal(apiCfg)
if err != nil {
log.Fatalf("unable to marshal api credentials: %s", err)
}
if dumpFile != "" {
err = os.WriteFile(dumpFile, apiConfigDump, 0600)
if err != nil {
log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
}
log.Printf("Central API credentials dumped to '%s'", dumpFile)
} else {
fmt.Printf("%s\n", string(apiConfigDump))
}
log.Warning(ReloadMessage())
},
}
cmd.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
cmd.Flags().StringVar(&capiUserPrefix, "schmilblick", "", "set a schmilblick (use in tests only)")
if err := cmd.Flags().MarkHidden("schmilblick"); err != nil {
cmdCapiRegister.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
cmdCapiRegister.Flags().StringVar(&capiUserPrefix, "schmilblick", "", "set a schmilblick (use in tests only)")
if err := cmdCapiRegister.Flags().MarkHidden("schmilblick"); err != nil {
log.Fatalf("failed to hide flag: %s", err)
}
return cmd
return cmdCapiRegister
}
func (cli *cliCapi) status() error {
cfg := cli.cfg()
if err := require.CAPIRegistered(cfg); err != nil {
return err
}
password := strfmt.Password(cfg.API.Server.OnlineClient.Credentials.Password)
apiurl, err := url.Parse(cfg.API.Server.OnlineClient.Credentials.URL)
if err != nil {
return fmt.Errorf("parsing api url ('%s'): %w", cfg.API.Server.OnlineClient.Credentials.URL, err)
}
hub, err := require.Hub(cfg, nil, nil)
if err != nil {
return err
}
scenarios, err := hub.GetInstalledNamesByType(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("failed to get scenarios: %w", err)
}
if len(scenarios) == 0 {
return errors.New("no scenarios installed, abort")
}
Client, err = apiclient.NewDefaultClient(apiurl, CAPIURLPrefix, fmt.Sprintf("crowdsec/%s", version.String()), nil)
if err != nil {
return fmt.Errorf("init default client: %w", err)
}
t := models.WatcherAuthRequest{
MachineID: &cfg.API.Server.OnlineClient.Credentials.Login,
Password: &password,
Scenarios: scenarios,
}
log.Infof("Loaded credentials from %s", cfg.API.Server.OnlineClient.CredentialsFilePath)
log.Infof("Trying to authenticate with username %s on %s", cfg.API.Server.OnlineClient.Credentials.Login, apiurl)
_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
if err != nil {
return fmt.Errorf("failed to authenticate to Central API (CAPI): %w", err)
}
log.Info("You can successfully interact with Central API (CAPI)")
return nil
}
func (cli *cliCapi) newStatusCmd() *cobra.Command {
cmd := &cobra.Command{
func NewCapiStatusCmd() *cobra.Command {
var cmdCapiStatus = &cobra.Command{
Use: "status",
Short: "Check status with the Central API (CAPI)",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.status()
Run: func(cmd *cobra.Command, args []string) {
var err error
if csConfig.API.Server == nil {
log.Fatalln("There is no configuration on 'api.server:'")
}
if csConfig.API.Server.OnlineClient == nil {
log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
}
if csConfig.API.Server.OnlineClient.Credentials == nil {
log.Fatalf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
}
password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
apiurl, err := url.Parse(csConfig.API.Server.OnlineClient.Credentials.URL)
if err != nil {
log.Fatalf("parsing api url ('%s'): %s", csConfig.API.Server.OnlineClient.Credentials.URL, err)
}
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to load hub index : %s", err)
}
scenarios, err := cwhub.GetInstalledScenariosAsString()
if err != nil {
log.Fatalf("failed to get scenarios : %s", err)
}
if len(scenarios) == 0 {
log.Fatalf("no scenarios installed, abort")
}
Client, err = apiclient.NewDefaultClient(apiurl, CAPIURLPrefix, fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), nil)
if err != nil {
log.Fatalf("init default client: %s", err)
}
t := models.WatcherAuthRequest{
MachineID: &csConfig.API.Server.OnlineClient.Credentials.Login,
Password: &password,
Scenarios: scenarios,
}
log.Infof("Loaded credentials from %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
log.Infof("Trying to authenticate with username %s on %s", csConfig.API.Server.OnlineClient.Credentials.Login, apiurl)
_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
if err != nil {
log.Fatalf("Failed to authenticate to Central API (CAPI) : %s", err)
}
log.Infof("You can successfully interact with Central API (CAPI)")
},
}
return cmd
return cmdCapiStatus
}

View file

@ -0,0 +1,183 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCollectionsCmd() *cobra.Command {
var cmdCollections = &cobra.Command{
Use: "collections [action]",
Short: "Manage collections from hub",
Long: `Install/Remove/Upgrade/Inspect collections from the CrowdSec Hub.`,
/*TBD fix help*/
Args: cobra.MinimumNArgs(1),
Aliases: []string{"collection"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if csConfig.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
var ignoreError bool
var cmdCollectionsInstall = &cobra.Command{
Use: "install collection",
Short: "Install given collection(s)",
Long: `Fetch and install given collection(s) from hub`,
Example: `cscli collections install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.COLLECTIONS, args, toComplete)
},
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
for _, name := range args {
t := cwhub.GetItem(cwhub.COLLECTIONS, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.COLLECTIONS, name)
Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
if !ignoreError {
log.Fatalf("Error while installing '%s': %s", name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
}
}
},
}
cmdCollectionsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdCollectionsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdCollectionsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple collections")
cmdCollections.AddCommand(cmdCollectionsInstall)
var cmdCollectionsRemove = &cobra.Command{
Use: "remove collection",
Short: "Remove given collection(s)",
Long: `Remove given collection(s) from hub`,
Example: `cscli collections remove crowdsec/xxx crowdsec/xyz`,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
if all {
cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, forceAction)
return
}
if len(args) == 0 {
log.Fatal("Specify at least one collection to remove or '--all' flag.")
}
for _, name := range args {
if !forceAction {
item := cwhub.GetItem(cwhub.COLLECTIONS, name)
if item == nil {
log.Fatalf("unable to retrieve: %s\n", name)
}
if len(item.BelongsToCollections) > 0 {
log.Warningf("%s belongs to other collections :\n%s\n", name, item.BelongsToCollections)
log.Printf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection\n", name)
continue
}
}
cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, forceAction)
}
},
}
cmdCollectionsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdCollectionsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdCollectionsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the collections")
cmdCollections.AddCommand(cmdCollectionsRemove)
var cmdCollectionsUpgrade = &cobra.Command{
Use: "upgrade collection",
Short: "Upgrade given collection(s)",
Long: `Fetch and upgrade given collection(s) from hub`,
Example: `cscli collections upgrade crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
} else {
if len(args) == 0 {
log.Fatalf("no target collection to upgrade")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, forceAction)
}
}
},
}
cmdCollectionsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the collections")
cmdCollectionsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
cmdCollections.AddCommand(cmdCollectionsUpgrade)
var cmdCollectionsInspect = &cobra.Command{
Use: "inspect collection",
Short: "Inspect given collection",
Long: `Inspect given collection`,
Example: `cscli collections inspect crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
for _, name := range args {
InspectItem(name, cwhub.COLLECTIONS)
}
},
}
cmdCollectionsInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
cmdCollections.AddCommand(cmdCollectionsInspect)
var cmdCollectionsList = &cobra.Command{
Use: "list collection [-a]",
Short: "List all collections",
Long: `List all collections`,
Example: `cscli collections list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all)
},
}
cmdCollectionsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
cmdCollections.AddCommand(cmdCollectionsList)
return cmdCollections
}

View file

@ -7,7 +7,8 @@ import (
)
func NewCompletionCmd() *cobra.Command {
completionCmd := &cobra.Command{
var completionCmd = &cobra.Command{
Use: "completion [bash|zsh|powershell|fish]",
Short: "Generate completion script",
Long: `To load completions:
@ -67,7 +68,7 @@ func NewCompletionCmd() *cobra.Command {
DisableFlagsInUseLine: true,
DisableAutoGenTag: true,
ValidArgs: []string{"bash", "zsh", "powershell", "fish"},
Args: cobra.MatchAll(cobra.ExactArgs(1), cobra.OnlyValidArgs),
Args: cobra.ExactValidArgs(1),
Run: func(cmd *cobra.Command, args []string) {
switch args[0] {
case "bash":
@ -81,6 +82,5 @@ func NewCompletionCmd() *cobra.Command {
}
},
}
return completionCmd
}

View file

@ -4,29 +4,18 @@ import (
"github.com/spf13/cobra"
)
type cliConfig struct {
cfg configGetter
}
func NewCLIConfig(cfg configGetter) *cliConfig {
return &cliConfig{
cfg: cfg,
}
}
func (cli *cliConfig) NewCommand() *cobra.Command {
cmd := &cobra.Command{
func NewConfigCmd() *cobra.Command {
cmdConfig := &cobra.Command{
Use: "config [command]",
Short: "Allows to view current config",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
}
cmd.AddCommand(cli.newShowCmd())
cmd.AddCommand(cli.newShowYAMLCmd())
cmd.AddCommand(cli.newBackupCmd())
cmd.AddCommand(cli.newRestoreCmd())
cmd.AddCommand(cli.newFeatureFlagsCmd())
cmdConfig.AddCommand(NewConfigShowCmd())
cmdConfig.AddCommand(NewConfigBackupCmd())
cmdConfig.AddCommand(NewConfigRestoreCmd())
cmdConfig.AddCommand(NewConfigFeatureFlagsCmd())
return cmd
return cmdConfig
}

View file

@ -1,99 +1,19 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"os"
"path/filepath"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func (cli *cliConfig) backupHub(dirPath string) error {
hub, err := require.Hub(cli.cfg(), nil, nil)
if err != nil {
return err
}
for _, itemType := range cwhub.ItemTypes {
clog := log.WithFields(log.Fields{
"type": itemType,
})
itemMap := hub.GetItemMap(itemType)
if itemMap == nil {
clog.Infof("No %s to backup.", itemType)
continue
}
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itemType)
if err = os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
return fmt.Errorf("error while creating %s: %w", itemDirectory, err)
}
upstreamParsers := []string{}
for k, v := range itemMap {
clog = clog.WithFields(log.Fields{
"file": v.Name,
})
if !v.State.Installed { // only backup installed ones
clog.Debugf("[%s]: not installed", k)
continue
}
// for the local/tainted ones, we back up the full file
if v.State.Tainted || v.State.IsLocal() || !v.State.UpToDate {
// we need to backup stages for parsers
if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS {
fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
if err = os.MkdirAll(fstagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage dir %s: %w", fstagedir, err)
}
}
clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.State.IsLocal(), v.State.UpToDate)
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
if err = CopyFile(v.State.LocalPath, tfile); err != nil {
return fmt.Errorf("failed copy %s %s to %s: %w", itemType, v.State.LocalPath, tfile, err)
}
clog.Infof("local/tainted saved %s to %s", v.State.LocalPath, tfile)
continue
}
clog.Debugf("[%s]: from hub, just backup name (up-to-date:%t)", k, v.State.UpToDate)
clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.State.UpToDate)
upstreamParsers = append(upstreamParsers, v.Name)
}
// write the upstream items
upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
if err != nil {
return fmt.Errorf("failed marshaling upstream parsers: %w", err)
}
err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0o644)
if err != nil {
return fmt.Errorf("unable to write to %s %s: %w", itemType, upstreamParsersFname, err)
}
clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
}
return nil
}
/*
Backup crowdsec configurations to directory <dirPath>:
/* Backup crowdsec configurations to directory <dirPath> :
- Main config (config.yaml)
- Profiles config (profiles.yaml)
@ -101,33 +21,30 @@ func (cli *cliConfig) backupHub(dirPath string) error {
- Backup of API credentials (local API and online API)
- List of scenarios, parsers, postoverflows and collections that are up-to-date
- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections
- Acquisition files (acquis.yaml, acquis.d/*.yaml)
*/
func (cli *cliConfig) backup(dirPath string) error {
func backupConfigToDirectory(dirPath string) error {
var err error
cfg := cli.cfg()
if dirPath == "" {
return errors.New("directory path can't be empty")
return fmt.Errorf("directory path can't be empty")
}
log.Infof("Starting configuration backup")
/*if parent directory doesn't exist, bail out. create final dir with Mkdir*/
parentDir := filepath.Dir(dirPath)
if _, err = os.Stat(parentDir); err != nil {
return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err)
if _, err := os.Stat(parentDir); err != nil {
return errors.Wrapf(err, "while checking parent directory %s existence", parentDir)
}
if err = os.Mkdir(dirPath, 0o700); err != nil {
return fmt.Errorf("while creating %s: %w", dirPath, err)
return errors.Wrapf(err, "while creating %s", dirPath)
}
if cfg.ConfigPaths.SimulationFilePath != "" {
if csConfig.ConfigPaths.SimulationFilePath != "" {
backupSimulation := filepath.Join(dirPath, "simulation.yaml")
if err = CopyFile(cfg.ConfigPaths.SimulationFilePath, backupSimulation); err != nil {
return fmt.Errorf("failed copy %s to %s: %w", cfg.ConfigPaths.SimulationFilePath, backupSimulation, err)
if err = types.CopyFile(csConfig.ConfigPaths.SimulationFilePath, backupSimulation); err != nil {
return errors.Wrapf(err, "failed copy %s to %s", csConfig.ConfigPaths.SimulationFilePath, backupSimulation)
}
log.Infof("Saved simulation to %s", backupSimulation)
@ -137,32 +54,32 @@ func (cli *cliConfig) backup(dirPath string) error {
- backup AcquisitionFilePath
- backup the other files of acquisition directory
*/
if cfg.Crowdsec != nil && cfg.Crowdsec.AcquisitionFilePath != "" {
if csConfig.Crowdsec != nil && csConfig.Crowdsec.AcquisitionFilePath != "" {
backupAcquisition := filepath.Join(dirPath, "acquis.yaml")
if err = CopyFile(cfg.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil {
return fmt.Errorf("failed copy %s to %s: %w", cfg.Crowdsec.AcquisitionFilePath, backupAcquisition, err)
if err = types.CopyFile(csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", csConfig.Crowdsec.AcquisitionFilePath, backupAcquisition, err)
}
}
acquisBackupDir := filepath.Join(dirPath, "acquis")
if err = os.Mkdir(acquisBackupDir, 0o700); err != nil {
return fmt.Errorf("error while creating %s: %w", acquisBackupDir, err)
return fmt.Errorf("error while creating %s : %s", acquisBackupDir, err)
}
if cfg.Crowdsec != nil && len(cfg.Crowdsec.AcquisitionFiles) > 0 {
for _, acquisFile := range cfg.Crowdsec.AcquisitionFiles {
if csConfig.Crowdsec != nil && len(csConfig.Crowdsec.AcquisitionFiles) > 0 {
for _, acquisFile := range csConfig.Crowdsec.AcquisitionFiles {
/*if it was the default one, it was already backup'ed*/
if cfg.Crowdsec.AcquisitionFilePath == acquisFile {
if csConfig.Crowdsec.AcquisitionFilePath == acquisFile {
continue
}
targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile)))
if err != nil {
return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err)
return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir)
}
if err = CopyFile(acquisFile, targetFname); err != nil {
return fmt.Errorf("failed copy %s to %s: %w", acquisFile, targetFname, err)
if err = types.CopyFile(acquisFile, targetFname); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err)
}
log.Infof("Saved acquis %s to %s", acquisFile, targetFname)
@ -171,49 +88,68 @@ func (cli *cliConfig) backup(dirPath string) error {
if ConfigFilePath != "" {
backupMain := fmt.Sprintf("%s/config.yaml", dirPath)
if err = CopyFile(ConfigFilePath, backupMain); err != nil {
return fmt.Errorf("failed copy %s to %s: %w", ConfigFilePath, backupMain, err)
if err = types.CopyFile(ConfigFilePath, backupMain); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", ConfigFilePath, backupMain, err)
}
log.Infof("Saved default yaml to %s", backupMain)
}
if cfg.API != nil && cfg.API.Server != nil && cfg.API.Server.OnlineClient != nil && cfg.API.Server.OnlineClient.CredentialsFilePath != "" {
if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" {
backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath)
if err = CopyFile(cfg.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil {
return fmt.Errorf("failed copy %s to %s: %w", cfg.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err)
if err = types.CopyFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.OnlineClient.CredentialsFilePath, backupCAPICreds, err)
}
log.Infof("Saved online API credentials to %s", backupCAPICreds)
}
if cfg.API != nil && cfg.API.Client != nil && cfg.API.Client.CredentialsFilePath != "" {
if csConfig.API != nil && csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" {
backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath)
if err = CopyFile(cfg.API.Client.CredentialsFilePath, backupLAPICreds); err != nil {
return fmt.Errorf("failed copy %s to %s: %w", cfg.API.Client.CredentialsFilePath, backupLAPICreds, err)
if err = types.CopyFile(csConfig.API.Client.CredentialsFilePath, backupLAPICreds); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Client.CredentialsFilePath, backupLAPICreds, err)
}
log.Infof("Saved local API credentials to %s", backupLAPICreds)
}
if cfg.API != nil && cfg.API.Server != nil && cfg.API.Server.ProfilesPath != "" {
if csConfig.API != nil && csConfig.API.Server != nil && csConfig.API.Server.ProfilesPath != "" {
backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath)
if err = CopyFile(cfg.API.Server.ProfilesPath, backupProfiles); err != nil {
return fmt.Errorf("failed copy %s to %s: %w", cfg.API.Server.ProfilesPath, backupProfiles, err)
if err = types.CopyFile(csConfig.API.Server.ProfilesPath, backupProfiles); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", csConfig.API.Server.ProfilesPath, backupProfiles, err)
}
log.Infof("Saved profiles to %s", backupProfiles)
}
if err = cli.backupHub(dirPath); err != nil {
return fmt.Errorf("failed to backup hub config: %w", err)
if err = BackupHub(dirPath); err != nil {
return fmt.Errorf("failed to backup hub config : %s", err)
}
return nil
}
func (cli *cliConfig) newBackupCmd() *cobra.Command {
cmd := &cobra.Command{
func runConfigBackup(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
return err
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
return fmt.Errorf("failed to get Hub index: %w", err)
}
if err := backupConfigToDirectory(args[0]); err != nil {
return fmt.Errorf("failed to backup config: %w", err)
}
return nil
}
func NewConfigBackupCmd() *cobra.Command {
cmdConfigBackup := &cobra.Command{
Use: `backup "directory"`,
Short: "Backup current config",
Long: `Backup the current crowdsec configuration including :
@ -227,14 +163,8 @@ func (cli *cliConfig) newBackupCmd() *cobra.Command {
Example: `cscli config backup ./my-backup`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
if err := cli.backup(args[0]); err != nil {
return fmt.Errorf("failed to backup config: %w", err)
}
return nil
},
RunE: runConfigBackup,
}
return cmd
return cmdConfigBackup
}

View file

@ -2,16 +2,21 @@ package main
import (
"fmt"
"path/filepath"
"github.com/fatih/color"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
)
func (cli *cliConfig) featureFlags(showRetired bool) error {
func runConfigFeatureFlags(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
showRetired, err := flags.GetBool("retired")
if err != nil {
return err
}
green := color.New(color.FgGreen).SprintFunc()
red := color.New(color.FgRed).SprintFunc()
yellow := color.New(color.FgYellow).SprintFunc()
@ -37,7 +42,6 @@ func (cli *cliConfig) featureFlags(showRetired bool) error {
if feat.State == fflag.RetiredState {
fmt.Printf("\n %s %s", magenta("RETIRED"), feat.DeprecationMsg)
}
fmt.Println()
}
@ -52,12 +56,10 @@ func (cli *cliConfig) featureFlags(showRetired bool) error {
retired = append(retired, feat)
continue
}
if feat.IsEnabled() {
enabled = append(enabled, feat)
continue
}
disabled = append(disabled, feat)
}
@ -85,14 +87,7 @@ func (cli *cliConfig) featureFlags(showRetired bool) error {
fmt.Println("To enable a feature you can: ")
fmt.Println(" - set the environment variable CROWDSEC_FEATURE_<uppercase_feature_name> to true")
featurePath, err := filepath.Abs(csconfig.GetFeatureFilePath(ConfigFilePath))
if err != nil {
// we already read the file, shouldn't happen
return err
}
fmt.Printf(" - add the line '- <feature_name>' to the file %s\n", featurePath)
fmt.Printf(" - add the line '- <feature_name>' to the file %s/feature.yaml\n", csConfig.ConfigPaths.ConfigDir)
fmt.Println()
if len(enabled) == 0 && len(disabled) == 0 {
@ -114,22 +109,18 @@ func (cli *cliConfig) featureFlags(showRetired bool) error {
return nil
}
func (cli *cliConfig) newFeatureFlagsCmd() *cobra.Command {
var showRetired bool
cmd := &cobra.Command{
func NewConfigFeatureFlagsCmd() *cobra.Command {
cmdConfigFeatureFlags := &cobra.Command{
Use: "feature-flags",
Short: "Displays feature flag status",
Long: `Displays the supported feature flags and their current status.`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.featureFlags(showRetired)
},
RunE: runConfigFeatureFlags,
}
flags := cmd.Flags()
flags.BoolVar(&showRetired, "retired", false, "Show retired features")
flags := cmdConfigFeatureFlags.Flags()
flags.Bool("retired", false, "Show retired features")
return cmd
return cmdConfigFeatureFlags
}

View file

@ -3,120 +3,26 @@ package main
import (
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func (cli *cliConfig) restoreHub(dirPath string) error {
cfg := cli.cfg()
hub, err := require.Hub(cfg, require.RemoteHub(cfg), nil)
if err != nil {
return err
}
for _, itype := range cwhub.ItemTypes {
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
if _, err = os.Stat(itemDirectory); err != nil {
log.Infof("no %s in backup", itype)
continue
}
/*restore the upstream items*/
upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
file, err := os.ReadFile(upstreamListFN)
if err != nil {
return fmt.Errorf("error while opening %s: %w", upstreamListFN, err)
}
var upstreamList []string
err = json.Unmarshal(file, &upstreamList)
if err != nil {
return fmt.Errorf("error unmarshaling %s: %w", upstreamListFN, err)
}
for _, toinstall := range upstreamList {
item := hub.GetItem(itype, toinstall)
if item == nil {
log.Errorf("Item %s/%s not found in hub", itype, toinstall)
continue
}
if err = item.Install(false, false); err != nil {
log.Errorf("Error while installing %s : %s", toinstall, err)
}
}
/*restore the local and tainted items*/
files, err := os.ReadDir(itemDirectory)
if err != nil {
return fmt.Errorf("failed enumerating files of %s: %w", itemDirectory, err)
}
for _, file := range files {
// this was the upstream data
if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
continue
}
if itype == cwhub.PARSERS || itype == cwhub.POSTOVERFLOWS {
// we expect a stage here
if !file.IsDir() {
continue
}
stage := file.Name()
stagedir := fmt.Sprintf("%s/%s/%s/", cfg.ConfigPaths.ConfigDir, itype, stage)
log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage directory %s: %w", stagedir, err)
}
// find items
ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
if err != nil {
return fmt.Errorf("failed enumerating files of %s: %w", itemDirectory+"/"+stage, err)
}
// finally copy item
for _, tfile := range ifiles {
log.Infof("Going to restore local/tainted [%s]", tfile.Name())
sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s: %w", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
} else {
log.Infof("Going to restore local/tainted [%s]", file.Name())
sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
destinationFile := fmt.Sprintf("%s/%s/%s", cfg.ConfigPaths.ConfigDir, itype, file.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s: %w", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
}
}
return nil
type OldAPICfg struct {
MachineID string `json:"machine_id"`
Password string `json:"password"`
}
/*
Restore crowdsec configurations to directory <dirPath>:
/* Restore crowdsec configurations to directory <dirPath> :
- Main config (config.yaml)
- Profiles config (profiles.yaml)
@ -124,66 +30,91 @@ func (cli *cliConfig) restoreHub(dirPath string) error {
- Backup of API credentials (local API and online API)
- List of scenarios, parsers, postoverflows and collections that are up-to-date
- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections
- Acquisition files (acquis.yaml, acquis.d/*.yaml)
*/
func (cli *cliConfig) restore(dirPath string) error {
func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
var err error
cfg := cli.cfg()
backupMain := fmt.Sprintf("%s/config.yaml", dirPath)
if _, err = os.Stat(backupMain); err == nil {
if cfg.ConfigPaths != nil && cfg.ConfigPaths.ConfigDir != "" {
if err = CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", cfg.ConfigPaths.ConfigDir)); err != nil {
return fmt.Errorf("failed copy %s to %s: %w", backupMain, cfg.ConfigPaths.ConfigDir, err)
if !oldBackup {
backupMain := fmt.Sprintf("%s/config.yaml", dirPath)
if _, err = os.Stat(backupMain); err == nil {
if csConfig.ConfigPaths != nil && csConfig.ConfigPaths.ConfigDir != "" {
if err = types.CopyFile(backupMain, fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", backupMain, csConfig.ConfigPaths.ConfigDir, err)
}
}
}
}
// Now we have config.yaml, we should regenerate config struct to have rights paths etc
ConfigFilePath = fmt.Sprintf("%s/config.yaml", cfg.ConfigPaths.ConfigDir)
// Now we have config.yaml, we should regenerate config struct to have rights paths etc
ConfigFilePath = fmt.Sprintf("%s/config.yaml", csConfig.ConfigPaths.ConfigDir)
log.Debug("Reloading configuration")
initConfig()
csConfig, _, err = loadConfigFor("config")
if err != nil {
return fmt.Errorf("failed to reload configuration: %w", err)
}
cfg = cli.cfg()
backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath)
if _, err = os.Stat(backupCAPICreds); err == nil {
if err = CopyFile(backupCAPICreds, cfg.API.Server.OnlineClient.CredentialsFilePath); err != nil {
return fmt.Errorf("failed copy %s to %s: %w", backupCAPICreds, cfg.API.Server.OnlineClient.CredentialsFilePath, err)
backupCAPICreds := fmt.Sprintf("%s/online_api_credentials.yaml", dirPath)
if _, err = os.Stat(backupCAPICreds); err == nil {
if err = types.CopyFile(backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", backupCAPICreds, csConfig.API.Server.OnlineClient.CredentialsFilePath, err)
}
}
}
backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath)
if _, err = os.Stat(backupLAPICreds); err == nil {
if err = CopyFile(backupLAPICreds, cfg.API.Client.CredentialsFilePath); err != nil {
return fmt.Errorf("failed copy %s to %s: %w", backupLAPICreds, cfg.API.Client.CredentialsFilePath, err)
backupLAPICreds := fmt.Sprintf("%s/local_api_credentials.yaml", dirPath)
if _, err = os.Stat(backupLAPICreds); err == nil {
if err = types.CopyFile(backupLAPICreds, csConfig.API.Client.CredentialsFilePath); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", backupLAPICreds, csConfig.API.Client.CredentialsFilePath, err)
}
}
}
backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath)
if _, err = os.Stat(backupProfiles); err == nil {
if err = CopyFile(backupProfiles, cfg.API.Server.ProfilesPath); err != nil {
return fmt.Errorf("failed copy %s to %s: %w", backupProfiles, cfg.API.Server.ProfilesPath, err)
backupProfiles := fmt.Sprintf("%s/profiles.yaml", dirPath)
if _, err = os.Stat(backupProfiles); err == nil {
if err = types.CopyFile(backupProfiles, csConfig.API.Server.ProfilesPath); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", backupProfiles, csConfig.API.Server.ProfilesPath, err)
}
}
} else {
var oldAPICfg OldAPICfg
backupOldAPICfg := fmt.Sprintf("%s/api_creds.json", dirPath)
jsonFile, err := os.Open(backupOldAPICfg)
if err != nil {
log.Warningf("failed to open %s : %s", backupOldAPICfg, err)
} else {
byteValue, _ := io.ReadAll(jsonFile)
err = json.Unmarshal(byteValue, &oldAPICfg)
if err != nil {
return fmt.Errorf("failed to load json file %s : %s", backupOldAPICfg, err)
}
apiCfg := csconfig.ApiCredentialsCfg{
Login: oldAPICfg.MachineID,
Password: oldAPICfg.Password,
URL: CAPIBaseURL,
}
apiConfigDump, err := yaml.Marshal(apiCfg)
if err != nil {
return fmt.Errorf("unable to dump api credentials: %s", err)
}
apiConfigDumpFile := fmt.Sprintf("%s/online_api_credentials.yaml", csConfig.ConfigPaths.ConfigDir)
if csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" {
apiConfigDumpFile = csConfig.API.Server.OnlineClient.CredentialsFilePath
}
err = os.WriteFile(apiConfigDumpFile, apiConfigDump, 0o644)
if err != nil {
return fmt.Errorf("write api credentials in '%s' failed: %s", apiConfigDumpFile, err)
}
log.Infof("Saved API credentials to %s", apiConfigDumpFile)
}
}
backupSimulation := fmt.Sprintf("%s/simulation.yaml", dirPath)
if _, err = os.Stat(backupSimulation); err == nil {
if err = CopyFile(backupSimulation, cfg.ConfigPaths.SimulationFilePath); err != nil {
return fmt.Errorf("failed copy %s to %s: %w", backupSimulation, cfg.ConfigPaths.SimulationFilePath, err)
if err = types.CopyFile(backupSimulation, csConfig.ConfigPaths.SimulationFilePath); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", backupSimulation, csConfig.ConfigPaths.SimulationFilePath, err)
}
}
/*if there is a acquisition dir, restore its content*/
if cfg.Crowdsec.AcquisitionDirPath != "" {
if err = os.MkdirAll(cfg.Crowdsec.AcquisitionDirPath, 0o700); err != nil {
return fmt.Errorf("error while creating %s: %w", cfg.Crowdsec.AcquisitionDirPath, err)
if csConfig.Crowdsec.AcquisitionDirPath != "" {
if err = os.Mkdir(csConfig.Crowdsec.AcquisitionDirPath, 0o700); err != nil {
return fmt.Errorf("error while creating %s : %s", csConfig.Crowdsec.AcquisitionDirPath, err)
}
}
@ -192,60 +123,86 @@ func (cli *cliConfig) restore(dirPath string) error {
if _, err = os.Stat(backupAcquisition); err == nil {
log.Debugf("restoring backup'ed %s", backupAcquisition)
if err = CopyFile(backupAcquisition, cfg.Crowdsec.AcquisitionFilePath); err != nil {
return fmt.Errorf("failed copy %s to %s: %w", backupAcquisition, cfg.Crowdsec.AcquisitionFilePath, err)
if err = types.CopyFile(backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", backupAcquisition, csConfig.Crowdsec.AcquisitionFilePath, err)
}
}
// if there are files in the acquis backup dir, restore them
// if there is files in the acquis backup dir, restore them
acquisBackupDir := filepath.Join(dirPath, "acquis", "*.yaml")
if acquisFiles, err := filepath.Glob(acquisBackupDir); err == nil {
for _, acquisFile := range acquisFiles {
targetFname, err := filepath.Abs(cfg.Crowdsec.AcquisitionDirPath + "/" + filepath.Base(acquisFile))
targetFname, err := filepath.Abs(csConfig.Crowdsec.AcquisitionDirPath + "/" + filepath.Base(acquisFile))
if err != nil {
return fmt.Errorf("while saving %s to %s: %w", acquisFile, targetFname, err)
return errors.Wrapf(err, "while saving %s to %s", acquisFile, targetFname)
}
log.Debugf("restoring %s to %s", acquisFile, targetFname)
if err = CopyFile(acquisFile, targetFname); err != nil {
return fmt.Errorf("failed copy %s to %s: %w", acquisFile, targetFname, err)
if err = types.CopyFile(acquisFile, targetFname); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err)
}
}
}
if cfg.Crowdsec != nil && len(cfg.Crowdsec.AcquisitionFiles) > 0 {
for _, acquisFile := range cfg.Crowdsec.AcquisitionFiles {
if csConfig.Crowdsec != nil && len(csConfig.Crowdsec.AcquisitionFiles) > 0 {
for _, acquisFile := range csConfig.Crowdsec.AcquisitionFiles {
log.Infof("backup filepath from dir -> %s", acquisFile)
// if it was the default one, it has already been backed up
if cfg.Crowdsec.AcquisitionFilePath == acquisFile {
if csConfig.Crowdsec.AcquisitionFilePath == acquisFile {
log.Infof("skip this one")
continue
}
targetFname, err := filepath.Abs(filepath.Join(acquisBackupDir, filepath.Base(acquisFile)))
if err != nil {
return fmt.Errorf("while saving %s to %s: %w", acquisFile, acquisBackupDir, err)
return errors.Wrapf(err, "while saving %s to %s", acquisFile, acquisBackupDir)
}
if err = CopyFile(acquisFile, targetFname); err != nil {
return fmt.Errorf("failed copy %s to %s: %w", acquisFile, targetFname, err)
if err = types.CopyFile(acquisFile, targetFname); err != nil {
return fmt.Errorf("failed copy %s to %s : %s", acquisFile, targetFname, err)
}
log.Infof("Saved acquis %s to %s", acquisFile, targetFname)
}
}
if err = cli.restoreHub(dirPath); err != nil {
return fmt.Errorf("failed to restore hub config: %w", err)
if err = RestoreHub(dirPath); err != nil {
return fmt.Errorf("failed to restore hub config : %s", err)
}
return nil
}
func (cli *cliConfig) newRestoreCmd() *cobra.Command {
cmd := &cobra.Command{
func runConfigRestore(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
oldBackup, err := flags.GetBool("old-backup")
if err != nil {
return err
}
if err := csConfig.LoadHub(); err != nil {
return err
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
return fmt.Errorf("failed to get Hub index: %w", err)
}
if err := restoreConfigFromDirectory(args[0], oldBackup); err != nil {
return fmt.Errorf("failed to restore config from %s: %w", args[0], err)
}
return nil
}
func NewConfigRestoreCmd() *cobra.Command {
cmdConfigRestore := &cobra.Command{
Use: `restore "directory"`,
Short: `Restore config in backup "directory"`,
Long: `Restore the crowdsec configuration from specified backup "directory" including:
@ -258,16 +215,11 @@ func (cli *cliConfig) newRestoreCmd() *cobra.Command {
- Backup of API credentials (local API and online API)`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
dirPath := args[0]
if err := cli.restore(dirPath); err != nil {
return fmt.Errorf("failed to restore config from %s: %w", dirPath, err)
}
return nil
},
RunE: runConfigRestore,
}
return cmd
flags := cmdConfigRestore.Flags()
flags.BoolP("old-backup", "", false, "To use when you are upgrading crowdsec v0.X to v1.X and you need to restore backup from v0.X")
return cmdConfigRestore
}

View file

@ -7,18 +7,15 @@ import (
"text/template"
"github.com/antonmedv/expr"
"github.com/sanity-io/litter"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
)
func (cli *cliConfig) showKey(key string) error {
cfg := cli.cfg()
func showConfigKey(key string) error {
type Env struct {
Config *csconfig.Config
}
@ -26,26 +23,25 @@ func (cli *cliConfig) showKey(key string) error {
opts := []expr.Option{}
opts = append(opts, exprhelpers.GetExprOptions(map[string]interface{}{})...)
opts = append(opts, expr.Env(Env{}))
program, err := expr.Compile(key, opts...)
if err != nil {
return err
}
output, err := expr.Run(program, Env{Config: cfg})
output, err := expr.Run(program, Env{Config: csConfig})
if err != nil {
return err
}
switch cfg.Cscli.Output {
switch csConfig.Cscli.Output {
case "human", "raw":
// Don't use litter for strings, it adds quotes
// that would break compatibility with previous versions
switch output.(type) {
case string:
fmt.Println(output)
fmt.Printf("%s\n", output)
case int:
fmt.Printf("%d\n", output)
default:
litter.Dump(output)
fmt.Printf("%v\n", output)
}
case "json":
data, err := json.MarshalIndent(output, "", " ")
@ -53,16 +49,15 @@ func (cli *cliConfig) showKey(key string) error {
return fmt.Errorf("failed to marshal configuration: %w", err)
}
fmt.Println(string(data))
fmt.Printf("%s\n", string(data))
}
return nil
}
func (cli *cliConfig) template() string {
return `Global:
var configShowTemplate = `Global:
{{- if .ConfigPaths }}
- Configuration Folder : {{.ConfigPaths.ConfigDir}}
- Configuration Folder : {{.ConfigPaths.ConfigDir}}
- Data Folder : {{.ConfigPaths.DataDir}}
- Hub Folder : {{.ConfigPaths.HubDir}}
@ -76,7 +71,7 @@ func (cli *cliConfig) template() string {
{{- end }}
{{- if .Crowdsec }}
Crowdsec{{if and .Crowdsec.Enable (not (ValueBool .Crowdsec.Enable))}} (disabled){{end}}:
Crowdsec:
- Acquisition File : {{.Crowdsec.AcquisitionFilePath}}
- Parsers routines : {{.Crowdsec.ParserRoutinesCount}}
{{- if .Crowdsec.AcquisitionDirPath }}
@ -88,6 +83,7 @@ Crowdsec{{if and .Crowdsec.Enable (not (ValueBool .Crowdsec.Enable))}} (disabled
cscli:
- Output : {{.Cscli.Output}}
- Hub Branch : {{.Cscli.HubBranch}}
- Hub Folder : {{.Cscli.HubDir}}
{{- end }}
{{- if .API }}
@ -101,9 +97,8 @@ API Client:
{{- end }}
{{- if .API.Server }}
Local API Server{{if and .API.Server.Enable (not (ValueBool .API.Server.Enable))}} (disabled){{end}}:
Local API Server:
- Listen URL : {{.API.Server.ListenURI}}
- Listen Socket : {{.API.Server.ListenSocket}}
- Profile File : {{.API.Server.ProfilesPath}}
{{- if .API.Server.TLS }}
@ -169,12 +164,6 @@ Central API:
- User : {{.DbConfig.User}}
- DB Name : {{.DbConfig.DbName}}
{{- end }}
{{- if .DbConfig.MaxOpenConns }}
- Max Open Conns : {{.DbConfig.MaxOpenConns}}
{{- end }}
{{- if ne .DbConfig.DecisionBulkSize 0 }}
- Decision Bulk Size : {{.DbConfig.DecisionBulkSize}}
{{- end }}
{{- if .DbConfig.Flush }}
{{- if .DbConfig.Flush.MaxAge }}
- Flush age : {{.DbConfig.Flush.MaxAge}}
@ -185,74 +174,64 @@ Central API:
{{- end }}
{{- end }}
`
}
func (cli *cliConfig) show() error {
cfg := cli.cfg()
func runConfigShow(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
switch cfg.Cscli.Output {
if err := csConfig.LoadAPIClient(); err != nil {
log.Errorf("failed to load API client configuration: %s", err)
// don't return, we can still show the configuration
}
key, err := flags.GetString("key")
if err != nil {
return err
}
if key != "" {
return showConfigKey(key)
}
switch csConfig.Cscli.Output {
case "human":
// The tests on .Enable look funny because the option has a true default which has
// not been set yet (we don't really load the LAPI) and go templates don't dereference
// pointers in boolean tests. Prefix notation is the cherry on top.
funcs := template.FuncMap{
// can't use generics here
"ValueBool": func(b *bool) bool { return b != nil && *b },
}
tmp, err := template.New("config").Funcs(funcs).Parse(cli.template())
tmp, err := template.New("config").Parse(configShowTemplate)
if err != nil {
return err
}
err = tmp.Execute(os.Stdout, cfg)
err = tmp.Execute(os.Stdout, csConfig)
if err != nil {
return err
}
case "json":
data, err := json.MarshalIndent(cfg, "", " ")
data, err := json.MarshalIndent(csConfig, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal configuration: %w", err)
}
fmt.Println(string(data))
fmt.Printf("%s\n", string(data))
case "raw":
data, err := yaml.Marshal(cfg)
data, err := yaml.Marshal(csConfig)
if err != nil {
return fmt.Errorf("failed to marshal configuration: %w", err)
}
fmt.Println(string(data))
fmt.Printf("%s\n", string(data))
}
return nil
}
func (cli *cliConfig) newShowCmd() *cobra.Command {
var key string
cmd := &cobra.Command{
func NewConfigShowCmd() *cobra.Command {
cmdConfigShow := &cobra.Command{
Use: "show",
Short: "Displays current config",
Long: `Displays the current cli configuration.`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
if err := cli.cfg().LoadAPIClient(); err != nil {
log.Errorf("failed to load API client configuration: %s", err)
// don't return, we can still show the configuration
}
if key != "" {
return cli.showKey(key)
}
return cli.show()
},
RunE: runConfigShow,
}
flags := cmd.Flags()
flags.StringVarP(&key, "key", "", "", "Display only this value (Config.API.Server.ListenURI)")
flags := cmdConfigShow.Flags()
flags.StringP("key", "", "", "Display only this value (Config.API.Server.ListenURI)")
return cmd
return cmdConfigShow
}

View file

@ -1,26 +0,0 @@
package main
import (
"fmt"
"github.com/spf13/cobra"
)
func (cli *cliConfig) showYAML() error {
fmt.Println(mergedConfig)
return nil
}
func (cli *cliConfig) newShowYAMLCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "show-yaml",
Short: "Displays merged config.yaml + config.yaml.local",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.showYAML()
},
}
return cmd
}

View file

@ -6,10 +6,9 @@ import (
"encoding/json"
"errors"
"fmt"
"io/fs"
"net/url"
"os"
"strconv"
"strings"
"github.com/fatih/color"
"github.com/go-openapi/strfmt"
@ -17,63 +16,49 @@ import (
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
type cliConsole struct {
cfg configGetter
}
func NewCLIConsole(cfg configGetter) *cliConsole {
return &cliConsole{
cfg: cfg,
}
}
func (cli *cliConsole) NewCommand() *cobra.Command {
cmd := &cobra.Command{
func NewConsoleCmd() *cobra.Command {
var cmdConsole = &cobra.Command{
Use: "console [action]",
Short: "Manage interaction with Crowdsec console (https://app.crowdsec.net)",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := require.LAPI(cfg); err != nil {
return err
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
var fdErr *fs.PathError
if errors.As(err, &fdErr) {
log.Fatalf("Unable to load Local API : %s", fdErr)
}
if err != nil {
log.Fatalf("Unable to load required Local API Configuration : %s", err)
}
log.Fatal("Local API is disabled, please run this command on the local API machine")
}
if err := require.CAPI(cfg); err != nil {
return err
if csConfig.DisableAPI {
log.Fatal("Local API is disabled, please run this command on the local API machine")
}
if err := require.CAPIRegistered(cfg); err != nil {
return err
if csConfig.API.Server.OnlineClient == nil {
log.Fatalf("No configuration for Central API (CAPI) in '%s'", *csConfig.FilePath)
}
if csConfig.API.Server.OnlineClient.Credentials == nil {
log.Fatal("You must configure Central API (CAPI) with `cscli capi register` before accessing console features.")
}
return nil
},
}
cmd.AddCommand(cli.newEnrollCmd())
cmd.AddCommand(cli.newEnableCmd())
cmd.AddCommand(cli.newDisableCmd())
cmd.AddCommand(cli.newStatusCmd())
return cmd
}
func (cli *cliConsole) newEnrollCmd() *cobra.Command {
name := ""
overwrite := false
tags := []string{}
opts := []string{}
cmd := &cobra.Command{
cmdEnroll := &cobra.Command{
Use: "enroll [enroll-key]",
Short: "Enroll this instance to https://app.crowdsec.net [requires local API]",
Long: `
@ -81,115 +66,71 @@ Enroll this instance to https://app.crowdsec.net
You can get your enrollment key by creating an account on https://app.crowdsec.net.
After running this command your will need to validate the enrollment in the webapp.`,
Example: fmt.Sprintf(`cscli console enroll YOUR-ENROLL-KEY
Example: `cscli console enroll YOUR-ENROLL-KEY
cscli console enroll --name [instance_name] YOUR-ENROLL-KEY
cscli console enroll --name [instance_name] --tags [tag_1] --tags [tag_2] YOUR-ENROLL-KEY
cscli console enroll --enable context,manual YOUR-ENROLL-KEY
valid options are : %s,all (see 'cscli console status' for details)`, strings.Join(csconfig.CONSOLE_CONFIGS, ",")),
`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
cfg := cli.cfg()
password := strfmt.Password(cfg.API.Server.OnlineClient.Credentials.Password)
apiURL, err := url.Parse(cfg.API.Server.OnlineClient.Credentials.URL)
Run: func(cmd *cobra.Command, args []string) {
password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
apiURL, err := url.Parse(csConfig.API.Server.OnlineClient.Credentials.URL)
if err != nil {
return fmt.Errorf("could not parse CAPI URL: %w", err)
log.Fatalf("Could not parse CAPI URL : %s", err)
}
hub, err := require.Hub(cfg, nil, nil)
if err != nil {
return err
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
scenarios, err := hub.GetInstalledNamesByType(cwhub.SCENARIOS)
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to load hub index : %s", err)
log.Info("Run 'sudo cscli hub update' to get the hub index")
}
scenarios, err := cwhub.GetInstalledScenariosAsString()
if err != nil {
return fmt.Errorf("failed to get installed scenarios: %w", err)
log.Fatalf("failed to get scenarios : %s", err)
}
if len(scenarios) == 0 {
scenarios = make([]string, 0)
}
enableOpts := []string{csconfig.SEND_MANUAL_SCENARIOS, csconfig.SEND_TAINTED_SCENARIOS}
if len(opts) != 0 {
for _, opt := range opts {
valid := false
if opt == "all" {
enableOpts = csconfig.CONSOLE_CONFIGS
break
}
for _, availableOpt := range csconfig.CONSOLE_CONFIGS {
if opt == availableOpt {
valid = true
enable := true
for _, enabledOpt := range enableOpts {
if opt == enabledOpt {
enable = false
continue
}
}
if enable {
enableOpts = append(enableOpts, opt)
}
break
}
}
if !valid {
return fmt.Errorf("option %s doesn't exist", opt)
}
}
}
c, _ := apiclient.NewClient(&apiclient.Config{
MachineID: cli.cfg().API.Server.OnlineClient.Credentials.Login,
MachineID: csConfig.API.Server.OnlineClient.Credentials.Login,
Password: password,
Scenarios: scenarios,
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
URL: apiURL,
VersionPrefix: "v3",
})
resp, err := c.Auth.EnrollWatcher(context.Background(), args[0], name, tags, overwrite)
if err != nil {
return fmt.Errorf("could not enroll instance: %w", err)
log.Fatalf("Could not enroll instance: %s", err)
}
if resp.Response.StatusCode == 200 && !overwrite {
log.Warning("Instance already enrolled. You can use '--overwrite' to force enroll")
return nil
return
}
if err := cli.setConsoleOpts(enableOpts, true); err != nil {
return err
SetConsoleOpts(csconfig.CONSOLE_CONFIGS, true)
if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
log.Fatalf("failed writing console config : %s", err)
}
for _, opt := range enableOpts {
log.Infof("Enabled %s : %s", opt, csconfig.CONSOLE_CONFIGS_HELP[opt])
}
log.Info("Watcher successfully enrolled. Visit https://app.crowdsec.net to accept it.")
log.Info("Please restart crowdsec after accepting the enrollment.")
return nil
log.Infof("Enabled tainted&manual alerts sharing, see 'cscli console status'.")
log.Infof("Watcher successfully enrolled. Visit https://app.crowdsec.net to accept it.")
log.Infof("Please restart crowdsec after accepting the enrollment.")
},
}
cmdEnroll.Flags().StringVarP(&name, "name", "n", "", "Name to display in the console")
cmdEnroll.Flags().BoolVarP(&overwrite, "overwrite", "", false, "Force enroll the instance")
cmdEnroll.Flags().StringSliceVarP(&tags, "tags", "t", tags, "Tags to display in the console")
cmdConsole.AddCommand(cmdEnroll)
flags := cmd.Flags()
flags.StringVarP(&name, "name", "n", "", "Name to display in the console")
flags.BoolVarP(&overwrite, "overwrite", "", false, "Force enroll the instance")
flags.StringSliceVarP(&tags, "tags", "t", tags, "Tags to display in the console")
flags.StringSliceVarP(&opts, "enable", "e", opts, "Enable console options")
var enableAll, disableAll bool
return cmd
}
func (cli *cliConsole) newEnableCmd() *cobra.Command {
var enableAll bool
cmd := &cobra.Command{
cmdEnable := &cobra.Command{
Use: "enable [option]",
Short: "Enable a console option",
Example: "sudo cscli console enable tainted",
@ -197,245 +138,195 @@ func (cli *cliConsole) newEnableCmd() *cobra.Command {
Enable given information push to the central API. Allows to empower the console`,
ValidArgs: csconfig.CONSOLE_CONFIGS,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
Run: func(cmd *cobra.Command, args []string) {
if enableAll {
if err := cli.setConsoleOpts(csconfig.CONSOLE_CONFIGS, true); err != nil {
return err
}
SetConsoleOpts(csconfig.CONSOLE_CONFIGS, true)
log.Infof("All features have been enabled successfully")
} else {
if len(args) == 0 {
return errors.New("you must specify at least one feature to enable")
}
if err := cli.setConsoleOpts(args, true); err != nil {
return err
log.Fatalf("You must specify at least one feature to enable")
}
SetConsoleOpts(args, true)
log.Infof("%v have been enabled", args)
}
if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
log.Fatalf("failed writing console config : %s", err)
}
log.Infof(ReloadMessage())
return nil
},
}
cmd.Flags().BoolVarP(&enableAll, "all", "a", false, "Enable all console options")
cmdEnable.Flags().BoolVarP(&enableAll, "all", "a", false, "Enable all console options")
cmdConsole.AddCommand(cmdEnable)
return cmd
}
func (cli *cliConsole) newDisableCmd() *cobra.Command {
var disableAll bool
cmd := &cobra.Command{
cmdDisable := &cobra.Command{
Use: "disable [option]",
Short: "Disable a console option",
Example: "sudo cscli console disable tainted",
Long: `
Disable given information push to the central API.`,
ValidArgs: csconfig.CONSOLE_CONFIGS,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
Run: func(cmd *cobra.Command, args []string) {
if disableAll {
if err := cli.setConsoleOpts(csconfig.CONSOLE_CONFIGS, false); err != nil {
return err
}
log.Infof("All features have been disabled")
SetConsoleOpts(csconfig.CONSOLE_CONFIGS, false)
} else {
if err := cli.setConsoleOpts(args, false); err != nil {
return err
}
log.Infof("%v have been disabled", args)
SetConsoleOpts(args, false)
}
if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
log.Fatalf("failed writing console config : %s", err)
}
if disableAll {
log.Infof("All features have been disabled")
} else {
log.Infof("%v have been disabled", args)
}
log.Infof(ReloadMessage())
return nil
},
}
cmd.Flags().BoolVarP(&disableAll, "all", "a", false, "Disable all console options")
cmdDisable.Flags().BoolVarP(&disableAll, "all", "a", false, "Disable all console options")
cmdConsole.AddCommand(cmdDisable)
return cmd
}
func (cli *cliConsole) newStatusCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "status",
Short: "Shows status of the console options",
cmdConsoleStatus := &cobra.Command{
Use: "status [option]",
Short: "Shows status of one or all console options",
Example: `sudo cscli console status`,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
consoleCfg := cfg.API.Server.ConsoleConfig
switch cfg.Cscli.Output {
Run: func(cmd *cobra.Command, args []string) {
switch csConfig.Cscli.Output {
case "human":
cmdConsoleStatusTable(color.Output, *consoleCfg)
cmdConsoleStatusTable(color.Output, *csConfig)
case "json":
out := map[string](*bool){
csconfig.SEND_MANUAL_SCENARIOS: consoleCfg.ShareManualDecisions,
csconfig.SEND_CUSTOM_SCENARIOS: consoleCfg.ShareCustomScenarios,
csconfig.SEND_TAINTED_SCENARIOS: consoleCfg.ShareTaintedScenarios,
csconfig.SEND_CONTEXT: consoleCfg.ShareContext,
csconfig.CONSOLE_MANAGEMENT: consoleCfg.ConsoleManagement,
}
data, err := json.MarshalIndent(out, "", " ")
data, err := json.MarshalIndent(csConfig.API.Server.ConsoleConfig, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal configuration: %w", err)
log.Fatalf("failed to marshal configuration: %s", err)
}
fmt.Println(string(data))
fmt.Printf("%s\n", string(data))
case "raw":
csvwriter := csv.NewWriter(os.Stdout)
err := csvwriter.Write([]string{"option", "enabled"})
if err != nil {
return err
log.Fatal(err)
}
rows := [][]string{
{csconfig.SEND_MANUAL_SCENARIOS, strconv.FormatBool(*consoleCfg.ShareManualDecisions)},
{csconfig.SEND_CUSTOM_SCENARIOS, strconv.FormatBool(*consoleCfg.ShareCustomScenarios)},
{csconfig.SEND_TAINTED_SCENARIOS, strconv.FormatBool(*consoleCfg.ShareTaintedScenarios)},
{csconfig.SEND_CONTEXT, strconv.FormatBool(*consoleCfg.ShareContext)},
{csconfig.CONSOLE_MANAGEMENT, strconv.FormatBool(*consoleCfg.ConsoleManagement)},
{csconfig.SEND_MANUAL_SCENARIOS, fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareManualDecisions)},
{csconfig.SEND_CUSTOM_SCENARIOS, fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareCustomScenarios)},
{csconfig.SEND_TAINTED_SCENARIOS, fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios)},
{csconfig.SEND_CONTEXT, fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ShareContext)},
{csconfig.CONSOLE_MANAGEMENT, fmt.Sprintf("%t", *csConfig.API.Server.ConsoleConfig.ConsoleManagement)},
}
for _, row := range rows {
err = csvwriter.Write(row)
if err != nil {
return err
log.Fatal(err)
}
}
csvwriter.Flush()
}
return nil
},
}
cmdConsole.AddCommand(cmdConsoleStatus)
return cmd
return cmdConsole
}
func (cli *cliConsole) dumpConfig() error {
serverCfg := cli.cfg().API.Server
out, err := yaml.Marshal(serverCfg.ConsoleConfig)
if err != nil {
return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", serverCfg.ConsoleConfigPath, err)
}
if serverCfg.ConsoleConfigPath == "" {
serverCfg.ConsoleConfigPath = csconfig.DefaultConsoleConfigFilePath
log.Debugf("Empty console_path, defaulting to %s", serverCfg.ConsoleConfigPath)
}
if err := os.WriteFile(serverCfg.ConsoleConfigPath, out, 0o600); err != nil {
return fmt.Errorf("while dumping console config to %s: %w", serverCfg.ConsoleConfigPath, err)
}
return nil
}
func (cli *cliConsole) setConsoleOpts(args []string, wanted bool) error {
cfg := cli.cfg()
consoleCfg := cfg.API.Server.ConsoleConfig
func SetConsoleOpts(args []string, wanted bool) {
for _, arg := range args {
switch arg {
case csconfig.CONSOLE_MANAGEMENT:
if !fflag.PapiClient.IsEnabled() {
continue
}
/*for each flag check if it's already set before setting it*/
if consoleCfg.ConsoleManagement != nil {
if *consoleCfg.ConsoleManagement == wanted {
if csConfig.API.Server.ConsoleConfig.ConsoleManagement != nil {
if *csConfig.API.Server.ConsoleConfig.ConsoleManagement == wanted {
log.Debugf("%s already set to %t", csconfig.CONSOLE_MANAGEMENT, wanted)
} else {
log.Infof("%s set to %t", csconfig.CONSOLE_MANAGEMENT, wanted)
*consoleCfg.ConsoleManagement = wanted
*csConfig.API.Server.ConsoleConfig.ConsoleManagement = wanted
}
} else {
log.Infof("%s set to %t", csconfig.CONSOLE_MANAGEMENT, wanted)
consoleCfg.ConsoleManagement = ptr.Of(wanted)
csConfig.API.Server.ConsoleConfig.ConsoleManagement = types.BoolPtr(wanted)
}
if cfg.API.Server.OnlineClient.Credentials != nil {
if csConfig.API.Server.OnlineClient.Credentials != nil {
changed := false
if wanted && cfg.API.Server.OnlineClient.Credentials.PapiURL == "" {
if wanted && csConfig.API.Server.OnlineClient.Credentials.PapiURL == "" {
changed = true
cfg.API.Server.OnlineClient.Credentials.PapiURL = types.PAPIBaseURL
} else if !wanted && cfg.API.Server.OnlineClient.Credentials.PapiURL != "" {
csConfig.API.Server.OnlineClient.Credentials.PapiURL = types.PAPIBaseURL
} else if !wanted && csConfig.API.Server.OnlineClient.Credentials.PapiURL != "" {
changed = true
cfg.API.Server.OnlineClient.Credentials.PapiURL = ""
csConfig.API.Server.OnlineClient.Credentials.PapiURL = ""
}
if changed {
fileContent, err := yaml.Marshal(cfg.API.Server.OnlineClient.Credentials)
fileContent, err := yaml.Marshal(csConfig.API.Server.OnlineClient.Credentials)
if err != nil {
return fmt.Errorf("cannot marshal credentials: %w", err)
log.Fatalf("Cannot marshal credentials: %s", err)
}
log.Infof("Updating credentials file: %s", cfg.API.Server.OnlineClient.CredentialsFilePath)
err = os.WriteFile(cfg.API.Server.OnlineClient.CredentialsFilePath, fileContent, 0o600)
log.Infof("Updating credentials file: %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
err = os.WriteFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, fileContent, 0600)
if err != nil {
return fmt.Errorf("cannot write credentials file: %w", err)
log.Fatalf("Cannot write credentials file: %s", err)
}
}
}
case csconfig.SEND_CUSTOM_SCENARIOS:
/*for each flag check if it's already set before setting it*/
if consoleCfg.ShareCustomScenarios != nil {
if *consoleCfg.ShareCustomScenarios == wanted {
if csConfig.API.Server.ConsoleConfig.ShareCustomScenarios != nil {
if *csConfig.API.Server.ConsoleConfig.ShareCustomScenarios == wanted {
log.Debugf("%s already set to %t", csconfig.SEND_CUSTOM_SCENARIOS, wanted)
} else {
log.Infof("%s set to %t", csconfig.SEND_CUSTOM_SCENARIOS, wanted)
*consoleCfg.ShareCustomScenarios = wanted
*csConfig.API.Server.ConsoleConfig.ShareCustomScenarios = wanted
}
} else {
log.Infof("%s set to %t", csconfig.SEND_CUSTOM_SCENARIOS, wanted)
consoleCfg.ShareCustomScenarios = ptr.Of(wanted)
csConfig.API.Server.ConsoleConfig.ShareCustomScenarios = types.BoolPtr(wanted)
}
case csconfig.SEND_TAINTED_SCENARIOS:
/*for each flag check if it's already set before setting it*/
if consoleCfg.ShareTaintedScenarios != nil {
if *consoleCfg.ShareTaintedScenarios == wanted {
if csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios != nil {
if *csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios == wanted {
log.Debugf("%s already set to %t", csconfig.SEND_TAINTED_SCENARIOS, wanted)
} else {
log.Infof("%s set to %t", csconfig.SEND_TAINTED_SCENARIOS, wanted)
*consoleCfg.ShareTaintedScenarios = wanted
*csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios = wanted
}
} else {
log.Infof("%s set to %t", csconfig.SEND_TAINTED_SCENARIOS, wanted)
consoleCfg.ShareTaintedScenarios = ptr.Of(wanted)
csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios = types.BoolPtr(wanted)
}
case csconfig.SEND_MANUAL_SCENARIOS:
/*for each flag check if it's already set before setting it*/
if consoleCfg.ShareManualDecisions != nil {
if *consoleCfg.ShareManualDecisions == wanted {
if csConfig.API.Server.ConsoleConfig.ShareManualDecisions != nil {
if *csConfig.API.Server.ConsoleConfig.ShareManualDecisions == wanted {
log.Debugf("%s already set to %t", csconfig.SEND_MANUAL_SCENARIOS, wanted)
} else {
log.Infof("%s set to %t", csconfig.SEND_MANUAL_SCENARIOS, wanted)
*consoleCfg.ShareManualDecisions = wanted
*csConfig.API.Server.ConsoleConfig.ShareManualDecisions = wanted
}
} else {
log.Infof("%s set to %t", csconfig.SEND_MANUAL_SCENARIOS, wanted)
consoleCfg.ShareManualDecisions = ptr.Of(wanted)
csConfig.API.Server.ConsoleConfig.ShareManualDecisions = types.BoolPtr(wanted)
}
case csconfig.SEND_CONTEXT:
/*for each flag check if it's already set before setting it*/
if consoleCfg.ShareContext != nil {
if *consoleCfg.ShareContext == wanted {
if csConfig.API.Server.ConsoleConfig.ShareContext != nil {
if *csConfig.API.Server.ConsoleConfig.ShareContext == wanted {
log.Debugf("%s already set to %t", csconfig.SEND_CONTEXT, wanted)
} else {
log.Infof("%s set to %t", csconfig.SEND_CONTEXT, wanted)
*consoleCfg.ShareContext = wanted
*csConfig.API.Server.ConsoleConfig.ShareContext = wanted
}
} else {
log.Infof("%s set to %t", csconfig.SEND_CONTEXT, wanted)
consoleCfg.ShareContext = ptr.Of(wanted)
csConfig.API.Server.ConsoleConfig.ShareContext = types.BoolPtr(wanted)
}
default:
return fmt.Errorf("unknown flag %s", arg)
log.Fatalf("unknown flag %s", arg)
}
}
if err := cli.dumpConfig(); err != nil {
return fmt.Errorf("failed writing console config: %w", err)
}
return nil
}

View file

@ -4,12 +4,12 @@ import (
"io"
"github.com/aquasecurity/table"
"github.com/enescakir/emoji"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/emoji"
)
func cmdConsoleStatusTable(out io.Writer, consoleCfg csconfig.ConsoleConfig) {
func cmdConsoleStatusTable(out io.Writer, csConfig csconfig.Config) {
t := newTable(out)
t.SetRowLines(false)
@ -17,32 +17,43 @@ func cmdConsoleStatusTable(out io.Writer, consoleCfg csconfig.ConsoleConfig) {
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
for _, option := range csconfig.CONSOLE_CONFIGS {
activated := emoji.CrossMark
switch option {
case csconfig.SEND_CUSTOM_SCENARIOS:
if *consoleCfg.ShareCustomScenarios {
activated = emoji.CheckMarkButton
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareCustomScenarios {
activated = string(emoji.CheckMarkButton)
}
case csconfig.SEND_MANUAL_SCENARIOS:
if *consoleCfg.ShareManualDecisions {
activated = emoji.CheckMarkButton
}
case csconfig.SEND_TAINTED_SCENARIOS:
if *consoleCfg.ShareTaintedScenarios {
activated = emoji.CheckMarkButton
}
case csconfig.SEND_CONTEXT:
if *consoleCfg.ShareContext {
activated = emoji.CheckMarkButton
}
case csconfig.CONSOLE_MANAGEMENT:
if *consoleCfg.ConsoleManagement {
activated = emoji.CheckMarkButton
}
}
t.AddRow(option, activated, csconfig.CONSOLE_CONFIGS_HELP[option])
t.AddRow(option, activated, "Send alerts from custom scenarios to the console")
case csconfig.SEND_MANUAL_SCENARIOS:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareManualDecisions {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send manual decisions to the console")
case csconfig.SEND_TAINTED_SCENARIOS:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareTaintedScenarios {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send alerts from tainted scenarios to the console")
case csconfig.SEND_CONTEXT:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ShareContext {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Send context with alerts to the console")
case csconfig.CONSOLE_MANAGEMENT:
activated := string(emoji.CrossMark)
if *csConfig.API.Server.ConsoleConfig.ConsoleManagement {
activated = string(emoji.CheckMarkButton)
}
t.AddRow(option, activated, "Receive decisions from console")
}
}
t.Render()

View file

@ -1,82 +0,0 @@
package main
import (
"fmt"
"io"
"os"
"path/filepath"
log "github.com/sirupsen/logrus"
)
/*help to copy the file, ioutil doesn't offer the feature*/
func copyFileContents(src, dst string) (err error) {
in, err := os.Open(src)
if err != nil {
return
}
defer in.Close()
out, err := os.Create(dst)
if err != nil {
return
}
defer func() {
cerr := out.Close()
if err == nil {
err = cerr
}
}()
if _, err = io.Copy(out, in); err != nil {
return
}
err = out.Sync()
return
}
/*copy the file, ioutile doesn't offer the feature*/
func CopyFile(sourceSymLink, destinationFile string) error {
sourceFile, err := filepath.EvalSymlinks(sourceSymLink)
if err != nil {
log.Infof("Not a symlink : %s", err)
sourceFile = sourceSymLink
}
sourceFileStat, err := os.Stat(sourceFile)
if err != nil {
return err
}
if !sourceFileStat.Mode().IsRegular() {
// cannot copy non-regular files (e.g., directories,
// symlinks, devices, etc.)
return fmt.Errorf("copyFile: non-regular source file %s (%q)", sourceFileStat.Name(), sourceFileStat.Mode().String())
}
destinationFileStat, err := os.Stat(destinationFile)
if err != nil {
if !os.IsNotExist(err) {
return err
}
} else {
if !(destinationFileStat.Mode().IsRegular()) {
return fmt.Errorf("copyFile: non-regular destination file %s (%q)", destinationFileStat.Name(), destinationFileStat.Mode().String())
}
if os.SameFile(sourceFileStat, destinationFileStat) {
return err
}
}
if err = os.Link(sourceFile, destinationFile); err != nil {
err = copyFileContents(sourceFile, destinationFile)
}
return err
}

View file

@ -1,8 +1,7 @@
//go:build linux
package main
import (
"errors"
"fmt"
"math"
"os"
@ -11,7 +10,6 @@ import (
"path/filepath"
"strconv"
"strings"
"syscall"
"unicode"
"github.com/AlecAivazis/survey/v2"
@ -19,18 +17,16 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/metabase"
)
var (
metabaseUser = "crowdsec@crowdsec.net"
metabasePassword string
metabaseDBPath string
metabaseDbPath string
metabaseConfigPath string
metabaseConfigFolder = "metabase/"
metabaseConfigFile = "metabase.yaml"
metabaseImage = "metabase/metabase:v0.46.6.1"
/**/
metabaseListenAddress = "127.0.0.1"
metabaseListenPort = "3000"
@ -39,21 +35,12 @@ var (
forceYes bool
// information needed to set up a random password on user's behalf
/*informations needed to setup a random password on user's behalf*/
)
type cliDashboard struct {
cfg configGetter
}
func NewCLIDashboard(cfg configGetter) *cliDashboard {
return &cliDashboard{
cfg: cfg,
}
}
func (cli *cliDashboard) NewCommand() *cobra.Command {
cmd := &cobra.Command{
func NewDashboardCmd() *cobra.Command {
/* ---- UPDATE COMMAND */
var cmdDashboard = &cobra.Command{
Use: "dashboard [command]",
Short: "Manage your metabase dashboard container [requires local API]",
Long: `Install/Start/Stop/Remove a metabase container exposing dashboard and metrics.
@ -67,24 +54,23 @@ cscli dashboard start
cscli dashboard stop
cscli dashboard remove
`,
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := require.LAPI(cfg); err != nil {
return err
}
PersistentPreRun: func(cmd *cobra.Command, args []string) {
if err := metabase.TestAvailability(); err != nil {
return err
log.Fatalf("%s", err)
}
metabaseConfigFolderPath := filepath.Join(cfg.ConfigPaths.ConfigDir, metabaseConfigFolder)
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
log.Fatal("Local API is disabled, please run this command on the local API machine")
}
metabaseConfigFolderPath := filepath.Join(csConfig.ConfigPaths.ConfigDir, metabaseConfigFolder)
metabaseConfigPath = filepath.Join(metabaseConfigFolderPath, metabaseConfigFile)
if err := os.MkdirAll(metabaseConfigFolderPath, os.ModePerm); err != nil {
return err
log.Fatal(err)
}
if err := require.DB(cfg); err != nil {
return err
if err := csConfig.LoadDBConfig(); err != nil {
log.Errorf("This command requires direct database access (must be run on the local API machine)")
log.Fatal(err)
}
/*
@ -98,24 +84,23 @@ cscli dashboard remove
metabaseContainerID = oldContainerID
}
}
return nil
},
}
cmd.AddCommand(cli.newSetupCmd())
cmd.AddCommand(cli.newStartCmd())
cmd.AddCommand(cli.newStopCmd())
cmd.AddCommand(cli.newShowPasswordCmd())
cmd.AddCommand(cli.newRemoveCmd())
cmdDashboard.AddCommand(NewDashboardSetupCmd())
cmdDashboard.AddCommand(NewDashboardStartCmd())
cmdDashboard.AddCommand(NewDashboardStopCmd())
cmdDashboard.AddCommand(NewDashboardShowPasswordCmd())
cmdDashboard.AddCommand(NewDashboardRemoveCmd())
return cmd
return cmdDashboard
}
func (cli *cliDashboard) newSetupCmd() *cobra.Command {
func NewDashboardSetupCmd() *cobra.Command {
var force bool
cmd := &cobra.Command{
var cmdDashSetup = &cobra.Command{
Use: "setup",
Short: "Setup a metabase container.",
Long: `Perform a metabase docker setup, download standard dashboards, create a fresh user and start the container`,
@ -126,9 +111,9 @@ cscli dashboard setup
cscli dashboard setup --listen 0.0.0.0
cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
`,
RunE: func(_ *cobra.Command, _ []string) error {
if metabaseDBPath == "" {
metabaseDBPath = cli.cfg().ConfigPaths.DataDir
Run: func(cmd *cobra.Command, args []string) {
if metabaseDbPath == "" {
metabaseDbPath = csConfig.ConfigPaths.DataDir
}
if metabasePassword == "" {
@ -138,26 +123,70 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
isValid = passwordIsValid(metabasePassword)
}
}
if err := checkSystemMemory(&forceYes); err != nil {
return err
var answer bool
if valid, err := checkSystemMemory(); err == nil && !valid {
if !forceYes {
prompt := &survey.Confirm{
Message: "Metabase requires 1-2GB of RAM, your system is below this requirement continue ?",
Default: true,
}
if err := survey.AskOne(prompt, &answer); err != nil {
log.Warnf("unable to ask about RAM check: %s", err)
}
if !answer {
log.Fatal("Unable to continue due to RAM requirement")
}
} else {
log.Warnf("Metabase requires 1-2GB of RAM, your system is below this requirement")
}
}
warnIfNotLoopback(metabaseListenAddress)
if err := disclaimer(&forceYes); err != nil {
return err
groupExist := false
dockerGroup, err := user.LookupGroup(crowdsecGroup)
if err == nil {
groupExist = true
}
dockerGroup, err := checkGroups(&forceYes)
if !forceYes && !groupExist {
prompt := &survey.Confirm{
Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup),
Default: true,
}
if err := survey.AskOne(prompt, &answer); err != nil {
log.Fatalf("unable to ask to force: %s", err)
}
}
if !answer && !forceYes && !groupExist {
log.Fatalf("unable to continue without creating '%s' group", crowdsecGroup)
}
if !groupExist {
groupAddCmd, err := exec.LookPath("groupadd")
if err != nil {
log.Fatalf("unable to find 'groupadd' command, can't continue")
}
groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}}
if err := groupAdd.Run(); err != nil {
log.Fatalf("unable to add group '%s': %s", dockerGroup, err)
}
dockerGroup, err = user.LookupGroup(crowdsecGroup)
if err != nil {
log.Fatalf("unable to lookup '%s' group: %+v", dockerGroup, err)
}
}
intID, err := strconv.Atoi(dockerGroup.Gid)
if err != nil {
return err
log.Fatalf("unable to convert group ID to int: %s", err)
}
if err = cli.chownDatabase(dockerGroup.Gid); err != nil {
return err
if err := os.Chown(csConfig.DbConfig.DbPath, 0, intID); err != nil {
log.Fatalf("unable to chown sqlite db file '%s': %s", csConfig.DbConfig.DbPath, err)
}
mb, err := metabase.SetupMetabase(cli.cfg().API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDBPath, dockerGroup.Gid, metabaseContainerID, metabaseImage)
mb, err := metabase.SetupMetabase(csConfig.API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDbPath, dockerGroup.Gid, metabaseContainerID)
if err != nil {
return err
log.Fatal(err)
}
if err := mb.DumpConfig(metabaseConfigPath); err != nil {
return err
log.Fatal(err)
}
log.Infof("Metabase is ready")
@ -165,96 +194,79 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
fmt.Printf("\tURL : '%s'\n", mb.Config.ListenURL)
fmt.Printf("\tusername : '%s'\n", mb.Config.Username)
fmt.Printf("\tpassword : '%s'\n", mb.Config.Password)
return nil
},
}
cmdDashSetup.Flags().BoolVarP(&force, "force", "f", false, "Force setup : override existing files.")
cmdDashSetup.Flags().StringVarP(&metabaseDbPath, "dir", "d", "", "Shared directory with metabase container.")
cmdDashSetup.Flags().StringVarP(&metabaseListenAddress, "listen", "l", metabaseListenAddress, "Listen address of container")
cmdDashSetup.Flags().StringVarP(&metabaseListenPort, "port", "p", metabaseListenPort, "Listen port of container")
cmdDashSetup.Flags().BoolVarP(&forceYes, "yes", "y", false, "force yes")
//cmdDashSetup.Flags().StringVarP(&metabaseUser, "user", "u", "crowdsec@crowdsec.net", "metabase user")
cmdDashSetup.Flags().StringVar(&metabasePassword, "password", "", "metabase password")
flags := cmd.Flags()
flags.BoolVarP(&force, "force", "f", false, "Force setup : override existing files")
flags.StringVarP(&metabaseDBPath, "dir", "d", "", "Shared directory with metabase container")
flags.StringVarP(&metabaseListenAddress, "listen", "l", metabaseListenAddress, "Listen address of container")
flags.StringVar(&metabaseImage, "metabase-image", metabaseImage, "Metabase image to use")
flags.StringVarP(&metabaseListenPort, "port", "p", metabaseListenPort, "Listen port of container")
flags.BoolVarP(&forceYes, "yes", "y", false, "force yes")
// flags.StringVarP(&metabaseUser, "user", "u", "crowdsec@crowdsec.net", "metabase user")
flags.StringVar(&metabasePassword, "password", "", "metabase password")
return cmd
return cmdDashSetup
}
func (cli *cliDashboard) newStartCmd() *cobra.Command {
cmd := &cobra.Command{
func NewDashboardStartCmd() *cobra.Command {
var cmdDashStart = &cobra.Command{
Use: "start",
Short: "Start the metabase container.",
Long: `Stats the metabase container using docker.`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
Run: func(cmd *cobra.Command, args []string) {
mb, err := metabase.NewMetabase(metabaseConfigPath, metabaseContainerID)
if err != nil {
return err
}
warnIfNotLoopback(mb.Config.ListenAddr)
if err := disclaimer(&forceYes); err != nil {
return err
log.Fatal(err)
}
if err := mb.Container.Start(); err != nil {
return fmt.Errorf("failed to start metabase container : %s", err)
log.Fatalf("Failed to start metabase container : %s", err)
}
log.Infof("Started metabase")
log.Infof("url : http://%s:%s", mb.Config.ListenAddr, mb.Config.ListenPort)
return nil
log.Infof("url : http://%s:%s", metabaseListenAddress, metabaseListenPort)
},
}
cmd.Flags().BoolVarP(&forceYes, "yes", "y", false, "force yes")
return cmd
return cmdDashStart
}
func (cli *cliDashboard) newStopCmd() *cobra.Command {
cmd := &cobra.Command{
func NewDashboardStopCmd() *cobra.Command {
var cmdDashStop = &cobra.Command{
Use: "stop",
Short: "Stops the metabase container.",
Long: `Stops the metabase container using docker.`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
Run: func(cmd *cobra.Command, args []string) {
if err := metabase.StopContainer(metabaseContainerID); err != nil {
return fmt.Errorf("unable to stop container '%s': %s", metabaseContainerID, err)
log.Fatalf("unable to stop container '%s': %s", metabaseContainerID, err)
}
return nil
},
}
return cmd
return cmdDashStop
}
func (cli *cliDashboard) newShowPasswordCmd() *cobra.Command {
cmd := &cobra.Command{Use: "show-password",
func NewDashboardShowPasswordCmd() *cobra.Command {
var cmdDashShowPassword = &cobra.Command{Use: "show-password",
Short: "displays password of metabase.",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
Run: func(cmd *cobra.Command, args []string) {
m := metabase.Metabase{}
if err := m.LoadConfig(metabaseConfigPath); err != nil {
return err
log.Fatal(err)
}
log.Printf("'%s'", m.Config.Password)
return nil
},
}
return cmd
return cmdDashShowPassword
}
func (cli *cliDashboard) newRemoveCmd() *cobra.Command {
func NewDashboardRemoveCmd() *cobra.Command {
var force bool
cmd := &cobra.Command{
var cmdDashRemove = &cobra.Command{
Use: "remove",
Short: "removes the metabase container.",
Long: `removes the metabase container using docker.`,
@ -264,77 +276,66 @@ func (cli *cliDashboard) newRemoveCmd() *cobra.Command {
cscli dashboard remove
cscli dashboard remove --force
`,
RunE: func(_ *cobra.Command, _ []string) error {
Run: func(cmd *cobra.Command, args []string) {
answer := true
if !forceYes {
var answer bool
prompt := &survey.Confirm{
Message: "Do you really want to remove crowdsec dashboard? (all your changes will be lost)",
Default: true,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask to force: %s", err)
}
if !answer {
return fmt.Errorf("user stated no to continue")
log.Fatalf("unable to ask to force: %s", err)
}
}
if metabase.IsContainerExist(metabaseContainerID) {
log.Debugf("Stopping container %s", metabaseContainerID)
if err := metabase.StopContainer(metabaseContainerID); err != nil {
log.Warningf("unable to stop container '%s': %s", metabaseContainerID, err)
}
dockerGroup, err := user.LookupGroup(crowdsecGroup)
if err == nil { // if group exist, remove it
groupDelCmd, err := exec.LookPath("groupdel")
if err != nil {
return fmt.Errorf("unable to find 'groupdel' command, can't continue")
if answer {
if metabase.IsContainerExist(metabaseContainerID) {
log.Debugf("Stopping container %s", metabaseContainerID)
if err := metabase.StopContainer(metabaseContainerID); err != nil {
log.Warningf("unable to stop container '%s': %s", metabaseContainerID, err)
}
dockerGroup, err := user.LookupGroup(crowdsecGroup)
if err == nil { // if group exist, remove it
groupDelCmd, err := exec.LookPath("groupdel")
if err != nil {
log.Fatalf("unable to find 'groupdel' command, can't continue")
}
groupDel := &exec.Cmd{Path: groupDelCmd, Args: []string{groupDelCmd, crowdsecGroup}}
if err := groupDel.Run(); err != nil {
log.Warnf("unable to delete group '%s': %s", dockerGroup, err)
groupDel := &exec.Cmd{Path: groupDelCmd, Args: []string{groupDelCmd, crowdsecGroup}}
if err := groupDel.Run(); err != nil {
log.Errorf("unable to delete group '%s': %s", dockerGroup, err)
}
}
log.Debugf("Removing container %s", metabaseContainerID)
if err := metabase.RemoveContainer(metabaseContainerID); err != nil {
log.Warningf("unable to remove container '%s': %s", metabaseContainerID, err)
}
log.Infof("container %s stopped & removed", metabaseContainerID)
}
log.Debugf("Removing container %s", metabaseContainerID)
if err := metabase.RemoveContainer(metabaseContainerID); err != nil {
log.Warnf("unable to remove container '%s': %s", metabaseContainerID, err)
log.Debugf("Removing metabase db %s", csConfig.ConfigPaths.DataDir)
if err := metabase.RemoveDatabase(csConfig.ConfigPaths.DataDir); err != nil {
log.Warningf("failed to remove metabase internal db : %s", err)
}
log.Infof("container %s stopped & removed", metabaseContainerID)
}
log.Debugf("Removing metabase db %s", cli.cfg().ConfigPaths.DataDir)
if err := metabase.RemoveDatabase(cli.cfg().ConfigPaths.DataDir); err != nil {
log.Warnf("failed to remove metabase internal db : %s", err)
}
if force {
m := metabase.Metabase{}
if err := m.LoadConfig(metabaseConfigPath); err != nil {
return err
}
if err := metabase.RemoveImageContainer(m.Config.Image); err != nil {
if !strings.Contains(err.Error(), "No such image") {
return fmt.Errorf("removing docker image: %s", err)
if force {
if err := metabase.RemoveImageContainer(); err != nil {
if !strings.Contains(err.Error(), "No such image") {
log.Fatalf("removing docker image: %s", err)
}
}
}
}
return nil
},
}
cmdDashRemove.Flags().BoolVarP(&force, "force", "f", false, "Remove also the metabase image")
cmdDashRemove.Flags().BoolVarP(&forceYes, "yes", "y", false, "force yes")
flags := cmd.Flags()
flags.BoolVarP(&force, "force", "f", false, "Remove also the metabase image")
flags.BoolVarP(&forceYes, "yes", "y", false, "force yes")
return cmd
return cmdDashRemove
}
func passwordIsValid(password string) bool {
hasDigit := false
for _, j := range password {
if unicode.IsDigit(j) {
hasDigit = true
break
}
}
@ -342,134 +343,17 @@ func passwordIsValid(password string) bool {
if !hasDigit || len(password) < 6 {
return false
}
return true
}
func checkSystemMemory(forceYes *bool) error {
func checkSystemMemory() (bool, error) {
totMem := memory.TotalMemory()
if totMem >= uint64(math.Pow(2, 30)) {
return nil
if totMem == 0 {
return true, errors.New("Unable to get system total memory")
}
if !*forceYes {
var answer bool
prompt := &survey.Confirm{
Message: "Metabase requires 1-2GB of RAM, your system is below this requirement continue ?",
Default: true,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask about RAM check: %s", err)
}
if !answer {
return fmt.Errorf("user stated no to continue")
}
return nil
if uint64(math.Pow(2, 30)) >= totMem {
return false, nil
}
log.Warn("Metabase requires 1-2GB of RAM, your system is below this requirement")
return nil
}
func warnIfNotLoopback(addr string) {
if addr == "127.0.0.1" || addr == "::1" {
return
}
log.Warnf("You are potentially exposing your metabase port to the internet (addr: %s), please consider using a reverse proxy", addr)
}
func disclaimer(forceYes *bool) error {
if !*forceYes {
var answer bool
prompt := &survey.Confirm{
Message: "CrowdSec takes no responsibility for the security of your metabase instance. Do you accept these responsibilities ?",
Default: true,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask to question: %s", err)
}
if !answer {
return fmt.Errorf("user stated no to responsibilities")
}
return nil
}
log.Warn("CrowdSec takes no responsibility for the security of your metabase instance. You used force yes, so you accept this disclaimer")
return nil
}
func checkGroups(forceYes *bool) (*user.Group, error) {
dockerGroup, err := user.LookupGroup(crowdsecGroup)
if err == nil {
return dockerGroup, nil
}
if !*forceYes {
var answer bool
prompt := &survey.Confirm{
Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup),
Default: true,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return dockerGroup, fmt.Errorf("unable to ask to question: %s", err)
}
if !answer {
return dockerGroup, fmt.Errorf("unable to continue without creating '%s' group", crowdsecGroup)
}
}
groupAddCmd, err := exec.LookPath("groupadd")
if err != nil {
return dockerGroup, fmt.Errorf("unable to find 'groupadd' command, can't continue")
}
groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}}
if err := groupAdd.Run(); err != nil {
return dockerGroup, fmt.Errorf("unable to add group '%s': %s", dockerGroup, err)
}
return user.LookupGroup(crowdsecGroup)
}
func (cli *cliDashboard) chownDatabase(gid string) error {
cfg := cli.cfg()
intID, err := strconv.Atoi(gid)
if err != nil {
return fmt.Errorf("unable to convert group ID to int: %s", err)
}
if stat, err := os.Stat(cfg.DbConfig.DbPath); !os.IsNotExist(err) {
info := stat.Sys()
if err := os.Chown(cfg.DbConfig.DbPath, int(info.(*syscall.Stat_t).Uid), intID); err != nil {
return fmt.Errorf("unable to chown sqlite db file '%s': %s", cfg.DbConfig.DbPath, err)
}
}
if cfg.DbConfig.Type == "sqlite" && cfg.DbConfig.UseWal != nil && *cfg.DbConfig.UseWal {
for _, ext := range []string{"-wal", "-shm"} {
file := cfg.DbConfig.DbPath + ext
if stat, err := os.Stat(file); !os.IsNotExist(err) {
info := stat.Sys()
if err := os.Chown(file, int(info.(*syscall.Stat_t).Uid), intID); err != nil {
return fmt.Errorf("unable to chown sqlite db file '%s': %s", file, err)
}
}
}
}
return nil
return true, nil
}

View file

@ -1,32 +0,0 @@
//go:build !linux
package main
import (
"runtime"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
type cliDashboard struct{
cfg configGetter
}
func NewCLIDashboard(cfg configGetter) *cliDashboard {
return &cliDashboard{
cfg: cfg,
}
}
func (cli cliDashboard) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "dashboard",
DisableAutoGenTag: true,
Run: func(_ *cobra.Command, _ []string) {
log.Infof("Dashboard command is disabled on %s", runtime.GOOS)
},
}
return cmd
}

View file

@ -4,29 +4,30 @@ import (
"context"
"encoding/csv"
"encoding/json"
"errors"
"fmt"
"net/url"
"os"
"path/filepath"
"strconv"
"strings"
"time"
"github.com/fatih/color"
"github.com/go-openapi/strfmt"
"github.com/jszwec/csvutil"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
var Client *apiclient.ApiClient
func (cli *cliDecisions) decisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
/*here we cheat a bit : to make it more readable for the user, we dedup some entries*/
spamLimit := make(map[string]bool)
skipped := 0
@ -34,36 +35,27 @@ func (cli *cliDecisions) decisionsToTable(alerts *models.GetAlertsResponse, prin
for aIdx := 0; aIdx < len(*alerts); aIdx++ {
alertItem := (*alerts)[aIdx]
newDecisions := make([]*models.Decision, 0)
for _, decisionItem := range alertItem.Decisions {
spamKey := fmt.Sprintf("%t:%s:%s:%s", *decisionItem.Simulated, *decisionItem.Type, *decisionItem.Scope, *decisionItem.Value)
if _, ok := spamLimit[spamKey]; ok {
skipped++
continue
}
spamLimit[spamKey] = true
newDecisions = append(newDecisions, decisionItem)
}
alertItem.Decisions = newDecisions
}
switch cli.cfg().Cscli.Output {
case "raw":
if csConfig.Cscli.Output == "raw" {
csvwriter := csv.NewWriter(os.Stdout)
header := []string{"id", "source", "ip", "reason", "action", "country", "as", "events_count", "expiration", "simulated", "alert_id"}
if printMachine {
header = append(header, "machine")
}
err := csvwriter.Write(header)
if err != nil {
return err
}
for _, alertItem := range *alerts {
for _, decisionItem := range alertItem.Decisions {
raw := []string{
@ -89,46 +81,25 @@ func (cli *cliDecisions) decisionsToTable(alerts *models.GetAlertsResponse, prin
}
}
}
csvwriter.Flush()
case "json":
if *alerts == nil {
// avoid returning "null" in `json"
// could be cleaner if we used slice of alerts directly
fmt.Println("[]")
return nil
}
} else if csConfig.Cscli.Output == "json" {
x, _ := json.MarshalIndent(alerts, "", " ")
fmt.Printf("%s", string(x))
case "human":
} else if csConfig.Cscli.Output == "human" {
if len(*alerts) == 0 {
fmt.Println("No active decisions")
return nil
}
cli.decisionsTable(color.Output, alerts, printMachine)
decisionsTable(color.Output, alerts, printMachine)
if skipped > 0 {
fmt.Printf("%d duplicated entries skipped\n", skipped)
}
}
return nil
}
type cliDecisions struct {
cfg configGetter
}
func NewCLIDecisions(cfg configGetter) *cliDecisions {
return &cliDecisions{
cfg: cfg,
}
}
func (cli *cliDecisions) NewCommand() *cobra.Command {
cmd := &cobra.Command{
func NewDecisionsCmd() *cobra.Command {
var cmdDecisions = &cobra.Command{
Use: "decisions [action]",
Short: "Manage decisions",
Long: `Add/List/Delete/Import decisions from LAPI`,
@ -137,40 +108,38 @@ func (cli *cliDecisions) NewCommand() *cobra.Command {
/*TBD example*/
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := cfg.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIClient(); err != nil {
return errors.Wrap(err, "loading api client")
}
password := strfmt.Password(cfg.API.Client.Credentials.Password)
apiurl, err := url.Parse(cfg.API.Client.Credentials.URL)
password := strfmt.Password(csConfig.API.Client.Credentials.Password)
apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL)
if err != nil {
return fmt.Errorf("parsing api url %s: %w", cfg.API.Client.Credentials.URL, err)
return errors.Wrapf(err, "parsing api url %s", csConfig.API.Client.Credentials.URL)
}
Client, err = apiclient.NewClient(&apiclient.Config{
MachineID: cfg.API.Client.Credentials.Login,
MachineID: csConfig.API.Client.Credentials.Login,
Password: password,
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
URL: apiurl,
VersionPrefix: "v1",
})
if err != nil {
return fmt.Errorf("creating api client: %w", err)
return errors.Wrap(err, "creating api client")
}
return nil
},
}
cmd.AddCommand(cli.newListCmd())
cmd.AddCommand(cli.newAddCmd())
cmd.AddCommand(cli.newDeleteCmd())
cmd.AddCommand(cli.newImportCmd())
cmdDecisions.AddCommand(NewDecisionsListCmd())
cmdDecisions.AddCommand(NewDecisionsAddCmd())
cmdDecisions.AddCommand(NewDecisionsDeleteCmd())
cmdDecisions.AddCommand(NewDecisionsImportCmd())
return cmd
return cmdDecisions
}
func (cli *cliDecisions) newListCmd() *cobra.Command {
func NewDecisionsListCmd() *cobra.Command {
var filter = apiclient.AlertsListOpts{
ValueEquals: new(string),
ScopeEquals: new(string),
@ -184,27 +153,25 @@ func (cli *cliDecisions) newListCmd() *cobra.Command {
IncludeCAPI: new(bool),
Limit: new(int),
}
NoSimu := new(bool)
contained := new(bool)
var printMachine bool
cmd := &cobra.Command{
var cmdDecisionsList = &cobra.Command{
Use: "list [options]",
Short: "List decisions from LAPI",
Example: `cscli decisions list -i 1.2.3.4
cscli decisions list -r 1.2.3.0/24
cscli decisions list -s crowdsecurity/ssh-bf
cscli decisions list --origin lists --scenario list_name
cscli decisions list -t ban
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, _ []string) error {
Run: func(cmd *cobra.Command, args []string) {
var err error
/*take care of shorthand options*/
if err = manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil {
return err
if err := manageCliDecisionAlerts(filter.IPEquals, filter.RangeEquals, filter.ScopeEquals, filter.ValueEquals); err != nil {
log.Fatalf("%s", err)
}
filter.ActiveDecisionEquals = new(bool)
*filter.ActiveDecisionEquals = true
@ -220,7 +187,7 @@ cscli decisions list --origin lists --scenario list_name
days, err := strconv.Atoi(realDuration)
if err != nil {
printHelp(cmd)
return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until)
log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until)
}
*filter.Until = fmt.Sprintf("%d%s", days*24, "h")
}
@ -233,7 +200,7 @@ cscli decisions list --origin lists --scenario list_name
days, err := strconv.Atoi(realDuration)
if err != nil {
printHelp(cmd)
return fmt.Errorf("can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Since)
log.Fatalf("Can't parse duration %s, valid durations format: 1d, 4h, 4h15m", *filter.Until)
}
*filter.Since = fmt.Sprintf("%d%s", days*24, "h")
}
@ -269,37 +236,35 @@ cscli decisions list --origin lists --scenario list_name
alerts, _, err := Client.Alerts.List(context.Background(), filter)
if err != nil {
return fmt.Errorf("unable to retrieve decisions: %w", err)
log.Fatalf("Unable to list decisions : %v", err)
}
err = cli.decisionsToTable(alerts, printMachine)
err = DecisionsToTable(alerts, printMachine)
if err != nil {
return fmt.Errorf("unable to print decisions: %w", err)
log.Fatalf("unable to list decisions : %v", err)
}
return nil
},
}
cmd.Flags().SortFlags = false
cmd.Flags().BoolVarP(filter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
cmd.Flags().StringVar(filter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
cmd.Flags().StringVar(filter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
cmd.Flags().StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)")
cmd.Flags().StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)")
cmd.Flags().StringVar(filter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmd.Flags().StringVarP(filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)")
cmd.Flags().StringVarP(filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)")
cmd.Flags().StringVarP(filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
cmd.Flags().StringVarP(filter.RangeEquals, "range", "r", "", "restrict to alerts from this source range (shorthand for --scope range --value <RANGE>)")
cmd.Flags().IntVarP(filter.Limit, "limit", "l", 100, "number of alerts to get (use 0 to remove the limit)")
cmd.Flags().BoolVar(NoSimu, "no-simu", false, "exclude decisions in simulation mode")
cmd.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that triggered decisions")
cmd.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
cmdDecisionsList.Flags().SortFlags = false
cmdDecisionsList.Flags().BoolVarP(filter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
cmdDecisionsList.Flags().StringVar(filter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
cmdDecisionsList.Flags().StringVar(filter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
cmdDecisionsList.Flags().StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)")
cmdDecisionsList.Flags().StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)")
cmdDecisionsList.Flags().StringVar(filter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmdDecisionsList.Flags().StringVarP(filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)")
cmdDecisionsList.Flags().StringVarP(filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)")
cmdDecisionsList.Flags().StringVarP(filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
cmdDecisionsList.Flags().StringVarP(filter.RangeEquals, "range", "r", "", "restrict to alerts from this source range (shorthand for --scope range --value <RANGE>)")
cmdDecisionsList.Flags().IntVarP(filter.Limit, "limit", "l", 100, "number of alerts to get (use 0 to remove the limit)")
cmdDecisionsList.Flags().BoolVar(NoSimu, "no-simu", false, "exclude decisions in simulation mode")
cmdDecisionsList.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that triggered decisions")
cmdDecisionsList.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
return cmd
return cmdDecisionsList
}
func (cli *cliDecisions) newAddCmd() *cobra.Command {
func NewDecisionsAddCmd() *cobra.Command {
var (
addIP string
addRange string
@ -310,7 +275,7 @@ func (cli *cliDecisions) newAddCmd() *cobra.Command {
addType string
)
cmd := &cobra.Command{
var cmdDecisionsAdd = &cobra.Command{
Use: "add [options]",
Short: "Add decision to LAPI",
Example: `cscli decisions add --ip 1.2.3.4
@ -321,7 +286,7 @@ cscli decisions add --scope username --value foobar
/*TBD : fix long and example*/
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, _ []string) error {
Run: func(cmd *cobra.Command, args []string) {
var err error
alerts := models.AddAlertsRequest{}
origin := types.CscliOrigin
@ -335,8 +300,8 @@ cscli decisions add --scope username --value foobar
createdAt := time.Now().UTC().Format(time.RFC3339)
/*take care of shorthand options*/
if err = manageCliDecisionAlerts(&addIP, &addRange, &addScope, &addValue); err != nil {
return err
if err := manageCliDecisionAlerts(&addIP, &addRange, &addScope, &addValue); err != nil {
log.Fatalf("%s", err)
}
if addIP != "" {
@ -347,11 +312,11 @@ cscli decisions add --scope username --value foobar
addScope = types.Range
} else if addValue == "" {
printHelp(cmd)
return errors.New("missing arguments, a value is required (--ip, --range or --scope and --value)")
log.Fatalf("Missing arguments, a value is required (--ip, --range or --scope and --value)")
}
if addReason == "" {
addReason = fmt.Sprintf("manual '%s' from '%s'", addType, cli.cfg().API.Client.Credentials.Login)
addReason = fmt.Sprintf("manual '%s' from '%s'", addType, csConfig.API.Client.Credentials.Login)
}
decision := models.Decision{
Duration: &addDuration,
@ -372,7 +337,7 @@ cscli decisions add --scope username --value foobar
Scenario: &addReason,
ScenarioVersion: &empty,
Simulated: &simulated,
// setting empty scope/value broke plugins, and it didn't seem to be needed anymore w/ latest papi changes
//setting empty scope/value broke plugins, and it didn't seem to be needed anymore w/ latest papi changes
Source: &models.Source{
AsName: empty,
AsNumber: empty,
@ -390,29 +355,27 @@ cscli decisions add --scope username --value foobar
_, _, err = Client.Alerts.Add(context.Background(), alerts)
if err != nil {
return err
log.Fatal(err)
}
log.Info("Decision successfully added")
return nil
},
}
cmd.Flags().SortFlags = false
cmd.Flags().StringVarP(&addIP, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmd.Flags().StringVarP(&addRange, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmd.Flags().StringVarP(&addDuration, "duration", "d", "4h", "Decision duration (ie. 1h,4h,30m)")
cmd.Flags().StringVarP(&addValue, "value", "v", "", "The value (ie. --scope username --value foobar)")
cmd.Flags().StringVar(&addScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)")
cmd.Flags().StringVarP(&addReason, "reason", "R", "", "Decision reason (ie. scenario-name)")
cmd.Flags().StringVarP(&addType, "type", "t", "ban", "Decision type (ie. ban,captcha,throttle)")
cmdDecisionsAdd.Flags().SortFlags = false
cmdDecisionsAdd.Flags().StringVarP(&addIP, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmdDecisionsAdd.Flags().StringVarP(&addRange, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmdDecisionsAdd.Flags().StringVarP(&addDuration, "duration", "d", "4h", "Decision duration (ie. 1h,4h,30m)")
cmdDecisionsAdd.Flags().StringVarP(&addValue, "value", "v", "", "The value (ie. --scope username --value foobar)")
cmdDecisionsAdd.Flags().StringVar(&addScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)")
cmdDecisionsAdd.Flags().StringVarP(&addReason, "reason", "R", "", "Decision reason (ie. scenario-name)")
cmdDecisionsAdd.Flags().StringVarP(&addType, "type", "t", "ban", "Decision type (ie. ban,captcha,throttle)")
return cmd
return cmdDecisionsAdd
}
func (cli *cliDecisions) newDeleteCmd() *cobra.Command {
delFilter := apiclient.DecisionsDeleteOpts{
func NewDecisionsDeleteCmd() *cobra.Command {
var delFilter = apiclient.DecisionsDeleteOpts{
ScopeEquals: new(string),
ValueEquals: new(string),
TypeEquals: new(string),
@ -421,14 +384,11 @@ func (cli *cliDecisions) newDeleteCmd() *cobra.Command {
ScenarioEquals: new(string),
OriginEquals: new(string),
}
var delDecisionID string
var delDecisionId string
var delDecisionAll bool
contained := new(bool)
cmd := &cobra.Command{
var cmdDecisionsDelete = &cobra.Command{
Use: "delete [options]",
Short: "Delete decisions",
DisableAutoGenTag: true,
@ -437,30 +397,27 @@ func (cli *cliDecisions) newDeleteCmd() *cobra.Command {
cscli decisions delete -i 1.2.3.4
cscli decisions delete --id 42
cscli decisions delete --type captcha
cscli decisions delete --origin lists --scenario list_name
`,
/*TBD : refaire le Long/Example*/
PreRunE: func(cmd *cobra.Command, _ []string) error {
PreRun: func(cmd *cobra.Command, args []string) {
if delDecisionAll {
return nil
return
}
if *delFilter.ScopeEquals == "" && *delFilter.ValueEquals == "" &&
*delFilter.TypeEquals == "" && *delFilter.IPEquals == "" &&
*delFilter.RangeEquals == "" && *delFilter.ScenarioEquals == "" &&
*delFilter.OriginEquals == "" && delDecisionID == "" {
*delFilter.OriginEquals == "" && delDecisionId == "" {
cmd.Usage()
return errors.New("at least one filter or --all must be specified")
log.Fatalln("At least one filter or --all must be specified")
}
return nil
},
RunE: func(_ *cobra.Command, _ []string) error {
Run: func(cmd *cobra.Command, args []string) {
var err error
var decisions *models.DeleteDecisionResponse
/*take care of shorthand options*/
if err = manageCliDecisionAlerts(delFilter.IPEquals, delFilter.RangeEquals, delFilter.ScopeEquals, delFilter.ValueEquals); err != nil {
return err
if err := manageCliDecisionAlerts(delFilter.IPEquals, delFilter.RangeEquals, delFilter.ScopeEquals, delFilter.ValueEquals); err != nil {
log.Fatalf("%s", err)
}
if *delFilter.ScopeEquals == "" {
delFilter.ScopeEquals = nil
@ -487,37 +444,190 @@ cscli decisions delete --origin lists --scenario list_name
delFilter.Contains = new(bool)
}
if delDecisionID == "" {
if delDecisionId == "" {
decisions, _, err = Client.Decisions.Delete(context.Background(), delFilter)
if err != nil {
return fmt.Errorf("unable to delete decisions: %v", err)
log.Fatalf("Unable to delete decisions : %v", err)
}
} else {
if _, err = strconv.Atoi(delDecisionID); err != nil {
return fmt.Errorf("id '%s' is not an integer: %v", delDecisionID, err)
if _, err = strconv.Atoi(delDecisionId); err != nil {
log.Fatalf("id '%s' is not an integer: %v", delDecisionId, err)
}
decisions, _, err = Client.Decisions.DeleteOne(context.Background(), delDecisionID)
decisions, _, err = Client.Decisions.DeleteOne(context.Background(), delDecisionId)
if err != nil {
return fmt.Errorf("unable to delete decision: %v", err)
log.Fatalf("Unable to delete decision : %v", err)
}
}
log.Infof("%s decision(s) deleted", decisions.NbDeleted)
return nil
},
}
cmd.Flags().SortFlags = false
cmd.Flags().StringVarP(delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmd.Flags().StringVarP(delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmd.Flags().StringVarP(delFilter.TypeEquals, "type", "t", "", "the decision type (ie. ban,captcha)")
cmd.Flags().StringVarP(delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmd.Flags().StringVarP(delFilter.ScenarioEquals, "scenario", "s", "", "the scenario name (ie. crowdsecurity/ssh-bf)")
cmd.Flags().StringVar(delFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmdDecisionsDelete.Flags().SortFlags = false
cmdDecisionsDelete.Flags().StringVarP(delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmdDecisionsDelete.Flags().StringVarP(delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmdDecisionsDelete.Flags().StringVarP(delFilter.TypeEquals, "type", "t", "", "the decision type (ie. ban,captcha)")
cmdDecisionsDelete.Flags().StringVarP(delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmdDecisionsDelete.Flags().StringVarP(delFilter.ScenarioEquals, "scenario", "s", "", "the scenario name (ie. crowdsecurity/ssh-bf)")
cmdDecisionsDelete.Flags().StringVar(delFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmd.Flags().StringVar(&delDecisionID, "id", "", "decision id")
cmd.Flags().BoolVar(&delDecisionAll, "all", false, "delete all decisions")
cmd.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
cmdDecisionsDelete.Flags().StringVar(&delDecisionId, "id", "", "decision id")
cmdDecisionsDelete.Flags().BoolVar(&delDecisionAll, "all", false, "delete all decisions")
cmdDecisionsDelete.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
return cmd
return cmdDecisionsDelete
}
func NewDecisionsImportCmd() *cobra.Command {
var (
defaultDuration = "4h"
defaultScope = "ip"
defaultType = "ban"
defaultReason = "manual"
importDuration string
importScope string
importReason string
importType string
importFile string
)
var cmdDecisionImport = &cobra.Command{
Use: "import [options]",
Short: "Import decisions from json or csv file",
Long: "expected format :\n" +
"csv : any of duration,origin,reason,scope,type,value, with a header line\n" +
`json : {"duration" : "24h", "origin" : "my-list", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`,
DisableAutoGenTag: true,
Example: `decisions.csv :
duration,scope,value
24h,ip,1.2.3.4
cscsli decisions import -i decisions.csv
decisions.json :
[{"duration" : "4h", "scope" : "ip", "type" : "ban", "value" : "1.2.3.4"}]
`,
Run: func(cmd *cobra.Command, args []string) {
if importFile == "" {
log.Fatalf("Please provide a input file containing decisions with -i flag")
}
csvData, err := os.ReadFile(importFile)
if err != nil {
log.Fatalf("unable to open '%s': %s", importFile, err)
}
type decisionRaw struct {
Duration string `csv:"duration,omitempty" json:"duration,omitempty"`
Origin string `csv:"origin,omitempty" json:"origin,omitempty"`
Scenario string `csv:"reason,omitempty" json:"reason,omitempty"`
Scope string `csv:"scope,omitempty" json:"scope,omitempty"`
Type string `csv:"type,omitempty" json:"type,omitempty"`
Value string `csv:"value" json:"value"`
}
var decisionsListRaw []decisionRaw
switch fileFormat := filepath.Ext(importFile); fileFormat {
case ".json":
if err := json.Unmarshal(csvData, &decisionsListRaw); err != nil {
log.Fatalf("unable to unmarshall json: '%s'", err)
}
case ".csv":
if err := csvutil.Unmarshal(csvData, &decisionsListRaw); err != nil {
log.Fatalf("unable to unmarshall csv: '%s'", err)
}
default:
log.Fatalf("file format not supported for '%s'. supported format are 'json' and 'csv'", importFile)
}
decisionsList := make([]*models.Decision, 0)
for i, decisionLine := range decisionsListRaw {
line := i + 2
if decisionLine.Value == "" {
log.Fatalf("please provide a 'value' in your csv line %d", line)
}
/*deal with defaults and cli-override*/
if decisionLine.Duration == "" {
decisionLine.Duration = defaultDuration
log.Debugf("No 'duration' line %d, using default value: '%s'", line, defaultDuration)
}
if importDuration != "" {
decisionLine.Duration = importDuration
log.Debugf("'duration' line %d, using supplied value: '%s'", line, importDuration)
}
decisionLine.Origin = types.CscliImportOrigin
if decisionLine.Scenario == "" {
decisionLine.Scenario = defaultReason
log.Debugf("No 'reason' line %d, using value: '%s'", line, decisionLine.Scenario)
}
if importReason != "" {
decisionLine.Scenario = importReason
log.Debugf("No 'reason' line %d, using supplied value: '%s'", line, importReason)
}
if decisionLine.Type == "" {
decisionLine.Type = defaultType
log.Debugf("No 'type' line %d, using default value: '%s'", line, decisionLine.Type)
}
if importType != "" {
decisionLine.Type = importType
log.Debugf("'type' line %d, using supplied value: '%s'", line, importType)
}
if decisionLine.Scope == "" {
decisionLine.Scope = defaultScope
log.Debugf("No 'scope' line %d, using default value: '%s'", line, decisionLine.Scope)
}
if importScope != "" {
decisionLine.Scope = importScope
log.Debugf("'scope' line %d, using supplied value: '%s'", line, importScope)
}
decision := models.Decision{
Value: types.StrPtr(decisionLine.Value),
Duration: types.StrPtr(decisionLine.Duration),
Origin: types.StrPtr(decisionLine.Origin),
Scenario: types.StrPtr(decisionLine.Scenario),
Type: types.StrPtr(decisionLine.Type),
Scope: types.StrPtr(decisionLine.Scope),
Simulated: new(bool),
}
decisionsList = append(decisionsList, &decision)
}
alerts := models.AddAlertsRequest{}
importAlert := models.Alert{
CreatedAt: time.Now().UTC().Format(time.RFC3339),
Scenario: types.StrPtr(fmt.Sprintf("import %s : %d IPs", importFile, len(decisionsList))),
Message: types.StrPtr(""),
Events: []*models.Event{},
Source: &models.Source{
Scope: types.StrPtr(""),
Value: types.StrPtr(""),
},
StartAt: types.StrPtr(time.Now().UTC().Format(time.RFC3339)),
StopAt: types.StrPtr(time.Now().UTC().Format(time.RFC3339)),
Capacity: types.Int32Ptr(0),
Simulated: types.BoolPtr(false),
EventsCount: types.Int32Ptr(int32(len(decisionsList))),
Leakspeed: types.StrPtr(""),
ScenarioHash: types.StrPtr(""),
ScenarioVersion: types.StrPtr(""),
Decisions: decisionsList,
}
alerts = append(alerts, &importAlert)
if len(decisionsList) > 1000 {
log.Infof("You are about to add %d decisions, this may take a while", len(decisionsList))
}
_, _, err = Client.Alerts.Add(context.Background(), alerts)
if err != nil {
log.Fatal(err)
}
log.Infof("%d decisions successfully imported", len(decisionsList))
},
}
cmdDecisionImport.Flags().SortFlags = false
cmdDecisionImport.Flags().StringVarP(&importFile, "input", "i", "", "Input file")
cmdDecisionImport.Flags().StringVarP(&importDuration, "duration", "d", "", "Decision duration (ie. 1h,4h,30m)")
cmdDecisionImport.Flags().StringVar(&importScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)")
cmdDecisionImport.Flags().StringVarP(&importReason, "reason", "R", "", "Decision reason (ie. scenario-name)")
cmdDecisionImport.Flags().StringVarP(&importType, "type", "t", "", "Decision type (ie. ban,captcha,throttle)")
return cmdDecisionImport
}

View file

@ -1,280 +0,0 @@
package main
import (
"bufio"
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"strings"
"time"
"github.com/jszwec/csvutil"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/crowdsecurity/go-cs-lib/slicetools"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
// decisionRaw is only used to unmarshall json/csv decisions
type decisionRaw struct {
Duration string `csv:"duration,omitempty" json:"duration,omitempty"`
Scenario string `csv:"reason,omitempty" json:"reason,omitempty"`
Scope string `csv:"scope,omitempty" json:"scope,omitempty"`
Type string `csv:"type,omitempty" json:"type,omitempty"`
Value string `csv:"value" json:"value"`
}
func parseDecisionList(content []byte, format string) ([]decisionRaw, error) {
ret := []decisionRaw{}
switch format {
case "values":
log.Infof("Parsing values")
scanner := bufio.NewScanner(bytes.NewReader(content))
for scanner.Scan() {
value := strings.TrimSpace(scanner.Text())
ret = append(ret, decisionRaw{Value: value})
}
if err := scanner.Err(); err != nil {
return nil, fmt.Errorf("unable to parse values: '%s'", err)
}
case "json":
log.Infof("Parsing json")
if err := json.Unmarshal(content, &ret); err != nil {
return nil, err
}
case "csv":
log.Infof("Parsing csv")
if err := csvutil.Unmarshal(content, &ret); err != nil {
return nil, fmt.Errorf("unable to parse csv: '%s'", err)
}
default:
return nil, fmt.Errorf("invalid format '%s', expected one of 'json', 'csv', 'values'", format)
}
return ret, nil
}
func (cli *cliDecisions) runImport(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
input, err := flags.GetString("input")
if err != nil {
return err
}
defaultDuration, err := flags.GetString("duration")
if err != nil {
return err
}
if defaultDuration == "" {
return errors.New("--duration cannot be empty")
}
defaultScope, err := flags.GetString("scope")
if err != nil {
return err
}
if defaultScope == "" {
return errors.New("--scope cannot be empty")
}
defaultReason, err := flags.GetString("reason")
if err != nil {
return err
}
if defaultReason == "" {
return errors.New("--reason cannot be empty")
}
defaultType, err := flags.GetString("type")
if err != nil {
return err
}
if defaultType == "" {
return errors.New("--type cannot be empty")
}
batchSize, err := flags.GetInt("batch")
if err != nil {
return err
}
format, err := flags.GetString("format")
if err != nil {
return err
}
var (
content []byte
fin *os.File
)
// set format if the file has a json or csv extension
if format == "" {
if strings.HasSuffix(input, ".json") {
format = "json"
} else if strings.HasSuffix(input, ".csv") {
format = "csv"
}
}
if format == "" {
return errors.New("unable to guess format from file extension, please provide a format with --format flag")
}
if input == "-" {
fin = os.Stdin
input = "stdin"
} else {
fin, err = os.Open(input)
if err != nil {
return fmt.Errorf("unable to open %s: %s", input, err)
}
}
content, err = io.ReadAll(fin)
if err != nil {
return fmt.Errorf("unable to read from %s: %s", input, err)
}
decisionsListRaw, err := parseDecisionList(content, format)
if err != nil {
return err
}
decisions := make([]*models.Decision, len(decisionsListRaw))
for i, d := range decisionsListRaw {
if d.Value == "" {
return fmt.Errorf("item %d: missing 'value'", i)
}
if d.Duration == "" {
d.Duration = defaultDuration
log.Debugf("item %d: missing 'duration', using default '%s'", i, defaultDuration)
}
if d.Scenario == "" {
d.Scenario = defaultReason
log.Debugf("item %d: missing 'reason', using default '%s'", i, defaultReason)
}
if d.Type == "" {
d.Type = defaultType
log.Debugf("item %d: missing 'type', using default '%s'", i, defaultType)
}
if d.Scope == "" {
d.Scope = defaultScope
log.Debugf("item %d: missing 'scope', using default '%s'", i, defaultScope)
}
decisions[i] = &models.Decision{
Value: ptr.Of(d.Value),
Duration: ptr.Of(d.Duration),
Origin: ptr.Of(types.CscliImportOrigin),
Scenario: ptr.Of(d.Scenario),
Type: ptr.Of(d.Type),
Scope: ptr.Of(d.Scope),
Simulated: ptr.Of(false),
}
}
if len(decisions) > 1000 {
log.Infof("You are about to add %d decisions, this may take a while", len(decisions))
}
for _, chunk := range slicetools.Chunks(decisions, batchSize) {
log.Debugf("Processing chunk of %d decisions", len(chunk))
importAlert := models.Alert{
CreatedAt: time.Now().UTC().Format(time.RFC3339),
Scenario: ptr.Of(fmt.Sprintf("import %s: %d IPs", input, len(chunk))),
Message: ptr.Of(""),
Events: []*models.Event{},
Source: &models.Source{
Scope: ptr.Of(""),
Value: ptr.Of(""),
},
StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
Capacity: ptr.Of(int32(0)),
Simulated: ptr.Of(false),
EventsCount: ptr.Of(int32(len(chunk))),
Leakspeed: ptr.Of(""),
ScenarioHash: ptr.Of(""),
ScenarioVersion: ptr.Of(""),
Decisions: chunk,
}
_, _, err = Client.Alerts.Add(context.Background(), models.AddAlertsRequest{&importAlert})
if err != nil {
return err
}
}
log.Infof("Imported %d decisions", len(decisions))
return nil
}
func (cli *cliDecisions) newImportCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "import [options]",
Short: "Import decisions from a file or pipe",
Long: "expected format:\n" +
"csv : any of duration,reason,scope,type,value, with a header line\n" +
"json :" + "`{" + `"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"` + "}`",
Args: cobra.NoArgs,
DisableAutoGenTag: true,
Example: `decisions.csv:
duration,scope,value
24h,ip,1.2.3.4
$ cscli decisions import -i decisions.csv
decisions.json:
[{"duration" : "4h", "scope" : "ip", "type" : "ban", "value" : "1.2.3.4"}]
The file format is detected from the extension, but can be forced with the --format option
which is required when reading from standard input.
Raw values, standard input:
$ echo "1.2.3.4" | cscli decisions import -i - --format values
`,
RunE: cli.runImport,
}
flags := cmd.Flags()
flags.SortFlags = false
flags.StringP("input", "i", "", "Input file")
flags.StringP("duration", "d", "4h", "Decision duration: 1h,4h,30m")
flags.String("scope", types.Ip, "Decision scope: ip,range,username")
flags.StringP("reason", "R", "manual", "Decision reason: <scenario-name>")
flags.StringP("type", "t", "ban", "Decision type: ban,captcha,throttle")
flags.Int("batch", 0, "Split import in batches of N decisions")
flags.String("format", "", "Input format: 'json', 'csv' or 'values' (each line is a value, no headers)")
cmd.MarkFlagRequired("input")
return cmd
}

View file

@ -8,15 +8,13 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/models"
)
func (cli *cliDecisions) decisionsTable(out io.Writer, alerts *models.GetAlertsResponse, printMachine bool) {
func decisionsTable(out io.Writer, alerts *models.GetAlertsResponse, printMachine bool) {
t := newTable(out)
t.SetRowLines(false)
header := []string{"ID", "Source", "Scope:Value", "Reason", "Action", "Country", "AS", "Events", "expiration", "Alert ID"}
if printMachine {
header = append(header, "Machine")
}
t.SetHeaders(header...)
for _, alertItem := range *alerts {
@ -24,7 +22,6 @@ func (cli *cliDecisions) decisionsTable(out io.Writer, alerts *models.GetAlertsR
if *alertItem.Simulated {
*decisionItem.Type = fmt.Sprintf("(simul)%s", *decisionItem.Type)
}
row := []string{
strconv.Itoa(int(decisionItem.ID)),
*decisionItem.Origin,
@ -45,6 +42,5 @@ func (cli *cliDecisions) decisionsTable(out io.Writer, alerts *models.GetAlertsR
t.AddRow(row...)
}
}
t.Render()
}

View file

@ -1,51 +0,0 @@
package main
import (
"fmt"
"path/filepath"
"strings"
"github.com/spf13/cobra"
"github.com/spf13/cobra/doc"
)
type cliDoc struct{}
func NewCLIDoc() *cliDoc {
return &cliDoc{}
}
func (cli cliDoc) NewCommand(rootCmd *cobra.Command) *cobra.Command {
cmd := &cobra.Command{
Use: "doc",
Short: "Generate the documentation in `./doc/`. Directory must exist.",
Args: cobra.ExactArgs(0),
Hidden: true,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
if err := doc.GenMarkdownTreeCustom(rootCmd, "./doc/", cli.filePrepender, cli.linkHandler); err != nil {
return fmt.Errorf("failed to generate cobra doc: %s", err)
}
return nil
},
}
return cmd
}
func (cli cliDoc) filePrepender(filename string) string {
const header = `---
id: %s
title: %s
---
`
name := filepath.Base(filename)
base := strings.TrimSuffix(name, filepath.Ext(name))
return fmt.Sprintf(header, base, strings.ReplaceAll(base, "_", " "))
}
func (cli cliDoc) linkHandler(name string) string {
return fmt.Sprintf("/cscli/%s", name)
}

View file

@ -2,7 +2,6 @@ package main
import (
"bufio"
"errors"
"fmt"
"io"
"os"
@ -12,115 +11,67 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/dumps"
"github.com/crowdsecurity/crowdsec/pkg/hubtest"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
func getLineCountForFile(filepath string) (int, error) {
f, err := os.Open(filepath)
if err != nil {
return 0, err
}
defer f.Close()
lc := 0
fs := bufio.NewReader(f)
for {
input, err := fs.ReadBytes('\n')
if len(input) > 1 {
lc++
}
if err != nil && err == io.EOF {
break
}
}
return lc, nil
}
type cliExplain struct {
cfg configGetter
flags struct {
logFile string
dsn string
logLine string
logType string
details bool
skipOk bool
onlySuccessfulParsers bool
noClean bool
crowdsec string
labels string
}
}
func NewCLIExplain(cfg configGetter) *cliExplain {
return &cliExplain{
cfg: cfg,
}
}
func (cli *cliExplain) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "explain",
Short: "Explain log pipeline",
Long: `
Explain log pipeline
`,
Example: `
cscli explain --file ./myfile.log --type nginx
cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog
cscli explain --dsn "file://myfile.log" --type nginx
tail -n 5 myfile.log | cscli explain --type nginx -f -
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.run()
},
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
fileInfo, _ := os.Stdin.Stat()
if cli.flags.logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) {
return errors.New("the option -f - is intended to work with pipes")
}
return nil
},
}
func runExplain(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
flags.StringVarP(&cli.flags.logFile, "file", "f", "", "Log file to test")
flags.StringVarP(&cli.flags.dsn, "dsn", "d", "", "DSN to test")
flags.StringVarP(&cli.flags.logLine, "log", "l", "", "Log line to test")
flags.StringVarP(&cli.flags.logType, "type", "t", "", "Type of the acquisition to test")
flags.StringVar(&cli.flags.labels, "labels", "", "Additional labels to add to the acquisition format (key:value,key2:value2)")
flags.BoolVarP(&cli.flags.details, "verbose", "v", false, "Display individual changes")
flags.BoolVar(&cli.flags.skipOk, "failures", false, "Only show failed lines")
flags.BoolVar(&cli.flags.onlySuccessfulParsers, "only-successful-parsers", false, "Only show successful parsers")
flags.StringVar(&cli.flags.crowdsec, "crowdsec", "crowdsec", "Path to crowdsec")
flags.BoolVar(&cli.flags.noClean, "no-clean", false, "Don't clean runtime environment after tests")
logFile, err := flags.GetString("file")
if err != nil {
return err
}
cmd.MarkFlagRequired("type")
cmd.MarkFlagsOneRequired("log", "file", "dsn")
dsn, err := flags.GetString("dsn")
if err != nil {
return err
}
return cmd
}
logLine, err := flags.GetString("log")
if err != nil {
return err
}
func (cli *cliExplain) run() error {
logFile := cli.flags.logFile
logLine := cli.flags.logLine
logType := cli.flags.logType
dsn := cli.flags.dsn
labels := cli.flags.labels
crowdsec := cli.flags.crowdsec
logType, err := flags.GetString("type")
if err != nil {
return err
}
opts := dumps.DumpOpts{
Details: cli.flags.details,
SkipOk: cli.flags.skipOk,
ShowNotOkParsers: !cli.flags.onlySuccessfulParsers,
opts := hubtest.DumpOpts{}
opts.Details, err = flags.GetBool("verbose")
if err != nil {
return err
}
opts.SkipOk, err = flags.GetBool("failures")
if err != nil {
return err
}
opts.ShowNotOkParsers, err = flags.GetBool("only-successful-parsers")
opts.ShowNotOkParsers = !opts.ShowNotOkParsers
if err != nil {
return err
}
crowdsec, err := flags.GetString("crowdsec")
if err != nil {
return err
}
fileInfo, _ := os.Stdin.Stat()
if logType == "" || (logLine == "" && logFile == "" && dsn == "") {
printHelp(cmd)
fmt.Println()
fmt.Printf("Please provide --type flag\n")
os.Exit(1)
}
if logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) {
return fmt.Errorf("the option -f - is intended to work with pipes")
}
var f *os.File
@ -128,25 +79,12 @@ func (cli *cliExplain) run() error {
// using empty string fallback to /tmp
dir, err := os.MkdirTemp("", "cscli_explain")
if err != nil {
return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %w", err)
return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %s", err)
}
defer func() {
if cli.flags.noClean {
return
}
if _, err := os.Stat(dir); !os.IsNotExist(err) {
if err := os.RemoveAll(dir); err != nil {
log.Errorf("unable to delete temporary directory '%s': %s", dir, err)
}
}
}()
tmpFile := ""
// we create a temporary log file if a log line/stdin has been provided
if logLine != "" || logFile == "-" {
tmpFile := filepath.Join(dir, "cscli_test_tmp.log")
tmpFile = filepath.Join(dir, "cscli_test_tmp.log")
f, err = os.Create(tmpFile)
if err != nil {
return err
@ -160,27 +98,20 @@ func (cli *cliExplain) run() error {
} else if logFile == "-" {
reader := bufio.NewReader(os.Stdin)
errCount := 0
for {
input, err := reader.ReadBytes('\n')
if err != nil && errors.Is(err, io.EOF) {
if err != nil && err == io.EOF {
break
}
if len(input) > 1 {
_, err = f.Write(input)
}
if err != nil || len(input) <= 1 {
_, err = f.Write(input)
if err != nil {
errCount++
}
}
if errCount > 0 {
log.Warnf("Failed to write %d lines to %s", errCount, tmpFile)
log.Warnf("Failed to write %d lines to tmp file", errCount)
}
}
f.Close()
// this is the file that was going to be read by crowdsec anyway
logFile = tmpFile
@ -191,59 +122,81 @@ func (cli *cliExplain) run() error {
if err != nil {
return fmt.Errorf("unable to get absolute path of '%s', exiting", logFile)
}
dsn = fmt.Sprintf("file://%s", absolutePath)
lineCount, err := getLineCountForFile(absolutePath)
if err != nil {
return err
}
log.Debugf("file %s has %d lines", absolutePath, lineCount)
if lineCount == 0 {
return fmt.Errorf("the log file is empty: %s", absolutePath)
}
lineCount := types.GetLineCountForFile(absolutePath)
if lineCount > 100 {
log.Warnf("%s contains %d lines. This may take a lot of resources.", absolutePath, lineCount)
log.Warnf("log file contains %d lines. This may take lot of resources.", lineCount)
}
}
if dsn == "" {
return errors.New("no acquisition (--file or --dsn) provided, can't run cscli test")
return fmt.Errorf("no acquisition (--file or --dsn) provided, can't run cscli test")
}
cmdArgs := []string{"-c", ConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", dir, "-no-api"}
if labels != "" {
log.Debugf("adding labels %s", labels)
cmdArgs = append(cmdArgs, "-label", labels)
}
crowdsecCmd := exec.Command(crowdsec, cmdArgs...)
output, err := crowdsecCmd.CombinedOutput()
if err != nil {
fmt.Println(string(output))
return fmt.Errorf("fail to run crowdsec for test: %w", err)
return fmt.Errorf("fail to run crowdsec for test: %v", err)
}
// rm the temporary log file if only a log line/stdin was provided
if tmpFile != "" {
if err := os.Remove(tmpFile); err != nil {
return fmt.Errorf("unable to remove tmp log file '%s': %+v", tmpFile, err)
}
}
parserDumpFile := filepath.Join(dir, hubtest.ParserResultFileName)
bucketStateDumpFile := filepath.Join(dir, hubtest.BucketPourResultFileName)
parserDump, err := dumps.LoadParserDump(parserDumpFile)
parserDump, err := hubtest.LoadParserDump(parserDumpFile)
if err != nil {
return fmt.Errorf("unable to load parser dump result: %w", err)
return fmt.Errorf("unable to load parser dump result: %s", err)
}
bucketStateDump, err := dumps.LoadBucketPourDump(bucketStateDumpFile)
bucketStateDump, err := hubtest.LoadBucketPourDump(bucketStateDumpFile)
if err != nil {
return fmt.Errorf("unable to load bucket dump result: %w", err)
return fmt.Errorf("unable to load bucket dump result: %s", err)
}
dumps.DumpTree(*parserDump, *bucketStateDump, opts)
hubtest.DumpTree(*parserDump, *bucketStateDump, opts)
if err := os.RemoveAll(dir); err != nil {
return fmt.Errorf("unable to delete temporary directory '%s': %s", dir, err)
}
return nil
}
func NewExplainCmd() *cobra.Command {
cmdExplain := &cobra.Command{
Use: "explain",
Short: "Explain log pipeline",
Long: `
Explain log pipeline
`,
Example: `
cscli explain --file ./myfile.log --type nginx
cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog
cscli explain --dsn "file://myfile.log" --type nginx
tail -n 5 myfile.log | cscli explain --type nginx -f -
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: runExplain,
}
flags := cmdExplain.Flags()
flags.StringP("file", "f", "", "Log file to test")
flags.StringP("dsn", "d", "", "DSN to test")
flags.StringP("log", "l", "", "Log line to test")
flags.StringP("type", "t", "", "Type of the acquisition to test")
flags.BoolP("verbose", "v", false, "Display individual changes")
flags.Bool("failures", false, "Only show failed lines")
flags.Bool("only-successful-parsers", false, "Only show successful parsers")
flags.String("crowdsec", "crowdsec", "Path to crowdsec")
return cmdExplain
}

View file

@ -1,29 +0,0 @@
package main
// Custom types for flag validation and conversion.
import (
"errors"
)
type MachinePassword string
func (p *MachinePassword) String() string {
return string(*p)
}
func (p *MachinePassword) Set(v string) error {
// a password can't be more than 72 characters
// due to bcrypt limitations
if len(v) > 72 {
return errors.New("password too long (max 72 characters)")
}
*p = MachinePassword(v)
return nil
}
func (p *MachinePassword) Type() string {
return "string"
}

View file

@ -1,228 +1,155 @@
package main
import (
"encoding/json"
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
type cliHub struct{
cfg configGetter
}
func NewCLIHub(cfg configGetter) *cliHub {
return &cliHub{
cfg: cfg,
}
}
func (cli *cliHub) NewCommand() *cobra.Command {
cmd := &cobra.Command{
func NewHubCmd() *cobra.Command {
var cmdHub = &cobra.Command{
Use: "hub [action]",
Short: "Manage hub index",
Long: `Hub management
Short: "Manage Hub",
Long: `
Hub management
List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net).
The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.`,
Example: `cscli hub list
cscli hub update
cscli hub upgrade`,
The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.
`,
Example: `
cscli hub list # List all installed configurations
cscli hub update # Download list of available configurations from the hub
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
}
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
cmd.AddCommand(cli.newListCmd())
cmd.AddCommand(cli.newUpdateCmd())
cmd.AddCommand(cli.newUpgradeCmd())
cmd.AddCommand(cli.newTypesCmd())
return cmd
}
func (cli *cliHub) list(all bool) error {
hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger())
if err != nil {
return err
}
for _, v := range hub.Warnings {
log.Info(v)
}
for _, line := range hub.ItemStats() {
log.Info(line)
}
items := make(map[string][]*cwhub.Item)
for _, itemType := range cwhub.ItemTypes {
items[itemType], err = selectItems(hub, itemType, nil, !all)
if err != nil {
return err
}
}
err = listItems(color.Output, cwhub.ItemTypes, items, true)
if err != nil {
return err
}
return nil
}
func (cli *cliHub) newListCmd() *cobra.Command {
var all bool
cmd := &cobra.Command{
Use: "list [-a]",
Short: "List all installed configurations",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.list(all)
return nil
},
}
cmdHub.PersistentFlags().StringVarP(&cwhub.HubBranch, "branch", "b", "", "Use given branch from hub")
flags := cmd.Flags()
flags.BoolVarP(&all, "all", "a", false, "List disabled items as well")
cmdHub.AddCommand(NewHubListCmd())
cmdHub.AddCommand(NewHubUpdateCmd())
cmdHub.AddCommand(NewHubUpgradeCmd())
return cmd
return cmdHub
}
func (cli *cliHub) update() error {
local := cli.cfg().Hub
remote := require.RemoteHub(cli.cfg())
func NewHubListCmd() *cobra.Command {
var cmdHubList = &cobra.Command{
Use: "list [-a]",
Short: "List installed configs",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
// don't use require.Hub because if there is no index file, it would fail
hub, err := cwhub.NewHub(local, remote, true, log.StandardLogger())
if err != nil {
return fmt.Errorf("failed to update hub: %w", err)
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
}
//use LocalSync to get warnings about tainted / outdated items
_, warn := cwhub.LocalSync(csConfig.Hub)
for _, v := range warn {
log.Info(v)
}
cwhub.DisplaySummary()
ListItems(color.Output, []string{
cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW,
}, args, true, false, all)
},
}
cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
for _, v := range hub.Warnings {
log.Info(v)
}
return nil
return cmdHubList
}
func (cli *cliHub) newUpdateCmd() *cobra.Command {
cmd := &cobra.Command{
func NewHubUpdateCmd() *cobra.Command {
var cmdHubUpdate = &cobra.Command{
Use: "update",
Short: "Download the latest index (catalog of available configurations)",
Short: "Fetch available configs from hub",
Long: `
Fetches the .index.json file from the hub, containing the list of available configs.
Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs.
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.update()
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
return nil
},
Run: func(cmd *cobra.Command, args []string) {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
}
//use LocalSync to get warnings about tainted / outdated items
_, warn := cwhub.LocalSync(csConfig.Hub)
for _, v := range warn {
log.Info(v)
}
},
}
return cmd
return cmdHubUpdate
}
func (cli *cliHub) upgrade(force bool) error {
hub, err := require.Hub(cli.cfg(), require.RemoteHub(cli.cfg()), log.StandardLogger())
if err != nil {
return err
}
for _, itemType := range cwhub.ItemTypes {
items, err := hub.GetInstalledItemsByType(itemType)
if err != nil {
return err
}
updated := 0
log.Infof("Upgrading %s", itemType)
for _, item := range items {
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
updated++
}
}
log.Infof("Upgraded %d %s", updated, itemType)
}
return nil
}
func (cli *cliHub) newUpgradeCmd() *cobra.Command {
var force bool
cmd := &cobra.Command{
func NewHubUpgradeCmd() *cobra.Command {
var cmdHubUpgrade = &cobra.Command{
Use: "upgrade",
Short: "Upgrade all configurations to their latest version",
Short: "Upgrade all configs installed from hub",
Long: `
Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available.
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.upgrade(force)
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
return nil
},
Run: func(cmd *cobra.Command, args []string) {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
}
log.Infof("Upgrading collections")
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
log.Infof("Upgrading parsers")
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
log.Infof("Upgrading scenarios")
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
log.Infof("Upgrading postoverflows")
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
},
}
cmdHubUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
flags := cmd.Flags()
flags.BoolVar(&force, "force", false, "Force upgrade: overwrite tainted and outdated files")
return cmd
}
func (cli *cliHub) types() error {
switch cli.cfg().Cscli.Output {
case "human":
s, err := yaml.Marshal(cwhub.ItemTypes)
if err != nil {
return err
}
fmt.Print(string(s))
case "json":
jsonStr, err := json.Marshal(cwhub.ItemTypes)
if err != nil {
return err
}
fmt.Println(string(jsonStr))
case "raw":
for _, itemType := range cwhub.ItemTypes {
fmt.Println(itemType)
}
}
return nil
}
func (cli *cliHub) newTypesCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "types",
Short: "List supported item types",
Long: `
List the types of supported hub items.
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.types()
},
}
return cmd
return cmdHubUpgrade
}

View file

@ -1,123 +0,0 @@
package main
import (
"fmt"
"os"
"golang.org/x/text/cases"
"golang.org/x/text/language"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/pkg/appsec"
"github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIAppsecConfig(cfg configGetter) *cliItem {
return &cliItem{
cfg: cfg,
name: cwhub.APPSEC_CONFIGS,
singular: "appsec-config",
oneOrMore: "appsec-config(s)",
help: cliHelp{
example: `cscli appsec-configs list -a
cscli appsec-configs install crowdsecurity/vpatch
cscli appsec-configs inspect crowdsecurity/vpatch
cscli appsec-configs upgrade crowdsecurity/vpatch
cscli appsec-configs remove crowdsecurity/vpatch
`,
},
installHelp: cliHelp{
example: `cscli appsec-configs install crowdsecurity/vpatch`,
},
removeHelp: cliHelp{
example: `cscli appsec-configs remove crowdsecurity/vpatch`,
},
upgradeHelp: cliHelp{
example: `cscli appsec-configs upgrade crowdsecurity/vpatch`,
},
inspectHelp: cliHelp{
example: `cscli appsec-configs inspect crowdsecurity/vpatch`,
},
listHelp: cliHelp{
example: `cscli appsec-configs list
cscli appsec-configs list -a
cscli appsec-configs list crowdsecurity/vpatch`,
},
}
}
func NewCLIAppsecRule(cfg configGetter) *cliItem {
inspectDetail := func(item *cwhub.Item) error {
// Only show the converted rules in human mode
if csConfig.Cscli.Output != "human" {
return nil
}
appsecRule := appsec.AppsecCollectionConfig{}
yamlContent, err := os.ReadFile(item.State.LocalPath)
if err != nil {
return fmt.Errorf("unable to read file %s: %w", item.State.LocalPath, err)
}
if err := yaml.Unmarshal(yamlContent, &appsecRule); err != nil {
return fmt.Errorf("unable to unmarshal yaml file %s: %w", item.State.LocalPath, err)
}
for _, ruleType := range appsec_rule.SupportedTypes() {
fmt.Printf("\n%s format:\n", cases.Title(language.Und, cases.NoLower).String(ruleType))
for _, rule := range appsecRule.Rules {
convertedRule, _, err := rule.Convert(ruleType, appsecRule.Name)
if err != nil {
return fmt.Errorf("unable to convert rule %s: %w", rule.Name, err)
}
fmt.Println(convertedRule)
}
switch ruleType { //nolint:gocritic
case appsec_rule.ModsecurityRuleType:
for _, rule := range appsecRule.SecLangRules {
fmt.Println(rule)
}
}
}
return nil
}
return &cliItem{
cfg: cfg,
name: "appsec-rules",
singular: "appsec-rule",
oneOrMore: "appsec-rule(s)",
help: cliHelp{
example: `cscli appsec-rules list -a
cscli appsec-rules install crowdsecurity/crs
cscli appsec-rules inspect crowdsecurity/crs
cscli appsec-rules upgrade crowdsecurity/crs
cscli appsec-rules remove crowdsecurity/crs
`,
},
installHelp: cliHelp{
example: `cscli appsec-rules install crowdsecurity/crs`,
},
removeHelp: cliHelp{
example: `cscli appsec-rules remove crowdsecurity/crs`,
},
upgradeHelp: cliHelp{
example: `cscli appsec-rules upgrade crowdsecurity/crs`,
},
inspectHelp: cliHelp{
example: `cscli appsec-rules inspect crowdsecurity/crs`,
},
inspectDetail: inspectDetail,
listHelp: cliHelp{
example: `cscli appsec-rules list
cscli appsec-rules list -a
cscli appsec-rules list crowdsecurity/crs`,
},
}
}

View file

@ -1,41 +0,0 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLICollection(cfg configGetter) *cliItem {
return &cliItem{
cfg: cfg,
name: cwhub.COLLECTIONS,
singular: "collection",
oneOrMore: "collection(s)",
help: cliHelp{
example: `cscli collections list -a
cscli collections install crowdsecurity/http-cve crowdsecurity/iptables
cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables
cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables
cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
`,
},
installHelp: cliHelp{
example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`,
},
removeHelp: cliHelp{
example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`,
},
upgradeHelp: cliHelp{
example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`,
},
inspectHelp: cliHelp{
example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`,
},
listHelp: cliHelp{
example: `cscli collections list
cscli collections list -a
cscli collections list crowdsecurity/http-cve crowdsecurity/iptables
List only enabled collections unless "-a" or names are specified.`,
},
}
}

View file

@ -1,41 +0,0 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIContext(cfg configGetter) *cliItem {
return &cliItem{
cfg: cfg,
name: cwhub.CONTEXTS,
singular: "context",
oneOrMore: "context(s)",
help: cliHelp{
example: `cscli contexts list -a
cscli contexts install crowdsecurity/yyy crowdsecurity/zzz
cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz
cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz
cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz
`,
},
installHelp: cliHelp{
example: `cscli contexts install crowdsecurity/yyy crowdsecurity/zzz`,
},
removeHelp: cliHelp{
example: `cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz`,
},
upgradeHelp: cliHelp{
example: `cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz`,
},
inspectHelp: cliHelp{
example: `cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz`,
},
listHelp: cliHelp{
example: `cscli contexts list
cscli contexts list -a
cscli contexts list crowdsecurity/yyy crowdsecurity/zzz
List only enabled contexts unless "-a" or names are specified.`,
},
}
}

View file

@ -1,41 +0,0 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIParser(cfg configGetter) *cliItem {
return &cliItem{
cfg: cfg,
name: cwhub.PARSERS,
singular: "parser",
oneOrMore: "parser(s)",
help: cliHelp{
example: `cscli parsers list -a
cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
`,
},
installHelp: cliHelp{
example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
removeHelp: cliHelp{
example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
upgradeHelp: cliHelp{
example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
inspectHelp: cliHelp{
example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`,
},
listHelp: cliHelp{
example: `cscli parsers list
cscli parsers list -a
cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs
List only enabled parsers unless "-a" or names are specified.`,
},
}
}

View file

@ -1,41 +0,0 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIPostOverflow(cfg configGetter) *cliItem {
return &cliItem{
cfg: cfg,
name: cwhub.POSTOVERFLOWS,
singular: "postoverflow",
oneOrMore: "postoverflow(s)",
help: cliHelp{
example: `cscli postoverflows list -a
cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
`,
},
installHelp: cliHelp{
example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
removeHelp: cliHelp{
example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
upgradeHelp: cliHelp{
example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
inspectHelp: cliHelp{
example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
listHelp: cliHelp{
example: `cscli postoverflows list
cscli postoverflows list -a
cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns
List only enabled postoverflows unless "-a" or names are specified.`,
},
}
}

View file

@ -1,41 +0,0 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIScenario(cfg configGetter) *cliItem {
return &cliItem{
cfg: cfg,
name: cwhub.SCENARIOS,
singular: "scenario",
oneOrMore: "scenario(s)",
help: cliHelp{
example: `cscli scenarios list -a
cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
`,
},
installHelp: cliHelp{
example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
removeHelp: cliHelp{
example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
upgradeHelp: cliHelp{
example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
inspectHelp: cliHelp{
example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
listHelp: cliHelp{
example: `cscli scenarios list
cscli scenarios list -a
cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing
List only enabled scenarios unless "-a" or names are specified.`,
},
}
}

View file

@ -2,106 +2,73 @@ package main
import (
"encoding/json"
"errors"
"fmt"
"math"
"os"
"path/filepath"
"strings"
"text/template"
"github.com/AlecAivazis/survey/v2"
"github.com/enescakir/emoji"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/dumps"
"github.com/crowdsecurity/crowdsec/pkg/emoji"
"github.com/crowdsecurity/crowdsec/pkg/hubtest"
)
var (
HubTest hubtest.HubTest
HubAppsecTests hubtest.HubTest
hubPtr *hubtest.HubTest
isAppsecTest bool
HubTest hubtest.HubTest
)
type cliHubTest struct {
cfg configGetter
}
func NewHubTestCmd() *cobra.Command {
var hubPath string
var crowdsecPath string
var cscliPath string
func NewCLIHubTest(cfg configGetter) *cliHubTest {
return &cliHubTest{
cfg: cfg,
}
}
func (cli *cliHubTest) NewCommand() *cobra.Command {
var (
hubPath string
crowdsecPath string
cscliPath string
)
cmd := &cobra.Command{
var cmdHubTest = &cobra.Command{
Use: "hubtest",
Short: "Run functional tests on hub configurations",
Long: "Run functional tests on hub configurations (parsers, scenarios, collections...)",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
var err error
HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, false)
HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath)
if err != nil {
return fmt.Errorf("unable to load hubtest: %+v", err)
}
HubAppsecTests, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, true)
if err != nil {
return fmt.Errorf("unable to load appsec specific hubtest: %+v", err)
}
// commands will use the hubPtr, will point to the default hubTest object, or the one dedicated to appsec tests
hubPtr = &HubTest
if isAppsecTest {
hubPtr = &HubAppsecTests
}
return nil
},
}
cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
cmd.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
cmd.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
cmd.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
cmd.PersistentFlags().BoolVar(&isAppsecTest, "appsec", false, "Command relates to appsec tests")
cmdHubTest.AddCommand(NewHubTestCreateCmd())
cmdHubTest.AddCommand(NewHubTestRunCmd())
cmdHubTest.AddCommand(NewHubTestCleanCmd())
cmdHubTest.AddCommand(NewHubTestInfoCmd())
cmdHubTest.AddCommand(NewHubTestListCmd())
cmdHubTest.AddCommand(NewHubTestCoverageCmd())
cmdHubTest.AddCommand(NewHubTestEvalCmd())
cmdHubTest.AddCommand(NewHubTestExplainCmd())
cmd.AddCommand(cli.NewCreateCmd())
cmd.AddCommand(cli.NewRunCmd())
cmd.AddCommand(cli.NewCleanCmd())
cmd.AddCommand(cli.NewInfoCmd())
cmd.AddCommand(cli.NewListCmd())
cmd.AddCommand(cli.NewCoverageCmd())
cmd.AddCommand(cli.NewEvalCmd())
cmd.AddCommand(cli.NewExplainCmd())
return cmd
return cmdHubTest
}
func (cli *cliHubTest) NewCreateCmd() *cobra.Command {
var (
ignoreParsers bool
labels map[string]string
logType string
)
func NewHubTestCreateCmd() *cobra.Command {
parsers := []string{}
postoverflows := []string{}
scenarios := []string{}
var ignoreParsers bool
var labels map[string]string
var logType string
cmd := &cobra.Command{
var cmdHubTestCreate = &cobra.Command{
Use: "create",
Short: "create [test_name]",
Example: `cscli hubtest create my-awesome-test --type syslog
@ -109,170 +76,134 @@ cscli hubtest create my-nginx-custom-test --type nginx
cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios crowdsecurity/http-probing`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
RunE: func(cmd *cobra.Command, args []string) error {
testName := args[0]
testPath := filepath.Join(hubPtr.HubTestPath, testName)
testPath := filepath.Join(HubTest.HubTestPath, testName)
if _, err := os.Stat(testPath); os.IsExist(err) {
return fmt.Errorf("test '%s' already exists in '%s', exiting", testName, testPath)
}
if isAppsecTest {
logType = "appsec"
}
if logType == "" {
return errors.New("please provide a type (--type) for the test")
return fmt.Errorf("please provide a type (--type) for the test")
}
if err := os.MkdirAll(testPath, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %+v", testPath, err)
}
configFilePath := filepath.Join(testPath, "config.yaml")
// create empty log file
logFileName := fmt.Sprintf("%s.log", testName)
logFilePath := filepath.Join(testPath, logFileName)
logFile, err := os.Create(logFilePath)
if err != nil {
return err
}
logFile.Close()
configFileData := &hubtest.HubTestItemConfig{}
if logType == "appsec" {
// create empty nuclei template file
nucleiFileName := fmt.Sprintf("%s.yaml", testName)
nucleiFilePath := filepath.Join(testPath, nucleiFileName)
// create empty parser assertion file
parserAssertFilePath := filepath.Join(testPath, hubtest.ParserAssertFileName)
parserAssertFile, err := os.Create(parserAssertFilePath)
if err != nil {
return err
}
parserAssertFile.Close()
nucleiFile, err := os.OpenFile(nucleiFilePath, os.O_RDWR|os.O_CREATE, 0o755)
if err != nil {
return err
}
// create empty scenario assertion file
scenarioAssertFilePath := filepath.Join(testPath, hubtest.ScenarioAssertFileName)
scenarioAssertFile, err := os.Create(scenarioAssertFilePath)
if err != nil {
return err
}
scenarioAssertFile.Close()
ntpl := template.Must(template.New("nuclei").Parse(hubtest.TemplateNucleiFile))
if ntpl == nil {
return errors.New("unable to parse nuclei template")
}
ntpl.ExecuteTemplate(nucleiFile, "nuclei", struct{ TestName string }{TestName: testName})
nucleiFile.Close()
configFileData.AppsecRules = []string{"./appsec-rules/<author>/your_rule_here.yaml"}
configFileData.NucleiTemplate = nucleiFileName
fmt.Println()
fmt.Printf(" Test name : %s\n", testName)
fmt.Printf(" Test path : %s\n", testPath)
fmt.Printf(" Config File : %s\n", configFilePath)
fmt.Printf(" Nuclei Template : %s\n", nucleiFilePath)
} else {
// create empty log file
logFileName := fmt.Sprintf("%s.log", testName)
logFilePath := filepath.Join(testPath, logFileName)
logFile, err := os.Create(logFilePath)
if err != nil {
return err
}
logFile.Close()
parsers = append(parsers, "crowdsecurity/syslog-logs")
parsers = append(parsers, "crowdsecurity/dateparse-enrich")
// create empty parser assertion file
parserAssertFilePath := filepath.Join(testPath, hubtest.ParserAssertFileName)
parserAssertFile, err := os.Create(parserAssertFilePath)
if err != nil {
return err
}
parserAssertFile.Close()
// create empty scenario assertion file
scenarioAssertFilePath := filepath.Join(testPath, hubtest.ScenarioAssertFileName)
scenarioAssertFile, err := os.Create(scenarioAssertFilePath)
if err != nil {
return err
}
scenarioAssertFile.Close()
parsers = append(parsers, "crowdsecurity/syslog-logs")
parsers = append(parsers, "crowdsecurity/dateparse-enrich")
if len(scenarios) == 0 {
scenarios = append(scenarios, "")
}
if len(postoverflows) == 0 {
postoverflows = append(postoverflows, "")
}
configFileData.Parsers = parsers
configFileData.Scenarios = scenarios
configFileData.PostOverflows = postoverflows
configFileData.LogFile = logFileName
configFileData.LogType = logType
configFileData.IgnoreParsers = ignoreParsers
configFileData.Labels = labels
fmt.Println()
fmt.Printf(" Test name : %s\n", testName)
fmt.Printf(" Test path : %s\n", testPath)
fmt.Printf(" Log file : %s (please fill it with logs)\n", logFilePath)
fmt.Printf(" Parser assertion file : %s (please fill it with assertion)\n", parserAssertFilePath)
fmt.Printf(" Scenario assertion file : %s (please fill it with assertion)\n", scenarioAssertFilePath)
fmt.Printf(" Configuration File : %s (please fill it with parsers, scenarios...)\n", configFilePath)
if len(scenarios) == 0 {
scenarios = append(scenarios, "")
}
fd, err := os.Create(configFilePath)
if len(postoverflows) == 0 {
postoverflows = append(postoverflows, "")
}
configFileData := &hubtest.HubTestItemConfig{
Parsers: parsers,
Scenarios: scenarios,
PostOVerflows: postoverflows,
LogFile: logFileName,
LogType: logType,
IgnoreParsers: ignoreParsers,
Labels: labels,
}
configFilePath := filepath.Join(testPath, "config.yaml")
fd, err := os.OpenFile(configFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
if err != nil {
return fmt.Errorf("open: %w", err)
return fmt.Errorf("open: %s", err)
}
data, err := yaml.Marshal(configFileData)
if err != nil {
return fmt.Errorf("marshal: %w", err)
return fmt.Errorf("marshal: %s", err)
}
_, err = fd.Write(data)
if err != nil {
return fmt.Errorf("write: %w", err)
return fmt.Errorf("write: %s", err)
}
if err := fd.Close(); err != nil {
return fmt.Errorf("close: %w", err)
return fmt.Errorf("close: %s", err)
}
fmt.Println()
fmt.Printf(" Test name : %s\n", testName)
fmt.Printf(" Test path : %s\n", testPath)
fmt.Printf(" Log file : %s (please fill it with logs)\n", logFilePath)
fmt.Printf(" Parser assertion file : %s (please fill it with assertion)\n", parserAssertFilePath)
fmt.Printf(" Scenario assertion file : %s (please fill it with assertion)\n", scenarioAssertFilePath)
fmt.Printf(" Configuration File : %s (please fill it with parsers, scenarios...)\n", configFilePath)
return nil
},
}
cmdHubTestCreate.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test")
cmdHubTestCreate.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test")
cmdHubTestCreate.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test")
cmdHubTestCreate.Flags().StringSliceVarP(&scenarios, "scenarios", "s", scenarios, "Scenarios to add to test")
cmdHubTestCreate.PersistentFlags().BoolVar(&ignoreParsers, "ignore-parsers", false, "Don't run test on parsers")
cmd.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test")
cmd.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test")
cmd.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test")
cmd.Flags().StringSliceVarP(&scenarios, "scenarios", "s", scenarios, "Scenarios to add to test")
cmd.PersistentFlags().BoolVar(&ignoreParsers, "ignore-parsers", false, "Don't run test on parsers")
return cmd
return cmdHubTestCreate
}
func (cli *cliHubTest) NewRunCmd() *cobra.Command {
var (
noClean bool
runAll bool
forceClean bool
NucleiTargetHost string
AppSecHost string
)
cmd := &cobra.Command{
func NewHubTestRunCmd() *cobra.Command {
var noClean bool
var runAll bool
var forceClean bool
var cmdHubTestRun = &cobra.Command{
Use: "run",
Short: "run [test_name]",
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
cfg := cli.cfg()
if !runAll && len(args) == 0 {
printHelp(cmd)
return errors.New("please provide test to run or --all flag")
return fmt.Errorf("Please provide test to run or --all flag")
}
hubPtr.NucleiTargetHost = NucleiTargetHost
hubPtr.AppSecHost = AppSecHost
if runAll {
if err := hubPtr.LoadAllTests(); err != nil {
if err := HubTest.LoadAllTests(); err != nil {
return fmt.Errorf("unable to load all tests: %+v", err)
}
} else {
for _, testName := range args {
_, err := hubPtr.LoadTestItem(testName)
_, err := HubTest.LoadTestItem(testName)
if err != nil {
return fmt.Errorf("unable to load test '%s': %w", testName, err)
return fmt.Errorf("unable to load test '%s': %s", testName, err)
}
}
}
// set timezone to avoid DST issues
os.Setenv("TZ", "UTC")
for _, test := range hubPtr.Tests {
if cfg.Cscli.Output == "human" {
for _, test := range HubTest.Tests {
if csConfig.Cscli.Output == "human" {
log.Infof("Running test '%s'", test.Name)
}
err := test.Run()
@ -283,13 +214,11 @@ func (cli *cliHubTest) NewRunCmd() *cobra.Command {
return nil
},
PersistentPostRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
PersistentPostRunE: func(cmd *cobra.Command, args []string) error {
success := true
testResult := make(map[string]bool)
for _, test := range hubPtr.Tests {
if test.AutoGen && !isAppsecTest {
for _, test := range HubTest.Tests {
if test.AutoGen {
if test.ParserAssert.AutoGenAssert {
log.Warningf("Assert file '%s' is empty, generating assertion:", test.ParserAssert.File)
fmt.Println()
@ -302,7 +231,7 @@ func (cli *cliHubTest) NewRunCmd() *cobra.Command {
}
if !noClean {
if err := test.Clean(); err != nil {
return fmt.Errorf("unable to clean test '%s' env: %w", test.Name, err)
return fmt.Errorf("unable to clean test '%s' env: %s", test.Name, err)
}
}
fmt.Printf("\nPlease fill your assert file(s) for test '%s', exiting\n", test.Name)
@ -310,18 +239,18 @@ func (cli *cliHubTest) NewRunCmd() *cobra.Command {
}
testResult[test.Name] = test.Success
if test.Success {
if cfg.Cscli.Output == "human" {
if csConfig.Cscli.Output == "human" {
log.Infof("Test '%s' passed successfully (%d assertions)\n", test.Name, test.ParserAssert.NbAssert+test.ScenarioAssert.NbAssert)
}
if !noClean {
if err := test.Clean(); err != nil {
return fmt.Errorf("unable to clean test '%s' env: %w", test.Name, err)
return fmt.Errorf("unable to clean test '%s' env: %s", test.Name, err)
}
}
} else {
success = false
cleanTestEnv := false
if cfg.Cscli.Output == "human" {
if csConfig.Cscli.Output == "human" {
if len(test.ParserAssert.Fails) > 0 {
fmt.Println()
log.Errorf("Parser test '%s' failed (%d errors)\n", test.Name, len(test.ParserAssert.Fails))
@ -352,23 +281,21 @@ func (cli *cliHubTest) NewRunCmd() *cobra.Command {
Default: true,
}
if err := survey.AskOne(prompt, &cleanTestEnv); err != nil {
return fmt.Errorf("unable to ask to remove runtime folder: %w", err)
return fmt.Errorf("unable to ask to remove runtime folder: %s", err)
}
}
}
if cleanTestEnv || forceClean {
if err := test.Clean(); err != nil {
return fmt.Errorf("unable to clean test '%s' env: %w", test.Name, err)
return fmt.Errorf("unable to clean test '%s' env: %s", test.Name, err)
}
}
}
}
switch cfg.Cscli.Output {
case "human":
if csConfig.Cscli.Output == "human" {
hubTestResultTable(color.Output, testResult)
case "json":
} else if csConfig.Cscli.Output == "json" {
jsonResult := make(map[string][]string, 0)
jsonResult["success"] = make([]string, 0)
jsonResult["fail"] = make([]string, 0)
@ -381,11 +308,9 @@ func (cli *cliHubTest) NewRunCmd() *cobra.Command {
}
jsonStr, err := json.Marshal(jsonResult)
if err != nil {
return fmt.Errorf("unable to json test result: %w", err)
return fmt.Errorf("unable to json test result: %s", err)
}
fmt.Println(string(jsonStr))
default:
return errors.New("only human/json output modes are supported")
}
if !success {
@ -395,30 +320,28 @@ func (cli *cliHubTest) NewRunCmd() *cobra.Command {
return nil
},
}
cmdHubTestRun.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
cmdHubTestRun.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
cmdHubTestRun.Flags().BoolVar(&runAll, "all", false, "Run all tests")
cmd.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
cmd.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
cmd.Flags().StringVar(&NucleiTargetHost, "target", hubtest.DefaultNucleiTarget, "Target for AppSec Test")
cmd.Flags().StringVar(&AppSecHost, "host", hubtest.DefaultAppsecHost, "Address to expose AppSec for hubtest")
cmd.Flags().BoolVar(&runAll, "all", false, "Run all tests")
return cmd
return cmdHubTestRun
}
func (cli *cliHubTest) NewCleanCmd() *cobra.Command {
cmd := &cobra.Command{
func NewHubTestCleanCmd() *cobra.Command {
var cmdHubTestClean = &cobra.Command{
Use: "clean",
Short: "clean [test_name]",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
RunE: func(cmd *cobra.Command, args []string) error {
for _, testName := range args {
test, err := hubPtr.LoadTestItem(testName)
test, err := HubTest.LoadTestItem(testName)
if err != nil {
return fmt.Errorf("unable to load test '%s': %w", testName, err)
return fmt.Errorf("unable to load test '%s': %s", testName, err)
}
if err := test.Clean(); err != nil {
return fmt.Errorf("unable to clean test '%s' env: %w", test.Name, err)
return fmt.Errorf("unable to clean test '%s' env: %s", test.Name, err)
}
}
@ -426,32 +349,28 @@ func (cli *cliHubTest) NewCleanCmd() *cobra.Command {
},
}
return cmd
return cmdHubTestClean
}
func (cli *cliHubTest) NewInfoCmd() *cobra.Command {
cmd := &cobra.Command{
func NewHubTestInfoCmd() *cobra.Command {
var cmdHubTestInfo = &cobra.Command{
Use: "info",
Short: "info [test_name]",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
RunE: func(cmd *cobra.Command, args []string) error {
for _, testName := range args {
test, err := hubPtr.LoadTestItem(testName)
test, err := HubTest.LoadTestItem(testName)
if err != nil {
return fmt.Errorf("unable to load test '%s': %w", testName, err)
return fmt.Errorf("unable to load test '%s': %s", testName, err)
}
fmt.Println()
fmt.Printf(" Test name : %s\n", test.Name)
fmt.Printf(" Test path : %s\n", test.Path)
if isAppsecTest {
fmt.Printf(" Nuclei Template : %s\n", test.Config.NucleiTemplate)
fmt.Printf(" Appsec Rules : %s\n", strings.Join(test.Config.AppsecRules, ", "))
} else {
fmt.Printf(" Log file : %s\n", filepath.Join(test.Path, test.Config.LogFile))
fmt.Printf(" Parser assertion file : %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName))
fmt.Printf(" Scenario assertion file : %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName))
}
fmt.Printf(" Log file : %s\n", filepath.Join(test.Path, test.Config.LogFile))
fmt.Printf(" Parser assertion file : %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName))
fmt.Printf(" Scenario assertion file : %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName))
fmt.Printf(" Configuration File : %s\n", filepath.Join(test.Path, "config.yaml"))
}
@ -459,80 +378,72 @@ func (cli *cliHubTest) NewInfoCmd() *cobra.Command {
},
}
return cmd
return cmdHubTestInfo
}
func (cli *cliHubTest) NewListCmd() *cobra.Command {
cmd := &cobra.Command{
func NewHubTestListCmd() *cobra.Command {
var cmdHubTestList = &cobra.Command{
Use: "list",
Short: "list",
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := hubPtr.LoadAllTests(); err != nil {
return fmt.Errorf("unable to load all tests: %w", err)
RunE: func(cmd *cobra.Command, args []string) error {
if err := HubTest.LoadAllTests(); err != nil {
return fmt.Errorf("unable to load all tests: %s", err)
}
switch cfg.Cscli.Output {
switch csConfig.Cscli.Output {
case "human":
hubTestListTable(color.Output, hubPtr.Tests)
hubTestListTable(color.Output, HubTest.Tests)
case "json":
j, err := json.MarshalIndent(hubPtr.Tests, " ", " ")
j, err := json.MarshalIndent(HubTest.Tests, " ", " ")
if err != nil {
return err
}
fmt.Println(string(j))
default:
return errors.New("only human/json output modes are supported")
return fmt.Errorf("only human/json output modes are supported")
}
return nil
},
}
return cmd
return cmdHubTestList
}
func (cli *cliHubTest) NewCoverageCmd() *cobra.Command {
var (
showParserCov bool
showScenarioCov bool
showOnlyPercent bool
showAppsecCov bool
)
cmd := &cobra.Command{
func NewHubTestCoverageCmd() *cobra.Command {
var showParserCov bool
var showScenarioCov bool
var showOnlyPercent bool
var cmdHubTestCoverage = &cobra.Command{
Use: "coverage",
Short: "coverage",
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
// for this one we explicitly don't do for appsec
RunE: func(cmd *cobra.Command, args []string) error {
if err := HubTest.LoadAllTests(); err != nil {
return fmt.Errorf("unable to load all tests: %+v", err)
}
var err error
scenarioCoverage := []hubtest.Coverage{}
parserCoverage := []hubtest.Coverage{}
appsecRuleCoverage := []hubtest.Coverage{}
scenarioCoverage := []hubtest.ScenarioCoverage{}
parserCoverage := []hubtest.ParserCoverage{}
scenarioCoveragePercent := 0
parserCoveragePercent := 0
appsecRuleCoveragePercent := 0
// if both are false (flag by default), show both
showAll := !showScenarioCov && !showParserCov && !showAppsecCov
showAll := !showScenarioCov && !showParserCov
if showParserCov || showAll {
parserCoverage, err = HubTest.GetParsersCoverage()
if err != nil {
return fmt.Errorf("while getting parser coverage: %w", err)
return fmt.Errorf("while getting parser coverage: %s", err)
}
parserTested := 0
for _, test := range parserCoverage {
if test.TestsCount > 0 {
parserTested++
parserTested += 1
}
}
parserCoveragePercent = int(math.Round((float64(parserTested) / float64(len(parserCoverage)) * 100)))
@ -541,50 +452,29 @@ func (cli *cliHubTest) NewCoverageCmd() *cobra.Command {
if showScenarioCov || showAll {
scenarioCoverage, err = HubTest.GetScenariosCoverage()
if err != nil {
return fmt.Errorf("while getting scenario coverage: %w", err)
return fmt.Errorf("while getting scenario coverage: %s", err)
}
scenarioTested := 0
for _, test := range scenarioCoverage {
if test.TestsCount > 0 {
scenarioTested++
scenarioTested += 1
}
}
scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100)))
}
if showAppsecCov || showAll {
appsecRuleCoverage, err = HubTest.GetAppsecCoverage()
if err != nil {
return fmt.Errorf("while getting scenario coverage: %w", err)
}
appsecRuleTested := 0
for _, test := range appsecRuleCoverage {
if test.TestsCount > 0 {
appsecRuleTested++
}
}
appsecRuleCoveragePercent = int(math.Round((float64(appsecRuleTested) / float64(len(appsecRuleCoverage)) * 100)))
}
if showOnlyPercent {
switch {
case showAll:
fmt.Printf("parsers=%d%%\nscenarios=%d%%\nappsec_rules=%d%%", parserCoveragePercent, scenarioCoveragePercent, appsecRuleCoveragePercent)
case showParserCov:
if showAll {
fmt.Printf("parsers=%d%%\nscenarios=%d%%", parserCoveragePercent, scenarioCoveragePercent)
} else if showParserCov {
fmt.Printf("parsers=%d%%", parserCoveragePercent)
case showScenarioCov:
} else if showScenarioCov {
fmt.Printf("scenarios=%d%%", scenarioCoveragePercent)
case showAppsecCov:
fmt.Printf("appsec_rules=%d%%", appsecRuleCoveragePercent)
}
os.Exit(0)
}
switch cfg.Cscli.Output {
case "human":
if csConfig.Cscli.Output == "human" {
if showParserCov || showAll {
hubTestParserCoverageTable(color.Output, parserCoverage)
}
@ -592,11 +482,6 @@ func (cli *cliHubTest) NewCoverageCmd() *cobra.Command {
if showScenarioCov || showAll {
hubTestScenarioCoverageTable(color.Output, scenarioCoverage)
}
if showAppsecCov || showAll {
hubTestAppsecRuleCoverageTable(color.Output, appsecRuleCoverage)
}
fmt.Println()
if showParserCov || showAll {
fmt.Printf("PARSERS : %d%% of coverage\n", parserCoveragePercent)
@ -604,10 +489,7 @@ func (cli *cliHubTest) NewCoverageCmd() *cobra.Command {
if showScenarioCov || showAll {
fmt.Printf("SCENARIOS : %d%% of coverage\n", scenarioCoveragePercent)
}
if showAppsecCov || showAll {
fmt.Printf("APPSEC RULES : %d%% of coverage\n", appsecRuleCoveragePercent)
}
case "json":
} else if csConfig.Cscli.Output == "json" {
dump, err := json.MarshalIndent(parserCoverage, "", " ")
if err != nil {
return err
@ -618,71 +500,29 @@ func (cli *cliHubTest) NewCoverageCmd() *cobra.Command {
return err
}
fmt.Printf("%s", dump)
dump, err = json.MarshalIndent(appsecRuleCoverage, "", " ")
if err != nil {
return err
}
fmt.Printf("%s", dump)
default:
return errors.New("only human/json output modes are supported")
} else {
return fmt.Errorf("only human/json output modes are supported")
}
return nil
},
}
cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
cmd.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
cmd.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
cmd.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
cmd.PersistentFlags().BoolVar(&showAppsecCov, "appsec", false, "Show only appsec coverage")
return cmd
return cmdHubTestCoverage
}
func (cli *cliHubTest) NewEvalCmd() *cobra.Command {
var evalExpression string
cmd := &cobra.Command{
func NewHubTestEvalCmd() *cobra.Command {
var evalExpression string
var cmdHubTestEval = &cobra.Command{
Use: "eval",
Short: "eval [test_name]",
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
for _, testName := range args {
test, err := hubPtr.LoadTestItem(testName)
if err != nil {
return fmt.Errorf("can't load test: %+v", err)
}
err = test.ParserAssert.LoadTest(test.ParserResultFile)
if err != nil {
return fmt.Errorf("can't load test results from '%s': %+v", test.ParserResultFile, err)
}
output, err := test.ParserAssert.EvalExpression(evalExpression)
if err != nil {
return err
}
fmt.Print(output)
}
return nil
},
}
cmd.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval")
return cmd
}
func (cli *cliHubTest) NewExplainCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "explain",
Short: "explain [test_name]",
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
RunE: func(cmd *cobra.Command, args []string) error {
for _, testName := range args {
test, err := HubTest.LoadTestItem(testName)
if err != nil {
@ -690,32 +530,66 @@ func (cli *cliHubTest) NewExplainCmd() *cobra.Command {
}
err = test.ParserAssert.LoadTest(test.ParserResultFile)
if err != nil {
if err = test.Run(); err != nil {
return fmt.Errorf("can't load test results from '%s': %+v", test.ParserResultFile, err)
}
output, err := test.ParserAssert.EvalExpression(evalExpression)
if err != nil {
return err
}
fmt.Print(output)
}
return nil
},
}
cmdHubTestEval.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval")
return cmdHubTestEval
}
func NewHubTestExplainCmd() *cobra.Command {
var cmdHubTestExplain = &cobra.Command{
Use: "explain",
Short: "explain [test_name]",
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
for _, testName := range args {
test, err := HubTest.LoadTestItem(testName)
if err != nil {
return fmt.Errorf("can't load test: %+v", err)
}
err = test.ParserAssert.LoadTest(test.ParserResultFile)
if err != nil {
err := test.Run()
if err != nil {
return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
}
if err = test.ParserAssert.LoadTest(test.ParserResultFile); err != nil {
return fmt.Errorf("unable to load parser result after run: %w", err)
err = test.ParserAssert.LoadTest(test.ParserResultFile)
if err != nil {
return fmt.Errorf("unable to load parser result after run: %s", err)
}
}
err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
if err != nil {
if err = test.Run(); err != nil {
err := test.Run()
if err != nil {
return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
}
if err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile); err != nil {
return fmt.Errorf("unable to load scenario result after run: %w", err)
err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
if err != nil {
return fmt.Errorf("unable to load scenario result after run: %s", err)
}
}
opts := dumps.DumpOpts{}
dumps.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData, opts)
opts := hubtest.DumpOpts{}
hubtest.DumpTree(*test.ParserAssert.TestData, *test.ScenarioAssert.PourData, opts)
}
return nil
},
}
return cmd
return cmdHubTestExplain
}

View file

@ -5,8 +5,8 @@ import (
"io"
"github.com/aquasecurity/table"
"github.com/enescakir/emoji"
"github.com/crowdsecurity/crowdsec/pkg/emoji"
"github.com/crowdsecurity/crowdsec/pkg/hubtest"
)
@ -17,9 +17,9 @@ func hubTestResultTable(out io.Writer, testResult map[string]bool) {
t.SetAlignment(table.AlignLeft)
for testName, success := range testResult {
status := emoji.CheckMarkButton
status := emoji.CheckMarkButton.String()
if !success {
status = emoji.CrossMark
status = emoji.CrossMark.String()
}
t.AddRow(testName, status)
@ -41,64 +41,39 @@ func hubTestListTable(out io.Writer, tests []*hubtest.HubTestItem) {
t.Render()
}
func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.ParserCoverage) {
t := newLightTable(out)
t.SetHeaders("Parser", "Status", "Number of tests")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
parserTested := 0
for _, test := range coverage {
status := emoji.RedCircle
status := emoji.RedCircle.String()
if test.TestsCount > 0 {
status = emoji.GreenCircle
status = emoji.GreenCircle.String()
parserTested++
}
t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
t.AddRow(test.Parser, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
}
t.Render()
}
func hubTestAppsecRuleCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
t := newLightTable(out)
t.SetHeaders("Appsec Rule", "Status", "Number of tests")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
parserTested := 0
for _, test := range coverage {
status := emoji.RedCircle
if test.TestsCount > 0 {
status = emoji.GreenCircle
parserTested++
}
t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
}
t.Render()
}
func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.ScenarioCoverage) {
t := newLightTable(out)
t.SetHeaders("Scenario", "Status", "Number of tests")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
parserTested := 0
for _, test := range coverage {
status := emoji.RedCircle
status := emoji.RedCircle.String()
if test.TestsCount > 0 {
status = emoji.GreenCircle
status = emoji.GreenCircle.String()
parserTested++
}
t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
t.AddRow(test.Scenario, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
}
t.Render()

View file

@ -1,327 +0,0 @@
package main
import (
"fmt"
"math"
"net/http"
"strconv"
"strings"
"time"
"github.com/fatih/color"
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/prom2json"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func ShowMetrics(hubItem *cwhub.Item) error {
switch hubItem.Type {
case cwhub.PARSERS:
metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
parserMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.SCENARIOS:
metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
scenarioMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.COLLECTIONS:
for _, sub := range hubItem.SubItems() {
if err := ShowMetrics(sub); err != nil {
return err
}
}
case cwhub.APPSEC_RULES:
metrics := GetAppsecRuleMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
appsecMetricsTable(color.Output, hubItem.Name, metrics)
default: // no metrics for this item type
}
return nil
}
// GetParserMetric is a complete rip from prom2json
func GetParserMetric(url string, itemName string) map[string]map[string]int {
stats := make(map[string]map[string]int)
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
source, ok := metric.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v", metric.Labels)
} else {
if srctype, ok := metric.Labels["type"]; ok {
source = srctype + ":" + source
}
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_reader_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
stats[source]["parsed"] = 0
stats[source]["reads"] = 0
stats[source]["unparsed"] = 0
stats[source]["hits"] = 0
}
stats[source]["reads"] += ival
case "cs_parser_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_parser_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
case "cs_node_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["hits"] += ival
case "cs_node_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_node_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
default:
continue
}
}
}
return stats
}
func GetScenarioMetric(url string, itemName string) map[string]int {
stats := make(map[string]int)
stats["instantiation"] = 0
stats["curr_count"] = 0
stats["overflow"] = 0
stats["pour"] = 0
stats["underflow"] = 0
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_bucket_created_total":
stats["instantiation"] += ival
case "cs_buckets":
stats["curr_count"] += ival
case "cs_bucket_overflowed_total":
stats["overflow"] += ival
case "cs_bucket_poured_total":
stats["pour"] += ival
case "cs_bucket_underflowed_total":
stats["underflow"] += ival
default:
continue
}
}
}
return stats
}
func GetAppsecRuleMetric(url string, itemName string) map[string]int {
stats := make(map[string]int)
stats["inband_hits"] = 0
stats["outband_hits"] = 0
results := GetPrometheusMetric(url)
for idx, fam := range results {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["rule_name"]
if !ok {
log.Debugf("no rule_name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
band, ok := metric.Labels["type"]
if !ok {
log.Debugf("no type in Metric %v", metric.Labels)
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_appsec_rule_hits":
switch band {
case "inband":
stats["inband_hits"] += ival
case "outband":
stats["outband_hits"] += ival
default:
continue
}
default:
continue
}
}
}
return stats
}
func GetPrometheusMetric(url string) []*prom2json.Family {
mfChan := make(chan *dto.MetricFamily, 1024)
// Start with the DefaultTransport for sane defaults.
transport := http.DefaultTransport.(*http.Transport).Clone()
// Conservatively disable HTTP keep-alives as this program will only
// ever need a single HTTP request.
transport.DisableKeepAlives = true
// Timeout early if the server doesn't even return the headers.
transport.ResponseHeaderTimeout = time.Minute
go func() {
defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
err := prom2json.FetchMetricFamilies(url, mfChan, transport)
if err != nil {
log.Fatalf("failed to fetch prometheus metrics : %v", err)
}
}()
result := []*prom2json.Family{}
for mf := range mfChan {
result = append(result, prom2json.NewFamily(mf))
}
log.Debugf("Finished reading prometheus output, %d entries", len(result))
return result
}
type unit struct {
value int64
symbol string
}
var ranges = []unit{
{value: 1e18, symbol: "E"},
{value: 1e15, symbol: "P"},
{value: 1e12, symbol: "T"},
{value: 1e9, symbol: "G"},
{value: 1e6, symbol: "M"},
{value: 1e3, symbol: "k"},
{value: 1, symbol: ""},
}
func formatNumber(num int) string {
goodUnit := unit{}
for _, u := range ranges {
if int64(num) >= u.value {
goodUnit = u
break
}
}
if goodUnit.value == 1 {
return fmt.Sprintf("%d%s", num, goodUnit.symbol)
}
res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
}

View file

@ -1,85 +0,0 @@
package main
import (
"fmt"
"slices"
"strings"
"github.com/agext/levenshtein"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
// suggestNearestMessage returns a message with the most similar item name, if one is found
func suggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string {
const maxDistance = 7
score := 100
nearest := ""
for _, item := range hub.GetItemMap(itemType) {
d := levenshtein.Distance(itemName, item.Name, nil)
if d < score {
score = d
nearest = item.Name
}
}
msg := fmt.Sprintf("can't find '%s' in %s", itemName, itemType)
if score < maxDistance {
msg += fmt.Sprintf(", did you mean '%s'?", nearest)
}
return msg
}
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
hub, err := require.Hub(csConfig, nil, nil)
if err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
for _, item := range hub.GetItemMap(itemType) {
if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
comp = append(comp, item.Name)
}
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
hub, err := require.Hub(csConfig, nil, nil)
if err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
items, err := hub.GetInstalledNamesByType(itemType)
if err != nil {
cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
if toComplete != "" {
for _, item := range items {
if strings.Contains(item, toComplete) {
comp = append(comp, item)
}
}
} else {
comp = items
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}

View file

@ -1,546 +0,0 @@
package main
import (
"errors"
"fmt"
"os"
"strings"
"github.com/fatih/color"
"github.com/hexops/gotextdiff"
"github.com/hexops/gotextdiff/myers"
"github.com/hexops/gotextdiff/span"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/coalesce"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
type cliHelp struct {
// Example is required, the others have a default value
// generated from the item type
use string
short string
long string
example string
}
type cliItem struct {
cfg configGetter
name string // plural, as used in the hub index
singular string
oneOrMore string // parenthetical pluralizaion: "parser(s)"
help cliHelp
installHelp cliHelp
removeHelp cliHelp
upgradeHelp cliHelp
inspectHelp cliHelp
inspectDetail func(item *cwhub.Item) error
listHelp cliHelp
}
func (cli cliItem) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: coalesce.String(cli.help.use, fmt.Sprintf("%s <action> [item]...", cli.name)),
Short: coalesce.String(cli.help.short, fmt.Sprintf("Manage hub %s", cli.name)),
Long: cli.help.long,
Example: cli.help.example,
Args: cobra.MinimumNArgs(1),
Aliases: []string{cli.singular},
DisableAutoGenTag: true,
}
cmd.AddCommand(cli.newInstallCmd())
cmd.AddCommand(cli.newRemoveCmd())
cmd.AddCommand(cli.newUpgradeCmd())
cmd.AddCommand(cli.newInspectCmd())
cmd.AddCommand(cli.newListCmd())
return cmd
}
func (cli cliItem) install(args []string, downloadOnly bool, force bool, ignoreError bool) error {
cfg := cli.cfg()
hub, err := require.Hub(cfg, require.RemoteHub(cfg), log.StandardLogger())
if err != nil {
return err
}
for _, name := range args {
item := hub.GetItem(cli.name, name)
if item == nil {
msg := suggestNearestMessage(hub, cli.name, name)
if !ignoreError {
return errors.New(msg)
}
log.Errorf(msg)
continue
}
if err := item.Install(force, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", item.Name, err)
}
log.Errorf("Error while installing '%s': %s", item.Name, err)
}
}
log.Infof(ReloadMessage())
return nil
}
func (cli cliItem) newInstallCmd() *cobra.Command {
var (
downloadOnly bool
force bool
ignoreError bool
)
cmd := &cobra.Command{
Use: coalesce.String(cli.installHelp.use, "install [item]..."),
Short: coalesce.String(cli.installHelp.short, fmt.Sprintf("Install given %s", cli.oneOrMore)),
Long: coalesce.String(cli.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", cli.name)),
Example: cli.installHelp.example,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cli.name, args, toComplete)
},
RunE: func(_ *cobra.Command, args []string) error {
return cli.install(args, downloadOnly, force, ignoreError)
},
}
flags := cmd.Flags()
flags.BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
flags.BoolVar(&force, "force", false, "Force install: overwrite tainted and outdated files")
flags.BoolVar(&ignoreError, "ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", cli.name))
return cmd
}
// return the names of the installed parents of an item, used to check if we can remove it
func istalledParentNames(item *cwhub.Item) []string {
ret := make([]string, 0)
for _, parent := range item.Ancestors() {
if parent.State.Installed {
ret = append(ret, parent.Name)
}
}
return ret
}
func (cli cliItem) remove(args []string, purge bool, force bool, all bool) error {
hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger())
if err != nil {
return err
}
if all {
getter := hub.GetInstalledItemsByType
if purge {
getter = hub.GetItemsByType
}
items, err := getter(cli.name)
if err != nil {
return err
}
removed := 0
for _, item := range items {
didRemove, err := item.Remove(purge, force)
if err != nil {
return err
}
if didRemove {
log.Infof("Removed %s", item.Name)
removed++
}
}
log.Infof("Removed %d %s", removed, cli.name)
if removed > 0 {
log.Infof(ReloadMessage())
}
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one %s to remove or '--all'", cli.singular)
}
removed := 0
for _, itemName := range args {
item := hub.GetItem(cli.name, itemName)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", itemName, cli.name)
}
parents := istalledParentNames(item)
if !force && len(parents) > 0 {
log.Warningf("%s belongs to collections: %s", item.Name, parents)
log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, cli.singular)
continue
}
didRemove, err := item.Remove(purge, force)
if err != nil {
return err
}
if didRemove {
log.Infof("Removed %s", item.Name)
removed++
}
}
log.Infof("Removed %d %s", removed, cli.name)
if removed > 0 {
log.Infof(ReloadMessage())
}
return nil
}
func (cli cliItem) newRemoveCmd() *cobra.Command {
var (
purge bool
force bool
all bool
)
cmd := &cobra.Command{
Use: coalesce.String(cli.removeHelp.use, "remove [item]..."),
Short: coalesce.String(cli.removeHelp.short, fmt.Sprintf("Remove given %s", cli.oneOrMore)),
Long: coalesce.String(cli.removeHelp.long, fmt.Sprintf("Remove one or more %s", cli.name)),
Example: cli.removeHelp.example,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cli.name, args, toComplete)
},
RunE: func(_ *cobra.Command, args []string) error {
return cli.remove(args, purge, force, all)
},
}
flags := cmd.Flags()
flags.BoolVar(&purge, "purge", false, "Delete source file too")
flags.BoolVar(&force, "force", false, "Force remove: remove tainted and outdated files")
flags.BoolVar(&all, "all", false, fmt.Sprintf("Remove all the %s", cli.name))
return cmd
}
func (cli cliItem) upgrade(args []string, force bool, all bool) error {
cfg := cli.cfg()
hub, err := require.Hub(cfg, require.RemoteHub(cfg), log.StandardLogger())
if err != nil {
return err
}
if all {
items, err := hub.GetInstalledItemsByType(cli.name)
if err != nil {
return err
}
updated := 0
for _, item := range items {
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
updated++
}
}
log.Infof("Updated %d %s", updated, cli.name)
if updated > 0 {
log.Infof(ReloadMessage())
}
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one %s to upgrade or '--all'", cli.singular)
}
updated := 0
for _, itemName := range args {
item := hub.GetItem(cli.name, itemName)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", itemName, cli.name)
}
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
log.Infof("Updated %s", item.Name)
updated++
}
}
if updated > 0 {
log.Infof(ReloadMessage())
}
return nil
}
func (cli cliItem) newUpgradeCmd() *cobra.Command {
var (
all bool
force bool
)
cmd := &cobra.Command{
Use: coalesce.String(cli.upgradeHelp.use, "upgrade [item]..."),
Short: coalesce.String(cli.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", cli.oneOrMore)),
Long: coalesce.String(cli.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", cli.name)),
Example: cli.upgradeHelp.example,
DisableAutoGenTag: true,
ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cli.name, args, toComplete)
},
RunE: func(_ *cobra.Command, args []string) error {
return cli.upgrade(args, force, all)
},
}
flags := cmd.Flags()
flags.BoolVarP(&all, "all", "a", false, fmt.Sprintf("Upgrade all the %s", cli.name))
flags.BoolVar(&force, "force", false, "Force upgrade: overwrite tainted and outdated files")
return cmd
}
func (cli cliItem) inspect(args []string, url string, diff bool, rev bool, noMetrics bool) error {
cfg := cli.cfg()
if rev && !diff {
return errors.New("--rev can only be used with --diff")
}
if url != "" {
cfg.Cscli.PrometheusUrl = url
}
remote := (*cwhub.RemoteHubCfg)(nil)
if diff {
remote = require.RemoteHub(cfg)
}
hub, err := require.Hub(cfg, remote, log.StandardLogger())
if err != nil {
return err
}
for _, name := range args {
item := hub.GetItem(cli.name, name)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", name, cli.name)
}
if diff {
fmt.Println(cli.whyTainted(hub, item, rev))
continue
}
if err = inspectItem(item, !noMetrics); err != nil {
return err
}
if cli.inspectDetail != nil {
if err = cli.inspectDetail(item); err != nil {
return err
}
}
}
return nil
}
func (cli cliItem) newInspectCmd() *cobra.Command {
var (
url string
diff bool
rev bool
noMetrics bool
)
cmd := &cobra.Command{
Use: coalesce.String(cli.inspectHelp.use, "inspect [item]..."),
Short: coalesce.String(cli.inspectHelp.short, fmt.Sprintf("Inspect given %s", cli.oneOrMore)),
Long: coalesce.String(cli.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", cli.name)),
Example: cli.inspectHelp.example,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cli.name, args, toComplete)
},
RunE: func(_ *cobra.Command, args []string) error {
return cli.inspect(args, url, diff, rev, noMetrics)
},
}
flags := cmd.Flags()
flags.StringVarP(&url, "url", "u", "", "Prometheus url")
flags.BoolVar(&diff, "diff", false, "Show diff with latest version (for tainted items)")
flags.BoolVar(&rev, "rev", false, "Reverse diff output")
flags.BoolVar(&noMetrics, "no-metrics", false, "Don't show metrics (when cscli.output=human)")
return cmd
}
func (cli cliItem) list(args []string, all bool) error {
hub, err := require.Hub(cli.cfg(), nil, log.StandardLogger())
if err != nil {
return err
}
items := make(map[string][]*cwhub.Item)
items[cli.name], err = selectItems(hub, cli.name, args, !all)
if err != nil {
return err
}
if err = listItems(color.Output, []string{cli.name}, items, false); err != nil {
return err
}
return nil
}
func (cli cliItem) newListCmd() *cobra.Command {
var all bool
cmd := &cobra.Command{
Use: coalesce.String(cli.listHelp.use, "list [item... | -a]"),
Short: coalesce.String(cli.listHelp.short, fmt.Sprintf("List %s", cli.oneOrMore)),
Long: coalesce.String(cli.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", cli.name)),
Example: cli.listHelp.example,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
return cli.list(args, all)
},
}
flags := cmd.Flags()
flags.BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmd
}
// return the diff between the installed version and the latest version
func (cli cliItem) itemDiff(item *cwhub.Item, reverse bool) (string, error) {
if !item.State.Installed {
return "", fmt.Errorf("'%s' is not installed", item.FQName())
}
latestContent, remoteURL, err := item.FetchLatest()
if err != nil {
return "", err
}
localContent, err := os.ReadFile(item.State.LocalPath)
if err != nil {
return "", fmt.Errorf("while reading %s: %w", item.State.LocalPath, err)
}
file1 := item.State.LocalPath
file2 := remoteURL
content1 := string(localContent)
content2 := string(latestContent)
if reverse {
file1, file2 = file2, file1
content1, content2 = content2, content1
}
edits := myers.ComputeEdits(span.URIFromPath(file1), content1, content2)
diff := gotextdiff.ToUnified(file1, file2, content1, edits)
return fmt.Sprintf("%s", diff), nil
}
func (cli cliItem) whyTainted(hub *cwhub.Hub, item *cwhub.Item, reverse bool) string {
if !item.State.Installed {
return fmt.Sprintf("# %s is not installed", item.FQName())
}
if !item.State.Tainted {
return fmt.Sprintf("# %s is not tainted", item.FQName())
}
if len(item.State.TaintedBy) == 0 {
return fmt.Sprintf("# %s is tainted but we don't know why. please report this as a bug", item.FQName())
}
ret := []string{
fmt.Sprintf("# Let's see why %s is tainted.", item.FQName()),
}
for _, fqsub := range item.State.TaintedBy {
ret = append(ret, fmt.Sprintf("\n-> %s\n", fqsub))
sub, err := hub.GetItemFQ(fqsub)
if err != nil {
ret = append(ret, err.Error())
}
diff, err := cli.itemDiff(sub, reverse)
if err != nil {
ret = append(ret, err.Error())
}
if diff != "" {
ret = append(ret, diff)
} else if len(sub.State.TaintedBy) > 0 {
taintList := strings.Join(sub.State.TaintedBy, ", ")
if sub.FQName() == taintList {
// hack: avoid message "item is tainted by itself"
continue
}
ret = append(ret, fmt.Sprintf("# %s is tainted by %s", sub.FQName(), taintList))
}
}
return strings.Join(ret, "\n")
}

View file

@ -1,183 +0,0 @@
package main
import (
"encoding/csv"
"encoding/json"
"fmt"
"io"
"os"
"path/filepath"
"slices"
"strings"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
// selectItems returns a slice of items of a given type, selected by name and sorted by case-insensitive name
func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]*cwhub.Item, error) {
itemNames := hub.GetNamesByType(itemType)
notExist := []string{}
if len(args) > 0 {
for _, arg := range args {
if !slices.Contains(itemNames, arg) {
notExist = append(notExist, arg)
}
}
}
if len(notExist) > 0 {
return nil, fmt.Errorf("item(s) '%s' not found in %s", strings.Join(notExist, ", "), itemType)
}
if len(args) > 0 {
itemNames = args
installedOnly = false
}
items := make([]*cwhub.Item, 0, len(itemNames))
for _, itemName := range itemNames {
item := hub.GetItem(itemType, itemName)
if installedOnly && !item.State.Installed {
continue
}
items = append(items, item)
}
cwhub.SortItemSlice(items)
return items, nil
}
func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item, omitIfEmpty bool) error {
switch csConfig.Cscli.Output {
case "human":
nothingToDisplay := true
for _, itemType := range itemTypes {
if omitIfEmpty && len(items[itemType]) == 0 {
continue
}
listHubItemTable(out, "\n"+strings.ToUpper(itemType), items[itemType])
nothingToDisplay = false
}
if nothingToDisplay {
fmt.Println("No items to display")
}
case "json":
type itemHubStatus struct {
Name string `json:"name"`
LocalVersion string `json:"local_version"`
LocalPath string `json:"local_path"`
Description string `json:"description"`
UTF8Status string `json:"utf8_status"`
Status string `json:"status"`
}
hubStatus := make(map[string][]itemHubStatus)
for _, itemType := range itemTypes {
// empty slice in case there are no items of this type
hubStatus[itemType] = make([]itemHubStatus, len(items[itemType]))
for i, item := range items[itemType] {
status := item.State.Text()
statusEmo := item.State.Emoji()
hubStatus[itemType][i] = itemHubStatus{
Name: item.Name,
LocalVersion: item.State.LocalVersion,
LocalPath: item.State.LocalPath,
Description: item.Description,
Status: status,
UTF8Status: fmt.Sprintf("%v %s", statusEmo, status),
}
}
}
x, err := json.MarshalIndent(hubStatus, "", " ")
if err != nil {
return fmt.Errorf("failed to unmarshal: %w", err)
}
out.Write(x)
case "raw":
csvwriter := csv.NewWriter(out)
header := []string{"name", "status", "version", "description"}
if len(itemTypes) > 1 {
header = append(header, "type")
}
if err := csvwriter.Write(header); err != nil {
return fmt.Errorf("failed to write header: %w", err)
}
for _, itemType := range itemTypes {
for _, item := range items[itemType] {
row := []string{
item.Name,
item.State.Text(),
item.State.LocalVersion,
item.Description,
}
if len(itemTypes) > 1 {
row = append(row, itemType)
}
if err := csvwriter.Write(row); err != nil {
return fmt.Errorf("failed to write raw output: %w", err)
}
}
}
csvwriter.Flush()
}
return nil
}
func inspectItem(item *cwhub.Item, showMetrics bool) error {
switch csConfig.Cscli.Output {
case "human", "raw":
enc := yaml.NewEncoder(os.Stdout)
enc.SetIndent(2)
if err := enc.Encode(item); err != nil {
return fmt.Errorf("unable to encode item: %w", err)
}
case "json":
b, err := json.MarshalIndent(*item, "", " ")
if err != nil {
return fmt.Errorf("unable to marshal item: %w", err)
}
fmt.Print(string(b))
}
if csConfig.Cscli.Output != "human" {
return nil
}
if item.State.Tainted {
fmt.Println()
fmt.Printf(`This item is tainted. Use "%s %s inspect --diff %s" to see why.`, filepath.Base(os.Args[0]), item.Type, item.Name)
fmt.Println()
}
if showMetrics {
fmt.Printf("\nCurrent metrics: \n")
if err := ShowMetrics(item); err != nil {
return err
}
}
return nil
}

View file

@ -2,403 +2,337 @@ package main
import (
"context"
"errors"
"fmt"
"net/url"
"os"
"slices"
"sort"
"strings"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
const LAPIURLPrefix = "v1"
var LAPIURLPrefix string = "v1"
type cliLapi struct {
cfg configGetter
}
func runLapiStatus(cmd *cobra.Command, args []string) error {
var err error
func NewCLILapi(cfg configGetter) *cliLapi {
return &cliLapi{
cfg: cfg,
}
}
func (cli *cliLapi) status() error {
cfg := cli.cfg()
password := strfmt.Password(cfg.API.Client.Credentials.Password)
login := cfg.API.Client.Credentials.Login
origURL := cfg.API.Client.Credentials.URL
apiURL, err := url.Parse(origURL)
password := strfmt.Password(csConfig.API.Client.Credentials.Password)
apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL)
login := csConfig.API.Client.Credentials.Login
if err != nil {
return fmt.Errorf("parsing api url: %w", err)
log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
}
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
hub, err := require.Hub(cfg, nil, nil)
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to load hub index : %s", err)
}
scenarios, err := cwhub.GetInstalledScenariosAsString()
if err != nil {
return err
log.Fatalf("failed to get scenarios : %s", err)
}
scenarios, err := hub.GetInstalledNamesByType(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("failed to get scenarios: %w", err)
}
Client, err = apiclient.NewDefaultClient(apiURL,
Client, err = apiclient.NewDefaultClient(apiurl,
LAPIURLPrefix,
fmt.Sprintf("crowdsec/%s", version.String()),
fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
nil)
if err != nil {
return fmt.Errorf("init default client: %w", err)
log.Fatalf("init default client: %s", err)
}
t := models.WatcherAuthRequest{
MachineID: &login,
Password: &password,
Scenarios: scenarios,
}
log.Infof("Loaded credentials from %s", cfg.API.Client.CredentialsFilePath)
// use the original string because apiURL would print 'http://unix/'
log.Infof("Trying to authenticate with username %s on %s", login, origURL)
log.Infof("Loaded credentials from %s", csConfig.API.Client.CredentialsFilePath)
log.Infof("Trying to authenticate with username %s on %s", login, apiurl)
_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
if err != nil {
return fmt.Errorf("failed to authenticate to Local API (LAPI): %w", err)
log.Fatalf("Failed to authenticate to Local API (LAPI) : %s", err)
} else {
log.Infof("You can successfully interact with Local API (LAPI)")
}
log.Infof("You can successfully interact with Local API (LAPI)")
return nil
}
func (cli *cliLapi) register(apiURL string, outputFile string, machine string) error {
func runLapiRegister(cmd *cobra.Command, args []string) error {
var err error
lapiUser := machine
cfg := cli.cfg()
flags := cmd.Flags()
apiURL, err := flags.GetString("url")
if err != nil {
return err
}
outputFile, err := flags.GetString("file")
if err != nil {
return err
}
lapiUser, err := flags.GetString("machine")
if err != nil {
return err
}
if lapiUser == "" {
lapiUser, err = generateID("")
if err != nil {
return fmt.Errorf("unable to generate machine id: %w", err)
log.Fatalf("unable to generate machine id: %s", err)
}
}
password := strfmt.Password(generatePassword(passwordLength))
apiurl, err := prepareAPIURL(cfg.API.Client, apiURL)
if err != nil {
return fmt.Errorf("parsing api url: %w", err)
if apiURL == "" {
if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" {
apiURL = csConfig.API.Client.Credentials.URL
} else {
log.Fatalf("No Local API URL. Please provide it in your configuration or with the -u parameter")
}
}
/*URL needs to end with /, but user doesn't care*/
if !strings.HasSuffix(apiURL, "/") {
apiURL += "/"
}
/*URL needs to start with http://, but user doesn't care*/
if !strings.HasPrefix(apiURL, "http://") && !strings.HasPrefix(apiURL, "https://") {
apiURL = "http://" + apiURL
}
apiurl, err := url.Parse(apiURL)
if err != nil {
log.Fatalf("parsing api url: %s", err)
}
_, err = apiclient.RegisterClient(&apiclient.Config{
MachineID: lapiUser,
Password: password,
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
URL: apiurl,
VersionPrefix: LAPIURLPrefix,
}, nil)
if err != nil {
return fmt.Errorf("api client register: %w", err)
log.Fatalf("api client register: %s", err)
}
log.Printf("Successfully registered to Local API (LAPI)")
var dumpFile string
if outputFile != "" {
dumpFile = outputFile
} else if cfg.API.Client.CredentialsFilePath != "" {
dumpFile = cfg.API.Client.CredentialsFilePath
} else if csConfig.API.Client.CredentialsFilePath != "" {
dumpFile = csConfig.API.Client.CredentialsFilePath
} else {
dumpFile = ""
}
apiCfg := csconfig.ApiCredentialsCfg{
Login: lapiUser,
Password: password.String(),
URL: apiURL,
}
apiConfigDump, err := yaml.Marshal(apiCfg)
if err != nil {
return fmt.Errorf("unable to marshal api credentials: %w", err)
log.Fatalf("unable to marshal api credentials: %s", err)
}
if dumpFile != "" {
err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
err = os.WriteFile(dumpFile, apiConfigDump, 0644)
if err != nil {
return fmt.Errorf("write api credentials to '%s' failed: %w", dumpFile, err)
log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
}
log.Printf("Local API credentials written to '%s'", dumpFile)
log.Printf("Local API credentials dumped to '%s'", dumpFile)
} else {
fmt.Printf("%s\n", string(apiConfigDump))
}
log.Warning(ReloadMessage())
return nil
}
// prepareAPIURL checks/fixes a LAPI connection url (http, https or socket) and returns an URL struct
func prepareAPIURL(clientCfg *csconfig.LocalApiClientCfg, apiURL string) (*url.URL, error) {
if apiURL == "" {
if clientCfg == nil || clientCfg.Credentials == nil || clientCfg.Credentials.URL == "" {
return nil, errors.New("no Local API URL. Please provide it in your configuration or with the -u parameter")
}
apiURL = clientCfg.Credentials.URL
}
// URL needs to end with /, but user doesn't care
if !strings.HasSuffix(apiURL, "/") {
apiURL += "/"
}
// URL needs to start with http://, but user doesn't care
if !strings.HasPrefix(apiURL, "http://") && !strings.HasPrefix(apiURL, "https://") && !strings.HasPrefix(apiURL, "/") {
apiURL = "http://" + apiURL
}
return url.Parse(apiURL)
}
func (cli *cliLapi) newStatusCmd() *cobra.Command {
func NewLapiStatusCmd() *cobra.Command {
cmdLapiStatus := &cobra.Command{
Use: "status",
Short: "Check authentication to Local API (LAPI)",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.status()
},
RunE: runLapiStatus,
}
return cmdLapiStatus
}
func (cli *cliLapi) newRegisterCmd() *cobra.Command {
var (
apiURL string
outputFile string
machine string
)
cmd := &cobra.Command{
func NewLapiRegisterCmd() *cobra.Command {
cmdLapiRegister := &cobra.Command{
Use: "register",
Short: "Register a machine to Local API (LAPI)",
Long: `Register your machine to the Local API (LAPI).
Long: `Register you machine to the Local API (LAPI).
Keep in mind the machine needs to be validated by an administrator on LAPI side to be effective.`,
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.register(apiURL, outputFile, machine)
},
RunE: runLapiRegister,
}
flags := cmd.Flags()
flags.StringVarP(&apiURL, "url", "u", "", "URL of the API (ie. http://127.0.0.1)")
flags.StringVarP(&outputFile, "file", "f", "", "output file destination")
flags.StringVar(&machine, "machine", "", "Name of the machine to register with")
flags := cmdLapiRegister.Flags()
flags.StringP("url", "u", "", "URL of the API (ie. http://127.0.0.1)")
flags.StringP("file", "f", "", "output file destination")
flags.String("machine", "", "Name of the machine to register with")
return cmd
return cmdLapiRegister
}
func (cli *cliLapi) NewCommand() *cobra.Command {
cmd := &cobra.Command{
func NewLapiCmd() *cobra.Command {
var cmdLapi = &cobra.Command{
Use: "lapi [action]",
Short: "Manage interaction with Local API (LAPI)",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
if err := cli.cfg().LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIClient(); err != nil {
return errors.Wrap(err, "loading api client")
}
return nil
},
}
cmd.AddCommand(cli.newRegisterCmd())
cmd.AddCommand(cli.newStatusCmd())
cmd.AddCommand(cli.newContextCmd())
cmdLapi.AddCommand(NewLapiRegisterCmd())
cmdLapi.AddCommand(NewLapiStatusCmd())
cmdLapi.AddCommand(NewLapiContextCmd())
return cmd
return cmdLapi
}
func (cli *cliLapi) addContext(key string, values []string) error {
cfg := cli.cfg()
func NewLapiContextCmd() *cobra.Command {
cmdContext := &cobra.Command{
Use: "context [command]",
Short: "Manage context to send with alerts",
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadCrowdsec(); err != nil {
fileNotFoundMessage := fmt.Sprintf("failed to open context file: open %s: no such file or directory", csConfig.Crowdsec.ConsoleContextPath)
if err.Error() != fileNotFoundMessage {
log.Fatalf("Unable to load CrowdSec Agent: %s", err)
}
}
if csConfig.DisableAgent {
log.Fatalf("Agent is disabled and lapi context can only be used on the agent")
}
if err := alertcontext.ValidateContextExpr(key, values); err != nil {
return fmt.Errorf("invalid context configuration: %w", err)
return nil
},
Run: func(cmd *cobra.Command, args []string) {
printHelp(cmd)
},
}
if _, ok := cfg.Crowdsec.ContextToSend[key]; !ok {
cfg.Crowdsec.ContextToSend[key] = make([]string, 0)
log.Infof("key '%s' added", key)
}
data := cfg.Crowdsec.ContextToSend[key]
for _, val := range values {
if !slices.Contains(data, val) {
log.Infof("value '%s' added to key '%s'", val, key)
data = append(data, val)
}
cfg.Crowdsec.ContextToSend[key] = data
}
if err := cfg.Crowdsec.DumpContextConfigFile(); err != nil {
return err
}
return nil
}
func (cli *cliLapi) newContextAddCmd() *cobra.Command {
var (
keyToAdd string
valuesToAdd []string
)
cmd := &cobra.Command{
var keyToAdd string
var valuesToAdd []string
cmdContextAdd := &cobra.Command{
Use: "add",
Short: "Add context to send with alerts. You must specify the output key with the expr value you want",
Example: `cscli lapi context add --key source_ip --value evt.Meta.source_ip
cscli lapi context add --key file_source --value evt.Line.Src
cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
`,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
hub, err := require.Hub(cli.cfg(), nil, nil)
if err != nil {
return err
Run: func(cmd *cobra.Command, args []string) {
if err := alertcontext.ValidateContextExpr(keyToAdd, valuesToAdd); err != nil {
log.Fatalf("invalid context configuration :%s", err)
}
if err = alertcontext.LoadConsoleContext(cli.cfg(), hub); err != nil {
return fmt.Errorf("while loading context: %w", err)
if _, ok := csConfig.Crowdsec.ContextToSend[keyToAdd]; !ok {
csConfig.Crowdsec.ContextToSend[keyToAdd] = make([]string, 0)
log.Infof("key '%s' added", keyToAdd)
}
if keyToAdd != "" {
if err := cli.addContext(keyToAdd, valuesToAdd); err != nil {
return err
data := csConfig.Crowdsec.ContextToSend[keyToAdd]
for _, val := range valuesToAdd {
if !inSlice(val, data) {
log.Infof("value '%s' added to key '%s'", val, keyToAdd)
data = append(data, val)
}
return nil
csConfig.Crowdsec.ContextToSend[keyToAdd] = data
}
for _, v := range valuesToAdd {
keySlice := strings.Split(v, ".")
key := keySlice[len(keySlice)-1]
value := []string{v}
if err := cli.addContext(key, value); err != nil {
return err
}
if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
log.Fatalf(err.Error())
}
return nil
},
}
cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
cmdContextAdd.Flags().StringSliceVar(&valuesToAdd, "value", []string{}, "The expr fields to associate with the key")
cmdContextAdd.MarkFlagRequired("key")
cmdContextAdd.MarkFlagRequired("value")
cmdContext.AddCommand(cmdContextAdd)
flags := cmd.Flags()
flags.StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
flags.StringSliceVar(&valuesToAdd, "value", []string{}, "The expr fields to associate with the key")
cmd.MarkFlagRequired("value")
return cmd
}
func (cli *cliLapi) newContextStatusCmd() *cobra.Command {
cmd := &cobra.Command{
cmdContextStatus := &cobra.Command{
Use: "status",
Short: "List context to send with alerts",
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
hub, err := require.Hub(cfg, nil, nil)
if err != nil {
return err
}
if err = alertcontext.LoadConsoleContext(cfg, hub); err != nil {
return fmt.Errorf("while loading context: %w", err)
}
if len(cfg.Crowdsec.ContextToSend) == 0 {
Run: func(cmd *cobra.Command, args []string) {
if len(csConfig.Crowdsec.ContextToSend) == 0 {
fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.")
return nil
return
}
dump, err := yaml.Marshal(cfg.Crowdsec.ContextToSend)
dump, err := yaml.Marshal(csConfig.Crowdsec.ContextToSend)
if err != nil {
return fmt.Errorf("unable to show context status: %w", err)
log.Fatalf("unable to show context status: %s", err)
}
fmt.Print(string(dump))
fmt.Println(string(dump))
return nil
},
}
cmdContext.AddCommand(cmdContextStatus)
return cmd
}
func (cli *cliLapi) newContextDetectCmd() *cobra.Command {
var detectAll bool
cmd := &cobra.Command{
cmdContextDetect := &cobra.Command{
Use: "detect",
Short: "Detect available fields from the installed parsers",
Example: `cscli lapi context detect --all
cscli lapi context detect crowdsecurity/sshd-logs
`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
cfg := cli.cfg()
Run: func(cmd *cobra.Command, args []string) {
var err error
if !detectAll && len(args) == 0 {
log.Infof("Please provide parsers to detect or --all flag.")
printHelp(cmd)
}
// to avoid all the log.Info from the loaders functions
log.SetLevel(log.WarnLevel)
log.SetLevel(log.ErrorLevel)
if err := exprhelpers.Init(nil); err != nil {
return fmt.Errorf("failed to init expr helpers: %w", err)
}
hub, err := require.Hub(cfg, nil, nil)
err = exprhelpers.Init(nil)
if err != nil {
return err
log.Fatalf("Failed to init expr helpers : %s", err)
}
csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(cfg, csParsers); err != nil {
return fmt.Errorf("unable to load parsers: %w", err)
// Populate cwhub package tools
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to load hub index : %s", err)
}
csParsers := parser.NewParsers()
if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil {
log.Fatalf("unable to load parsers: %s", err)
}
fieldByParsers := make(map[string][]string)
for _, node := range csParsers.Nodes {
if !detectAll && !slices.Contains(args, node.Name) {
if !detectAll && !inSlice(node.Name, args) {
continue
}
if !detectAll {
@ -409,10 +343,11 @@ cscli lapi context detect crowdsecurity/sshd-logs
subNodeFields := detectSubNode(node, *csParsers.Ctx)
for _, field := range subNodeFields {
if !slices.Contains(fieldByParsers[node.Name], field) {
if !inSlice(field, fieldByParsers[node.Name]) {
fieldByParsers[node.Name] = append(fieldByParsers[node.Name], field)
}
}
}
fmt.Printf("Acquisition :\n\n")
@ -445,90 +380,85 @@ cscli lapi context detect crowdsecurity/sshd-logs
log.Errorf("parser '%s' not found, can't detect fields", parserNotFound)
}
}
return nil
},
}
cmd.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser")
cmdContextDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser")
cmdContext.AddCommand(cmdContextDetect)
return cmd
}
func (cli *cliLapi) newContextDeleteCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "delete",
var keysToDelete []string
var valuesToDelete []string
cmdContextDelete := &cobra.Command{
Use: "delete",
Short: "Delete context to send with alerts",
Example: `cscli lapi context delete --key source_ip
cscli lapi context delete --value evt.Line.Src
`,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
filePath := cli.cfg().Crowdsec.ConsoleContextPath
if filePath == "" {
filePath = "the context file"
Run: func(cmd *cobra.Command, args []string) {
if len(keysToDelete) == 0 && len(valuesToDelete) == 0 {
log.Fatalf("please provide at least a key or a value to delete")
}
fmt.Printf("Command 'delete' is deprecated, please manually edit %s.", filePath)
return nil
},
}
return cmd
}
func (cli *cliLapi) newContextCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "context [command]",
Short: "Manage context to send with alerts",
DisableAutoGenTag: true,
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := cfg.LoadCrowdsec(); err != nil {
fileNotFoundMessage := fmt.Sprintf("failed to open context file: open %s: no such file or directory", cfg.Crowdsec.ConsoleContextPath)
if err.Error() != fileNotFoundMessage {
return fmt.Errorf("unable to load CrowdSec agent configuration: %w", err)
for _, key := range keysToDelete {
if _, ok := csConfig.Crowdsec.ContextToSend[key]; ok {
delete(csConfig.Crowdsec.ContextToSend, key)
log.Infof("key '%s' has been removed", key)
} else {
log.Warningf("key '%s' doesn't exist", key)
}
}
if cfg.DisableAgent {
return errors.New("agent is disabled and lapi context can only be used on the agent")
for _, value := range valuesToDelete {
valueFound := false
for key, context := range csConfig.Crowdsec.ContextToSend {
if inSlice(value, context) {
valueFound = true
csConfig.Crowdsec.ContextToSend[key] = removeFromSlice(value, context)
log.Infof("value '%s' has been removed from key '%s'", value, key)
}
if len(csConfig.Crowdsec.ContextToSend[key]) == 0 {
delete(csConfig.Crowdsec.ContextToSend, key)
}
}
if !valueFound {
log.Warningf("value '%s' not found", value)
}
}
if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
log.Fatalf(err.Error())
}
return nil
},
Run: func(cmd *cobra.Command, _ []string) {
printHelp(cmd)
},
}
cmdContextDelete.Flags().StringSliceVarP(&keysToDelete, "key", "k", []string{}, "The keys to delete")
cmdContextDelete.Flags().StringSliceVar(&valuesToDelete, "value", []string{}, "The expr fields to delete")
cmdContext.AddCommand(cmdContextDelete)
cmd.AddCommand(cli.newContextAddCmd())
cmd.AddCommand(cli.newContextStatusCmd())
cmd.AddCommand(cli.newContextDetectCmd())
cmd.AddCommand(cli.newContextDeleteCmd())
return cmd
return cmdContext
}
func detectStaticField(grokStatics []parser.ExtraField) []string {
func detectStaticField(GrokStatics []types.ExtraField) []string {
ret := make([]string, 0)
for _, static := range grokStatics {
for _, static := range GrokStatics {
if static.Parsed != "" {
fieldName := fmt.Sprintf("evt.Parsed.%s", static.Parsed)
if !slices.Contains(ret, fieldName) {
if !inSlice(fieldName, ret) {
ret = append(ret, fieldName)
}
}
if static.Meta != "" {
fieldName := fmt.Sprintf("evt.Meta.%s", static.Meta)
if !slices.Contains(ret, fieldName) {
if !inSlice(fieldName, ret) {
ret = append(ret, fieldName)
}
}
if static.TargetByName != "" {
fieldName := static.TargetByName
if !strings.HasPrefix(fieldName, "evt.") {
fieldName = "evt." + fieldName
}
if !slices.Contains(ret, fieldName) {
if !inSlice(fieldName, ret) {
ret = append(ret, fieldName)
}
}
@ -538,12 +468,11 @@ func detectStaticField(grokStatics []parser.ExtraField) []string {
}
func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
ret := make([]string, 0)
var ret = make([]string, 0)
if node.Grok.RunTimeRegexp != nil {
for _, capturedField := range node.Grok.RunTimeRegexp.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
if !slices.Contains(ret, fieldName) {
if !inSlice(fieldName, ret) {
ret = append(ret, fieldName)
}
}
@ -551,13 +480,13 @@ func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
if node.Grok.RegexpName != "" {
grokCompiled, err := parserCTX.Grok.Get(node.Grok.RegexpName)
// ignore error (parser does not exist?)
if err == nil {
for _, capturedField := range grokCompiled.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
if !slices.Contains(ret, fieldName) {
ret = append(ret, fieldName)
}
if err != nil {
log.Warningf("Can't get subgrok: %s", err)
}
for _, capturedField := range grokCompiled.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
if !inSlice(fieldName, ret) {
ret = append(ret, fieldName)
}
}
}
@ -565,7 +494,7 @@ func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
if len(node.Grok.Statics) > 0 {
staticsField := detectStaticField(node.Grok.Statics)
for _, staticField := range staticsField {
if !slices.Contains(ret, staticField) {
if !inSlice(staticField, ret) {
ret = append(ret, staticField)
}
}
@ -574,7 +503,7 @@ func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
if len(node.Statics) > 0 {
staticsField := detectStaticField(node.Statics)
for _, staticField := range staticsField {
if !slices.Contains(ret, staticField) {
if !inSlice(staticField, ret) {
ret = append(ret, staticField)
}
}
@ -584,27 +513,26 @@ func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
}
func detectSubNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
ret := make([]string, 0)
var ret = make([]string, 0)
for _, subnode := range node.LeavesNodes {
if subnode.Grok.RunTimeRegexp != nil {
for _, capturedField := range subnode.Grok.RunTimeRegexp.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
if !slices.Contains(ret, fieldName) {
if !inSlice(fieldName, ret) {
ret = append(ret, fieldName)
}
}
}
if subnode.Grok.RegexpName != "" {
grokCompiled, err := parserCTX.Grok.Get(subnode.Grok.RegexpName)
if err == nil {
// ignore error (parser does not exist?)
for _, capturedField := range grokCompiled.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
if !slices.Contains(ret, fieldName) {
ret = append(ret, fieldName)
}
if err != nil {
log.Warningf("Can't get subgrok: %s", err)
}
for _, capturedField := range grokCompiled.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
if !inSlice(fieldName, ret) {
ret = append(ret, fieldName)
}
}
}
@ -612,7 +540,7 @@ func detectSubNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
if len(subnode.Grok.Statics) > 0 {
staticsField := detectStaticField(subnode.Grok.Statics)
for _, staticField := range staticsField {
if !slices.Contains(ret, staticField) {
if !inSlice(staticField, ret) {
ret = append(ret, staticField)
}
}
@ -621,7 +549,7 @@ func detectSubNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
if len(subnode.Statics) > 0 {
staticsField := detectStaticField(subnode.Statics)
for _, staticField := range staticsField {
if !slices.Contains(ret, staticField) {
if !inSlice(staticField, ret) {
ret = append(ret, staticField)
}
}

View file

@ -1,49 +0,0 @@
package main
import (
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
func TestPrepareAPIURL_NoProtocol(t *testing.T) {
url, err := prepareAPIURL(nil, "localhost:81")
require.NoError(t, err)
assert.Equal(t, "http://localhost:81/", url.String())
}
func TestPrepareAPIURL_Http(t *testing.T) {
url, err := prepareAPIURL(nil, "http://localhost:81")
require.NoError(t, err)
assert.Equal(t, "http://localhost:81/", url.String())
}
func TestPrepareAPIURL_Https(t *testing.T) {
url, err := prepareAPIURL(nil, "https://localhost:81")
require.NoError(t, err)
assert.Equal(t, "https://localhost:81/", url.String())
}
func TestPrepareAPIURL_UnixSocket(t *testing.T) {
url, err := prepareAPIURL(nil, "/path/socket")
require.NoError(t, err)
assert.Equal(t, "/path/socket/", url.String())
}
func TestPrepareAPIURL_Empty(t *testing.T) {
_, err := prepareAPIURL(nil, "")
require.Error(t, err)
}
func TestPrepareAPIURL_Empty_ConfigOverride(t *testing.T) {
url, err := prepareAPIURL(&csconfig.LocalApiClientCfg{
Credentials: &csconfig.ApiCredentialsCfg{
URL: "localhost:80",
},
}, "")
require.NoError(t, err)
assert.Equal(t, "http://localhost:80/", url.String())
}

View file

@ -4,49 +4,48 @@ import (
saferand "crypto/rand"
"encoding/csv"
"encoding/json"
"errors"
"fmt"
"io"
"math/big"
"os"
"slices"
"strings"
"time"
"github.com/AlecAivazis/survey/v2"
"github.com/enescakir/emoji"
"github.com/fatih/color"
"github.com/go-openapi/strfmt"
"github.com/google/uuid"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/machineid"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/database/ent"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
const passwordLength = 64
var (
passwordLength = 64
)
func generatePassword(length int) string {
upper := "ABCDEFGHIJKLMNOPQRSTUVWXY"
lower := "abcdefghijklmnopqrstuvwxyz"
upper := "ABCDEFGHIJKLMNOPQRSTUVWXY"
lower := "abcdefghijklmnopqrstuvwxyz"
digits := "0123456789"
charset := upper + lower + digits
charsetLength := len(charset)
buf := make([]byte, length)
for i := 0; i < length; i++ {
rInt, err := saferand.Int(saferand.Reader, big.NewInt(int64(charsetLength)))
if err != nil {
log.Fatalf("failed getting data from prng for password generation : %s", err)
}
buf[i] = charset[rInt.Int64()]
}
@ -61,14 +60,12 @@ func generateIDPrefix() (string, error) {
if err == nil {
return prefix, nil
}
log.Debugf("failed to get machine-id with usual files: %s", err)
bID, err := uuid.NewRandom()
bId, err := uuid.NewRandom()
if err == nil {
return bID.String(), nil
return bId.String(), nil
}
return "", fmt.Errorf("generating machine id: %w", err)
}
@ -79,427 +76,356 @@ func generateID(prefix string) (string, error) {
if prefix == "" {
prefix, err = generateIDPrefix()
}
if err != nil {
return "", err
}
prefix = strings.ReplaceAll(prefix, "-", "")[:32]
suffix := generatePassword(16)
return prefix + suffix, nil
}
// getLastHeartbeat returns the last heartbeat timestamp of a machine
// and a boolean indicating if the machine is considered active or not.
func getLastHeartbeat(m *ent.Machine) (string, bool) {
if m.LastHeartbeat == nil {
return "-", false
func displayLastHeartBeat(m *ent.Machine, fancy bool) string {
var hbDisplay string
if m.LastHeartbeat != nil {
lastHeartBeat := time.Now().UTC().Sub(*m.LastHeartbeat)
hbDisplay = lastHeartBeat.Truncate(time.Second).String()
if fancy && lastHeartBeat > 2*time.Minute {
hbDisplay = fmt.Sprintf("%s %s", emoji.Warning.String(), lastHeartBeat.Truncate(time.Second).String())
}
} else {
hbDisplay = "-"
if fancy {
hbDisplay = emoji.Warning.String() + " -"
}
}
elapsed := time.Now().UTC().Sub(*m.LastHeartbeat)
hb := elapsed.Truncate(time.Second).String()
if elapsed > 2*time.Minute {
return hb, false
}
return hb, true
return hbDisplay
}
type cliMachines struct {
db *database.Client
cfg configGetter
}
func NewCLIMachines(cfg configGetter) *cliMachines {
return &cliMachines{
cfg: cfg,
func getAgents(out io.Writer, dbClient *database.Client) error {
machines, err := dbClient.ListMachines()
if err != nil {
return fmt.Errorf("unable to list machines: %s", err)
}
}
func (cli *cliMachines) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "machines [action]",
Short: "Manage local API machines [requires local API]",
Long: `To list/add/delete/validate/prune machines.
Note: This command requires database direct access, so is intended to be run on the local API machine.
`,
Example: `cscli machines [action]`,
DisableAutoGenTag: true,
Aliases: []string{"machine"},
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
var err error
if err = require.LAPI(cli.cfg()); err != nil {
return err
if csConfig.Cscli.Output == "human" {
getAgentsTable(out, machines)
} else if csConfig.Cscli.Output == "json" {
enc := json.NewEncoder(out)
enc.SetIndent("", " ")
if err := enc.Encode(machines); err != nil {
return fmt.Errorf("failed to marshal")
}
return nil
} else if csConfig.Cscli.Output == "raw" {
csvwriter := csv.NewWriter(out)
err := csvwriter.Write([]string{"machine_id", "ip_address", "updated_at", "validated", "version", "auth_type", "last_heartbeat"})
if err != nil {
return fmt.Errorf("failed to write header: %s", err)
}
for _, m := range machines {
var validated string
if m.IsValidated {
validated = "true"
} else {
validated = "false"
}
cli.db, err = database.NewClient(cli.cfg().DbConfig)
err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, false)})
if err != nil {
return fmt.Errorf("unable to create new database client: %w", err)
return fmt.Errorf("failed to write raw output : %s", err)
}
}
csvwriter.Flush()
} else {
log.Errorf("unknown output '%s'", csConfig.Cscli.Output)
}
return nil
}
func NewMachinesListCmd() *cobra.Command {
cmdMachinesList := &cobra.Command{
Use: "list",
Short: "List machines",
Long: `List `,
Example: `cscli machines list`,
Args: cobra.MaximumNArgs(1),
DisableAutoGenTag: true,
PreRunE: func(cmd *cobra.Command, args []string) error {
var err error
dbClient, err = database.NewClient(csConfig.DbConfig)
if err != nil {
return fmt.Errorf("unable to create new database client: %s", err)
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
err := getAgents(color.Output, dbClient)
if err != nil {
return fmt.Errorf("unable to list machines: %s", err)
}
return nil
},
}
cmd.AddCommand(cli.newListCmd())
cmd.AddCommand(cli.newAddCmd())
cmd.AddCommand(cli.newDeleteCmd())
cmd.AddCommand(cli.newValidateCmd())
cmd.AddCommand(cli.newPruneCmd())
return cmd
return cmdMachinesList
}
func (cli *cliMachines) list() error {
out := color.Output
machines, err := cli.db.ListMachines()
if err != nil {
return fmt.Errorf("unable to list machines: %w", err)
}
switch cli.cfg().Cscli.Output {
case "human":
getAgentsTable(out, machines)
case "json":
enc := json.NewEncoder(out)
enc.SetIndent("", " ")
if err := enc.Encode(machines); err != nil {
return errors.New("failed to marshal")
}
return nil
case "raw":
csvwriter := csv.NewWriter(out)
err := csvwriter.Write([]string{"machine_id", "ip_address", "updated_at", "validated", "version", "auth_type", "last_heartbeat"})
if err != nil {
return fmt.Errorf("failed to write header: %w", err)
}
for _, m := range machines {
validated := "false"
if m.IsValidated {
validated = "true"
}
hb, _ := getLastHeartbeat(m)
if err := csvwriter.Write([]string{m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb}); err != nil {
return fmt.Errorf("failed to write raw output: %w", err)
}
}
csvwriter.Flush()
}
return nil
}
func (cli *cliMachines) newListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "list",
Short: "list all machines in the database",
Long: `list all machines in the database with their status and last heartbeat`,
Example: `cscli machines list`,
Args: cobra.NoArgs,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.list()
},
}
return cmd
}
func (cli *cliMachines) newAddCmd() *cobra.Command {
var (
password MachinePassword
dumpFile string
apiURL string
interactive bool
autoAdd bool
force bool
)
cmd := &cobra.Command{
func NewMachinesAddCmd() *cobra.Command {
cmdMachinesAdd := &cobra.Command{
Use: "add",
Short: "add a single machine to the database",
Short: "add machine to the database.",
DisableAutoGenTag: true,
Long: `Register a new machine in the database. cscli should be on the same machine as LAPI.`,
Example: `cscli machines add --auto
Example: `
cscli machines add --auto
cscli machines add MyTestMachine --auto
cscli machines add MyTestMachine --password MyPassword
cscli machines add -f- --auto > /tmp/mycreds.yaml`,
RunE: func(_ *cobra.Command, args []string) error {
return cli.add(args, string(password), dumpFile, apiURL, interactive, autoAdd, force)
`,
PreRunE: func(cmd *cobra.Command, args []string) error {
var err error
dbClient, err = database.NewClient(csConfig.DbConfig)
if err != nil {
return fmt.Errorf("unable to create new database client: %s", err)
}
return nil
},
RunE: runMachinesAdd,
}
flags := cmd.Flags()
flags.VarP(&password, "password", "p", "machine password to login to the API")
flags.StringVarP(&dumpFile, "file", "f", "", "output file destination (defaults to "+csconfig.DefaultConfigPath("local_api_credentials.yaml")+")")
flags.StringVarP(&apiURL, "url", "u", "", "URL of the local API")
flags.BoolVarP(&interactive, "interactive", "i", false, "interfactive mode to enter the password")
flags.BoolVarP(&autoAdd, "auto", "a", false, "automatically generate password (and username if not provided)")
flags.BoolVar(&force, "force", false, "will force add the machine if it already exist")
flags := cmdMachinesAdd.Flags()
flags.StringP("password", "p", "", "machine password to login to the API")
flags.StringP("file", "f", "", "output file destination (defaults to "+csconfig.DefaultConfigPath("local_api_credentials.yaml")+")")
flags.StringP("url", "u", "", "URL of the local API")
flags.BoolP("interactive", "i", false, "interfactive mode to enter the password")
flags.BoolP("auto", "a", false, "automatically generate password (and username if not provided)")
flags.Bool("force", false, "will force add the machine if it already exist")
return cmd
return cmdMachinesAdd
}
func (cli *cliMachines) add(args []string, machinePassword string, dumpFile string, apiURL string, interactive bool, autoAdd bool, force bool) error {
var (
err error
machineID string
)
func runMachinesAdd(cmd *cobra.Command, args []string) error {
var dumpFile string
var err error
flags := cmd.Flags()
machinePassword, err := flags.GetString("password")
if err != nil {
return err
}
outputFile, err := flags.GetString("file")
if err != nil {
return err
}
apiURL, err := flags.GetString("url")
if err != nil {
return err
}
interactive, err := flags.GetBool("interactive")
if err != nil {
return err
}
autoAdd, err := flags.GetBool("auto")
if err != nil {
return err
}
forceAdd, err := flags.GetBool("force")
if err != nil {
return err
}
var machineID string
// create machineID if not specified by user
if len(args) == 0 {
if !autoAdd {
return errors.New("please specify a machine name to add, or use --auto")
printHelp(cmd)
return nil
}
machineID, err = generateID("")
if err != nil {
return fmt.Errorf("unable to generate machine id: %w", err)
return fmt.Errorf("unable to generate machine id: %s", err)
}
} else {
machineID = args[0]
}
clientCfg := cli.cfg().API.Client
serverCfg := cli.cfg().API.Server
/*check if file already exists*/
if dumpFile == "" && clientCfg != nil && clientCfg.CredentialsFilePath != "" {
credFile := clientCfg.CredentialsFilePath
// use the default only if the file does not exist
_, err = os.Stat(credFile)
switch {
case os.IsNotExist(err) || force:
dumpFile = credFile
case err != nil:
return fmt.Errorf("unable to stat '%s': %w", credFile, err)
default:
return fmt.Errorf(`credentials file '%s' already exists: please remove it, use "--force" or specify a different file with "-f" ("-f -" for standard output)`, credFile)
}
}
if dumpFile == "" {
return errors.New(`please specify a file to dump credentials to, with -f ("-f -" for standard output)`)
if outputFile != "" {
dumpFile = outputFile
} else if csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" {
dumpFile = csConfig.API.Client.CredentialsFilePath
}
// create a password if it's not specified by user
if machinePassword == "" && !interactive {
if !autoAdd {
return errors.New("please specify a password with --password or use --auto")
printHelp(cmd)
return nil
}
machinePassword = generatePassword(passwordLength)
} else if machinePassword == "" && interactive {
qs := &survey.Password{
Message: "Please provide a password for the machine:",
Message: "Please provide a password for the machine",
}
survey.AskOne(qs, &machinePassword)
}
password := strfmt.Password(machinePassword)
_, err = cli.db.CreateMachine(&machineID, &password, "", true, force, types.PasswordAuthType)
_, err = dbClient.CreateMachine(&machineID, &password, "", true, forceAdd, types.PasswordAuthType)
if err != nil {
return fmt.Errorf("unable to create machine: %w", err)
return fmt.Errorf("unable to create machine: %s", err)
}
fmt.Fprintf(os.Stderr, "Machine '%s' successfully added to the local API.\n", machineID)
log.Infof("Machine '%s' successfully added to the local API", machineID)
if apiURL == "" {
if clientCfg != nil && clientCfg.Credentials != nil && clientCfg.Credentials.URL != "" {
apiURL = clientCfg.Credentials.URL
} else if serverCfg.ClientURL() != "" {
apiURL = serverCfg.ClientURL()
if csConfig.API.Client != nil && csConfig.API.Client.Credentials != nil && csConfig.API.Client.Credentials.URL != "" {
apiURL = csConfig.API.Client.Credentials.URL
} else if csConfig.API.Server != nil && csConfig.API.Server.ListenURI != "" {
apiURL = "http://" + csConfig.API.Server.ListenURI
} else {
return errors.New("unable to dump an api URL. Please provide it in your configuration or with the -u parameter")
return fmt.Errorf("unable to dump an api URL. Please provide it in your configuration or with the -u parameter")
}
}
apiCfg := csconfig.ApiCredentialsCfg{
Login: machineID,
Password: password.String(),
URL: apiURL,
}
apiConfigDump, err := yaml.Marshal(apiCfg)
if err != nil {
return fmt.Errorf("unable to marshal api credentials: %w", err)
return fmt.Errorf("unable to marshal api credentials: %s", err)
}
if dumpFile != "" && dumpFile != "-" {
if err = os.WriteFile(dumpFile, apiConfigDump, 0o600); err != nil {
return fmt.Errorf("write api credentials in '%s' failed: %w", dumpFile, err)
err = os.WriteFile(dumpFile, apiConfigDump, 0644)
if err != nil {
return fmt.Errorf("write api credentials in '%s' failed: %s", dumpFile, err)
}
fmt.Fprintf(os.Stderr, "API credentials written to '%s'.\n", dumpFile)
log.Printf("API credentials dumped to '%s'", dumpFile)
} else {
fmt.Print(string(apiConfigDump))
fmt.Printf("%s\n", string(apiConfigDump))
}
return nil
}
func (cli *cliMachines) deleteValid(_ *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
machines, err := cli.db.ListMachines()
if err != nil {
cobra.CompError("unable to list machines " + err.Error())
func NewMachinesDeleteCmd() *cobra.Command {
cmdMachinesDelete := &cobra.Command{
Use: "delete [machine_name]...",
Short: "delete machines",
Example: `cscli machines delete "machine1" "machine2"`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"remove"},
DisableAutoGenTag: true,
PreRunE: func(cmd *cobra.Command, args []string) error {
var err error
dbClient, err = database.NewClient(csConfig.DbConfig)
if err != nil {
return fmt.Errorf("unable to create new database client: %s", err)
}
return nil
},
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
var err error
dbClient, err = getDBClient()
if err != nil {
cobra.CompError("unable to create new database client: " + err.Error())
return nil, cobra.ShellCompDirectiveNoFileComp
}
machines, err := dbClient.ListMachines()
if err != nil {
cobra.CompError("unable to list machines " + err.Error())
}
ret := make([]string, 0)
for _, machine := range machines {
if strings.Contains(machine.MachineId, toComplete) && !inSlice(machine.MachineId, args) {
ret = append(ret, machine.MachineId)
}
}
return ret, cobra.ShellCompDirectiveNoFileComp
},
RunE: runMachinesDelete,
}
ret := []string{}
for _, machine := range machines {
if strings.Contains(machine.MachineId, toComplete) && !slices.Contains(args, machine.MachineId) {
ret = append(ret, machine.MachineId)
}
}
return ret, cobra.ShellCompDirectiveNoFileComp
return cmdMachinesDelete
}
func (cli *cliMachines) delete(machines []string) error {
for _, machineID := range machines {
if err := cli.db.DeleteWatcher(machineID); err != nil {
func runMachinesDelete(cmd *cobra.Command, args []string) error {
for _, machineID := range args {
err := dbClient.DeleteWatcher(machineID)
if err != nil {
log.Errorf("unable to delete machine '%s': %s", machineID, err)
return nil
}
log.Infof("machine '%s' deleted successfully", machineID)
}
return nil
}
func (cli *cliMachines) newDeleteCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "delete [machine_name]...",
Short: "delete machine(s) by name",
Example: `cscli machines delete "machine1" "machine2"`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"remove"},
DisableAutoGenTag: true,
ValidArgsFunction: cli.deleteValid,
RunE: func(_ *cobra.Command, args []string) error {
return cli.delete(args)
},
}
return cmd
}
func (cli *cliMachines) prune(duration time.Duration, notValidOnly bool, force bool) error {
if duration < 2*time.Minute && !notValidOnly {
if yes, err := askYesNo(
"The duration you provided is less than 2 minutes. " +
"This can break installations if the machines are only temporarily disconnected. Continue?", false); err != nil {
return err
} else if !yes {
fmt.Println("User aborted prune. No changes were made.")
return nil
}
}
machines := []*ent.Machine{}
if pending, err := cli.db.QueryPendingMachine(); err == nil {
machines = append(machines, pending...)
}
if !notValidOnly {
if pending, err := cli.db.QueryLastValidatedHeartbeatLT(time.Now().UTC().Add(-duration)); err == nil {
machines = append(machines, pending...)
}
}
if len(machines) == 0 {
fmt.Println("No machines to prune.")
return nil
}
getAgentsTable(color.Output, machines)
if !force {
if yes, err := askYesNo(
"You are about to PERMANENTLY remove the above machines from the database. " +
"These will NOT be recoverable. Continue?", false); err != nil {
return err
} else if !yes {
fmt.Println("User aborted prune. No changes were made.")
return nil
}
}
deleted, err := cli.db.BulkDeleteWatchers(machines)
if err != nil {
return fmt.Errorf("unable to prune machines: %w", err)
}
fmt.Fprintf(os.Stderr, "successfully delete %d machines\n", deleted)
return nil
}
func (cli *cliMachines) newPruneCmd() *cobra.Command {
var (
duration time.Duration
notValidOnly bool
force bool
)
const defaultDuration = 10 * time.Minute
cmd := &cobra.Command{
Use: "prune",
Short: "prune multiple machines from the database",
Long: `prune multiple machines that are not validated or have not connected to the local API in a given duration.`,
Example: `cscli machines prune
cscli machines prune --duration 1h
cscli machines prune --not-validated-only --force`,
Args: cobra.NoArgs,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.prune(duration, notValidOnly, force)
},
}
flags := cmd.Flags()
flags.DurationVarP(&duration, "duration", "d", defaultDuration, "duration of time since validated machine last heartbeat")
flags.BoolVar(&notValidOnly, "not-validated-only", false, "only prune machines that are not validated")
flags.BoolVar(&force, "force", false, "force prune without asking for confirmation")
return cmd
}
func (cli *cliMachines) validate(machineID string) error {
if err := cli.db.ValidateMachine(machineID); err != nil {
return fmt.Errorf("unable to validate machine '%s': %w", machineID, err)
}
log.Infof("machine '%s' validated successfully", machineID)
return nil
}
func (cli *cliMachines) newValidateCmd() *cobra.Command {
cmd := &cobra.Command{
func NewMachinesValidateCmd() *cobra.Command {
cmdMachinesValidate := &cobra.Command{
Use: "validate",
Short: "validate a machine to access the local API",
Long: `validate a machine to access the local API.`,
Example: `cscli machines validate "machine_name"`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
return cli.validate(args[0])
PreRunE: func(cmd *cobra.Command, args []string) error {
var err error
dbClient, err = database.NewClient(csConfig.DbConfig)
if err != nil {
return fmt.Errorf("unable to create new database client: %s", err)
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
machineID := args[0]
if err := dbClient.ValidateMachine(machineID); err != nil {
return fmt.Errorf("unable to validate machine '%s': %s", machineID, err)
}
log.Infof("machine '%s' validated successfully", machineID)
return nil
},
}
return cmd
return cmdMachinesValidate
}
func NewMachinesCmd() *cobra.Command {
var cmdMachines = &cobra.Command{
Use: "machines [action]",
Short: "Manage local API machines [requires local API]",
Long: `To list/add/delete/validate machines.
Note: This command requires database direct access, so is intended to be run on the local API machine.
`,
Example: `cscli machines [action]`,
DisableAutoGenTag: true,
Aliases: []string{"machine"},
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
if err != nil {
log.Errorf("local api : %s", err)
}
return fmt.Errorf("local API is disabled, please run this command on the local API machine")
}
return nil
},
}
cmdMachines.AddCommand(NewMachinesListCmd())
cmdMachines.AddCommand(NewMachinesAddCmd())
cmdMachines.AddCommand(NewMachinesDeleteCmd())
cmdMachines.AddCommand(NewMachinesValidateCmd())
return cmdMachines
}

View file

@ -5,9 +5,9 @@ import (
"time"
"github.com/aquasecurity/table"
"github.com/enescakir/emoji"
"github.com/crowdsecurity/crowdsec/pkg/database/ent"
"github.com/crowdsecurity/crowdsec/pkg/emoji"
)
func getAgentsTable(out io.Writer, machines []*ent.Machine) {
@ -17,17 +17,14 @@ func getAgentsTable(out io.Writer, machines []*ent.Machine) {
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
for _, m := range machines {
validated := emoji.Prohibited
var validated string
if m.IsValidated {
validated = emoji.CheckMark
validated = emoji.CheckMark.String()
} else {
validated = emoji.Prohibited.String()
}
hb, active := getLastHeartbeat(m)
if !active {
hb = emoji.Warning + " " + hb
}
t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, hb)
t.AddRow(m.MachineId, m.IpAddress, m.UpdatedAt.Format(time.RFC3339), validated, m.Version, m.AuthType, displayLastHeartBeat(m, true))
}
t.Render()

View file

@ -3,111 +3,64 @@ package main
import (
"fmt"
"os"
"path"
"path/filepath"
"slices"
"time"
"strings"
"github.com/fatih/color"
cc "github.com/ivanpirog/coloredcobra"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/spf13/cobra/doc"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
)
var (
ConfigFilePath string
csConfig *csconfig.Config
dbClient *database.Client
)
var trace_lvl, dbg_lvl, nfo_lvl, wrn_lvl, err_lvl bool
type configGetter func() *csconfig.Config
var ConfigFilePath string
var csConfig *csconfig.Config
var dbClient *database.Client
var mergedConfig string
var OutputFormat string
var OutputColor string
type cliRoot struct {
logTrace bool
logDebug bool
logInfo bool
logWarn bool
logErr bool
outputColor string
outputFormat string
// flagBranch overrides the value in csConfig.Cscli.HubBranch
flagBranch string
}
var downloadOnly bool
var forceAction bool
var purge bool
var all bool
func newCliRoot() *cliRoot {
return &cliRoot{}
}
var prometheusURL string
// cfg() is a helper function to get the configuration loaded from config.yaml,
// we pass it to subcommands because the file is not read until the Execute() call
func (cli *cliRoot) cfg() *csconfig.Config {
return csConfig
}
// wantedLogLevel returns the log level requested in the command line flags.
func (cli *cliRoot) wantedLogLevel() log.Level {
switch {
case cli.logTrace:
return log.TraceLevel
case cli.logDebug:
return log.DebugLevel
case cli.logInfo:
return log.InfoLevel
case cli.logWarn:
return log.WarnLevel
case cli.logErr:
return log.ErrorLevel
default:
return log.InfoLevel
}
}
// loadConfigFor loads the configuration file for the given sub-command.
// If the sub-command does not need it, it returns a default configuration.
func loadConfigFor(command string) (*csconfig.Config, string, error) {
noNeedConfig := []string{
"doc",
"help",
"completion",
"version",
"hubtest",
}
if !slices.Contains(noNeedConfig, command) {
log.Debugf("Using %s as configuration file", ConfigFilePath)
config, merged, err := csconfig.NewConfig(ConfigFilePath, false, false, true)
if err != nil {
return nil, "", err
}
// set up directory for trace files
if err := trace.Init(filepath.Join(config.ConfigPaths.DataDir, "trace")); err != nil {
return nil, "", fmt.Errorf("while setting up trace directory: %w", err)
}
return config, merged, nil
}
return csconfig.NewDefaultConfig(), "", nil
}
// initialize is called before the subcommand is executed.
func (cli *cliRoot) initialize() {
func initConfig() {
var err error
if trace_lvl {
log.SetLevel(log.TraceLevel)
} else if dbg_lvl {
log.SetLevel(log.DebugLevel)
} else if nfo_lvl {
log.SetLevel(log.InfoLevel)
} else if wrn_lvl {
log.SetLevel(log.WarnLevel)
} else if err_lvl {
log.SetLevel(log.ErrorLevel)
}
log.SetLevel(cli.wantedLogLevel())
csConfig, mergedConfig, err = loadConfigFor(os.Args[1])
if err != nil {
log.Fatal(err)
if !inSlice(os.Args[1], NoNeedConfig) {
csConfig, err = csconfig.NewConfig(ConfigFilePath, false, false, true)
if err != nil {
log.Fatal(err)
}
log.Debugf("Using %s as configuration file", ConfigFilePath)
if err := csConfig.LoadCSCLI(); err != nil {
log.Fatal(err)
}
} else {
csConfig = csconfig.NewDefaultConfig()
}
// recap of the enabled feature flags, because logging
@ -116,22 +69,22 @@ func (cli *cliRoot) initialize() {
log.Debugf("Enabled feature flags: %s", fflist)
}
if cli.flagBranch != "" {
csConfig.Cscli.HubBranch = cli.flagBranch
if csConfig.Cscli == nil {
log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath)
}
if cli.outputFormat != "" {
csConfig.Cscli.Output = cli.outputFormat
if cwhub.HubBranch == "" && csConfig.Cscli.HubBranch != "" {
cwhub.HubBranch = csConfig.Cscli.HubBranch
}
if OutputFormat != "" {
csConfig.Cscli.Output = OutputFormat
if OutputFormat != "json" && OutputFormat != "raw" && OutputFormat != "human" {
log.Fatalf("output format %s unknown", OutputFormat)
}
}
if csConfig.Cscli.Output == "" {
csConfig.Cscli.Output = "human"
}
if csConfig.Cscli.Output != "human" && csConfig.Cscli.Output != "json" && csConfig.Cscli.Output != "raw" {
log.Fatalf("output format '%s' not supported: must be one of human, json, raw", csConfig.Cscli.Output)
}
if csConfig.Cscli.Output == "json" {
log.SetFormatter(&log.JSONFormatter{})
log.SetLevel(log.ErrorLevel)
@ -139,44 +92,47 @@ func (cli *cliRoot) initialize() {
log.SetLevel(log.ErrorLevel)
}
if cli.outputColor != "" {
csConfig.Cscli.Color = cli.outputColor
if cli.outputColor != "yes" && cli.outputColor != "no" && cli.outputColor != "auto" {
log.Fatalf("output color %s unknown", cli.outputColor)
if OutputColor != "" {
csConfig.Cscli.Color = OutputColor
if OutputColor != "yes" && OutputColor != "no" && OutputColor != "auto" {
log.Fatalf("output color %s unknown", OutputColor)
}
}
}
// list of valid subcommands for the shell completion
var validArgs = []string{
"alerts", "appsec-configs", "appsec-rules", "bouncers", "capi", "collections",
"completion", "config", "console", "contexts", "dashboard", "decisions", "explain",
"hub", "hubtest", "lapi", "machines", "metrics", "notifications", "parsers",
"postoverflows", "scenarios", "simulation", "support", "version",
"scenarios", "parsers", "collections", "capi", "lapi", "postoverflows", "machines",
"metrics", "bouncers", "alerts", "decisions", "simulation", "hub", "dashboard",
"config", "completion", "version", "console", "notifications", "support",
}
func (cli *cliRoot) colorize(cmd *cobra.Command) {
cc.Init(&cc.Config{
RootCmd: cmd,
Headings: cc.Yellow,
Commands: cc.Green + cc.Bold,
CmdShortDescr: cc.Cyan,
Example: cc.Italic,
ExecName: cc.Bold,
Aliases: cc.Bold + cc.Italic,
FlagsDataType: cc.White,
Flags: cc.Green,
FlagsDescr: cc.Cyan,
NoExtraNewlines: true,
NoBottomNewline: true,
})
cmd.SetOut(color.Output)
func prepender(filename string) string {
const header = `---
id: %s
title: %s
---
`
name := filepath.Base(filename)
base := strings.TrimSuffix(name, path.Ext(name))
return fmt.Sprintf(header, base, strings.ReplaceAll(base, "_", " "))
}
func (cli *cliRoot) NewCommand() *cobra.Command {
func linkHandler(name string) string {
return fmt.Sprintf("/cscli/%s", name)
}
var (
NoNeedConfig = []string{
"help",
"completion",
"version",
"hubtest",
}
)
func main() {
// set the formatter asap and worry about level later
logFormatter := &log.TextFormatter{TimestampFormat: time.RFC3339, FullTimestamp: true}
logFormatter := &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}
log.SetFormatter(logFormatter)
if err := fflag.RegisterAllFeatures(); err != nil {
@ -187,7 +143,7 @@ func (cli *cliRoot) NewCommand() *cobra.Command {
log.Fatalf("failed to set feature flags from env: %s", err)
}
cmd := &cobra.Command{
var rootCmd = &cobra.Command{
Use: "cscli",
Short: "cscli allows you to manage crowdsec",
Long: `cscli is the main command to interact with your crowdsec service, scenarios & db.
@ -199,25 +155,57 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
/*TBD examples*/
}
cli.colorize(cmd)
cc.Init(&cc.Config{
RootCmd: rootCmd,
Headings: cc.Yellow,
Commands: cc.Green + cc.Bold,
CmdShortDescr: cc.Cyan,
Example: cc.Italic,
ExecName: cc.Bold,
Aliases: cc.Bold + cc.Italic,
FlagsDataType: cc.White,
Flags: cc.Green,
FlagsDescr: cc.Cyan,
})
rootCmd.SetOut(color.Output)
/*don't sort flags so we can enforce order*/
cmd.Flags().SortFlags = false
var cmdDocGen = &cobra.Command{
Use: "doc",
Short: "Generate the documentation in `./doc/`. Directory must exist.",
Args: cobra.ExactArgs(0),
Hidden: true,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if err := doc.GenMarkdownTreeCustom(rootCmd, "./doc/", prepender, linkHandler); err != nil {
return fmt.Errorf("Failed to generate cobra doc: %s", err)
}
return nil
},
}
rootCmd.AddCommand(cmdDocGen)
/*usage*/
var cmdVersion = &cobra.Command{
Use: "version",
Short: "Display version and exit.",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
cwversion.Show()
},
}
rootCmd.AddCommand(cmdVersion)
pflags := cmd.PersistentFlags()
pflags.SortFlags = false
rootCmd.PersistentFlags().StringVarP(&ConfigFilePath, "config", "c", csconfig.DefaultConfigPath("config.yaml"), "path to crowdsec config file")
rootCmd.PersistentFlags().StringVarP(&OutputFormat, "output", "o", "", "Output format: human, json, raw.")
rootCmd.PersistentFlags().StringVarP(&OutputColor, "color", "", "auto", "Output color: yes, no, auto.")
rootCmd.PersistentFlags().BoolVar(&dbg_lvl, "debug", false, "Set logging to debug.")
rootCmd.PersistentFlags().BoolVar(&nfo_lvl, "info", false, "Set logging to info.")
rootCmd.PersistentFlags().BoolVar(&wrn_lvl, "warning", false, "Set logging to warning.")
rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error.")
rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace.")
pflags.StringVarP(&ConfigFilePath, "config", "c", csconfig.DefaultConfigPath("config.yaml"), "path to crowdsec config file")
pflags.StringVarP(&cli.outputFormat, "output", "o", "", "Output format: human, json, raw")
pflags.StringVarP(&cli.outputColor, "color", "", "auto", "Output color: yes, no, auto")
pflags.BoolVar(&cli.logDebug, "debug", false, "Set logging to debug")
pflags.BoolVar(&cli.logInfo, "info", false, "Set logging to info")
pflags.BoolVar(&cli.logWarn, "warning", false, "Set logging to warning")
pflags.BoolVar(&cli.logErr, "error", false, "Set logging to error")
pflags.BoolVar(&cli.logTrace, "trace", false, "Set logging to trace")
pflags.StringVar(&cli.flagBranch, "branch", "", "Override hub branch on github")
if err := pflags.MarkHidden("branch"); err != nil {
rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "", "Override hub branch on github")
if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil {
log.Fatalf("failed to hide flag: %s", err)
}
@ -237,47 +225,45 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
}
if len(os.Args) > 1 {
cobra.OnInitialize(cli.initialize)
cobra.OnInitialize(initConfig)
}
cmd.AddCommand(NewCLIDoc().NewCommand(cmd))
cmd.AddCommand(NewCLIVersion().NewCommand())
cmd.AddCommand(NewCLIConfig(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIHub(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIMetrics(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIDashboard(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIDecisions(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIAlerts(cli.cfg).NewCommand())
cmd.AddCommand(NewCLISimulation(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIBouncers(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIMachines(cli.cfg).NewCommand())
cmd.AddCommand(NewCLICapi(cli.cfg).NewCommand())
cmd.AddCommand(NewCLILapi(cli.cfg).NewCommand())
cmd.AddCommand(NewCompletionCmd())
cmd.AddCommand(NewCLIConsole(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIExplain(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIHubTest(cli.cfg).NewCommand())
cmd.AddCommand(NewCLINotifications(cli.cfg).NewCommand())
cmd.AddCommand(NewCLISupport().NewCommand())
cmd.AddCommand(NewCLIPapi(cli.cfg).NewCommand())
cmd.AddCommand(NewCLICollection(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIParser(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIScenario(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIPostOverflow(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIContext(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIAppsecConfig(cli.cfg).NewCommand())
cmd.AddCommand(NewCLIAppsecRule(cli.cfg).NewCommand())
/*don't sort flags so we can enforce order*/
rootCmd.Flags().SortFlags = false
rootCmd.PersistentFlags().SortFlags = false
rootCmd.AddCommand(NewConfigCmd())
rootCmd.AddCommand(NewHubCmd())
rootCmd.AddCommand(NewMetricsCmd())
rootCmd.AddCommand(NewDashboardCmd())
rootCmd.AddCommand(NewDecisionsCmd())
rootCmd.AddCommand(NewAlertsCmd())
// rootCmd.AddCommand(NewInspectCmd())
rootCmd.AddCommand(NewSimulationCmds())
rootCmd.AddCommand(NewBouncersCmd())
rootCmd.AddCommand(NewMachinesCmd())
rootCmd.AddCommand(NewParsersCmd())
rootCmd.AddCommand(NewScenariosCmd())
rootCmd.AddCommand(NewCollectionsCmd())
rootCmd.AddCommand(NewPostOverflowsCmd())
rootCmd.AddCommand(NewCapiCmd())
rootCmd.AddCommand(NewLapiCmd())
rootCmd.AddCommand(NewCompletionCmd())
rootCmd.AddCommand(NewConsoleCmd())
rootCmd.AddCommand(NewExplainCmd())
rootCmd.AddCommand(NewHubTestCmd())
rootCmd.AddCommand(NewNotificationsCmd())
rootCmd.AddCommand(NewSupportCmd())
if fflag.CscliSetup.IsEnabled() {
cmd.AddCommand(NewSetupCmd())
rootCmd.AddCommand(NewSetupCmd())
}
return cmd
}
if fflag.PapiClient.IsEnabled() {
rootCmd.AddCommand(NewPapiCmd())
}
func main() {
cmd := newCliRoot().NewCommand()
if err := cmd.Execute(); err != nil {
if err := rootCmd.Execute(); err != nil {
log.Fatal(err)
}
}

View file

@ -2,10 +2,10 @@ package main
import (
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"strconv"
"strings"
"time"
@ -15,67 +15,14 @@ import (
"github.com/prometheus/prom2json"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/maptools"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
type (
statAcquis map[string]map[string]int
statParser map[string]map[string]int
statBucket map[string]map[string]int
statWhitelist map[string]map[string]map[string]int
statLapi map[string]map[string]int
statLapiMachine map[string]map[string]map[string]int
statLapiBouncer map[string]map[string]map[string]int
statLapiDecision map[string]struct {
NonEmpty int
Empty int
}
statDecision map[string]map[string]map[string]int
statAppsecEngine map[string]map[string]int
statAppsecRule map[string]map[string]map[string]int
statAlert map[string]int
statStash map[string]struct {
Type string
Count int
}
)
var (
ErrMissingConfig = errors.New("prometheus section missing, can't show metrics")
ErrMetricsDisabled = errors.New("prometheus is not enabled, can't show metrics")
)
type metricSection interface {
Table(out io.Writer, noUnit bool, showEmpty bool)
Description() (string, string)
}
type metricStore map[string]metricSection
func NewMetricStore() metricStore {
return metricStore{
"acquisition": statAcquis{},
"scenarios": statBucket{},
"parsers": statParser{},
"lapi": statLapi{},
"lapi-machine": statLapiMachine{},
"lapi-bouncer": statLapiBouncer{},
"lapi-decisions": statLapiDecision{},
"decisions": statDecision{},
"alerts": statAlert{},
"stash": statStash{},
"appsec-engine": statAppsecEngine{},
"appsec-rule": statAppsecRule{},
"whitelists": statWhitelist{},
}
}
func (ms metricStore) Fetch(url string) error {
// FormatPrometheusMetrics is a complete rip from prom2json
func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error {
mfChan := make(chan *dto.MetricFamily, 1024)
errChan := make(chan error, 1)
// Start with the DefaultTransport for sane defaults.
transport := http.DefaultTransport.(*http.Transport).Clone()
@ -85,61 +32,51 @@ func (ms metricStore) Fetch(url string) error {
// Timeout early if the server doesn't even return the headers.
transport.ResponseHeaderTimeout = time.Minute
go func() {
defer trace.CatchPanic("crowdsec/ShowPrometheus")
defer types.CatchPanic("crowdsec/ShowPrometheus")
err := prom2json.FetchMetricFamilies(url, mfChan, transport)
if err != nil {
errChan <- fmt.Errorf("failed to fetch metrics: %w", err)
return
log.Fatalf("failed to fetch prometheus metrics : %v", err)
}
errChan <- nil
}()
result := []*prom2json.Family{}
for mf := range mfChan {
result = append(result, prom2json.NewFamily(mf))
}
if err := <-errChan; err != nil {
return err
}
log.Debugf("Finished reading metrics output, %d entries", len(result))
log.Debugf("Finished reading prometheus output, %d entries", len(result))
/*walk*/
mAcquis := ms["acquisition"].(statAcquis)
mParser := ms["parsers"].(statParser)
mBucket := ms["scenarios"].(statBucket)
mLapi := ms["lapi"].(statLapi)
mLapiMachine := ms["lapi-machine"].(statLapiMachine)
mLapiBouncer := ms["lapi-bouncer"].(statLapiBouncer)
mLapiDecision := ms["lapi-decisions"].(statLapiDecision)
mDecision := ms["decisions"].(statDecision)
mAppsecEngine := ms["appsec-engine"].(statAppsecEngine)
mAppsecRule := ms["appsec-rule"].(statAppsecRule)
mAlert := ms["alerts"].(statAlert)
mStash := ms["stash"].(statStash)
mWhitelist := ms["whitelists"].(statWhitelist)
lapi_decisions_stats := map[string]struct {
NonEmpty int
Empty int
}{}
acquis_stats := map[string]map[string]int{}
parsers_stats := map[string]map[string]int{}
buckets_stats := map[string]map[string]int{}
lapi_stats := map[string]map[string]int{}
lapi_machine_stats := map[string]map[string]map[string]int{}
lapi_bouncer_stats := map[string]map[string]map[string]int{}
decisions_stats := map[string]map[string]map[string]int{}
alerts_stats := map[string]int{}
stash_stats := map[string]struct {
Type string
Count int
}{}
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
source, ok := metric.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v for %s", metric.Labels, fam.Name)
@ -160,338 +97,204 @@ func (ms metricStore) Fetch(url string) error {
origin := metric.Labels["origin"]
action := metric.Labels["action"]
appsecEngine := metric.Labels["appsec_engine"]
appsecRule := metric.Labels["rule_name"]
mtype := metric.Labels["type"]
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
}
ival := int(fval)
switch fam.Name {
//
// buckets
//
/*buckets*/
case "cs_bucket_created_total":
mBucket.Process(name, "instantiation", ival)
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
buckets_stats[name]["instantiation"] += ival
case "cs_buckets":
mBucket.Process(name, "curr_count", ival)
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
buckets_stats[name]["curr_count"] += ival
case "cs_bucket_overflowed_total":
mBucket.Process(name, "overflow", ival)
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
buckets_stats[name]["overflow"] += ival
case "cs_bucket_poured_total":
mBucket.Process(name, "pour", ival)
mAcquis.Process(source, "pour", ival)
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
}
buckets_stats[name]["pour"] += ival
acquis_stats[source]["pour"] += ival
case "cs_bucket_underflowed_total":
mBucket.Process(name, "underflow", ival)
//
// parsers
//
if _, ok := buckets_stats[name]; !ok {
buckets_stats[name] = make(map[string]int)
}
buckets_stats[name]["underflow"] += ival
/*acquis*/
case "cs_parser_hits_total":
mAcquis.Process(source, "reads", ival)
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
}
acquis_stats[source]["reads"] += ival
case "cs_parser_hits_ok_total":
mAcquis.Process(source, "parsed", ival)
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
}
acquis_stats[source]["parsed"] += ival
case "cs_parser_hits_ko_total":
mAcquis.Process(source, "unparsed", ival)
if _, ok := acquis_stats[source]; !ok {
acquis_stats[source] = make(map[string]int)
}
acquis_stats[source]["unparsed"] += ival
case "cs_node_hits_total":
mParser.Process(name, "hits", ival)
if _, ok := parsers_stats[name]; !ok {
parsers_stats[name] = make(map[string]int)
}
parsers_stats[name]["hits"] += ival
case "cs_node_hits_ok_total":
mParser.Process(name, "parsed", ival)
if _, ok := parsers_stats[name]; !ok {
parsers_stats[name] = make(map[string]int)
}
parsers_stats[name]["parsed"] += ival
case "cs_node_hits_ko_total":
mParser.Process(name, "unparsed", ival)
//
// whitelists
//
case "cs_node_wl_hits_total":
mWhitelist.Process(name, reason, "hits", ival)
case "cs_node_wl_hits_ok_total":
mWhitelist.Process(name, reason, "whitelisted", ival)
// track as well whitelisted lines at acquis level
mAcquis.Process(source, "whitelisted", ival)
//
// lapi
//
if _, ok := parsers_stats[name]; !ok {
parsers_stats[name] = make(map[string]int)
}
parsers_stats[name]["unparsed"] += ival
case "cs_lapi_route_requests_total":
mLapi.Process(route, method, ival)
if _, ok := lapi_stats[route]; !ok {
lapi_stats[route] = make(map[string]int)
}
lapi_stats[route][method] += ival
case "cs_lapi_machine_requests_total":
mLapiMachine.Process(machine, route, method, ival)
if _, ok := lapi_machine_stats[machine]; !ok {
lapi_machine_stats[machine] = make(map[string]map[string]int)
}
if _, ok := lapi_machine_stats[machine][route]; !ok {
lapi_machine_stats[machine][route] = make(map[string]int)
}
lapi_machine_stats[machine][route][method] += ival
case "cs_lapi_bouncer_requests_total":
mLapiBouncer.Process(bouncer, route, method, ival)
if _, ok := lapi_bouncer_stats[bouncer]; !ok {
lapi_bouncer_stats[bouncer] = make(map[string]map[string]int)
}
if _, ok := lapi_bouncer_stats[bouncer][route]; !ok {
lapi_bouncer_stats[bouncer][route] = make(map[string]int)
}
lapi_bouncer_stats[bouncer][route][method] += ival
case "cs_lapi_decisions_ko_total", "cs_lapi_decisions_ok_total":
mLapiDecision.Process(bouncer, fam.Name, ival)
//
// decisions
//
if _, ok := lapi_decisions_stats[bouncer]; !ok {
lapi_decisions_stats[bouncer] = struct {
NonEmpty int
Empty int
}{}
}
x := lapi_decisions_stats[bouncer]
if fam.Name == "cs_lapi_decisions_ko_total" {
x.Empty += ival
} else if fam.Name == "cs_lapi_decisions_ok_total" {
x.NonEmpty += ival
}
lapi_decisions_stats[bouncer] = x
case "cs_active_decisions":
mDecision.Process(reason, origin, action, ival)
if _, ok := decisions_stats[reason]; !ok {
decisions_stats[reason] = make(map[string]map[string]int)
}
if _, ok := decisions_stats[reason][origin]; !ok {
decisions_stats[reason][origin] = make(map[string]int)
}
decisions_stats[reason][origin][action] += ival
case "cs_alerts":
mAlert.Process(reason, ival)
//
// stash
//
/*if _, ok := alerts_stats[scenario]; !ok {
alerts_stats[scenario] = make(map[string]int)
}*/
alerts_stats[reason] += ival
case "cs_cache_size":
mStash.Process(name, mtype, ival)
//
// appsec
//
case "cs_appsec_reqs_total":
mAppsecEngine.Process(appsecEngine, "processed", ival)
case "cs_appsec_block_total":
mAppsecEngine.Process(appsecEngine, "blocked", ival)
case "cs_appsec_rule_hits":
mAppsecRule.Process(appsecEngine, appsecRule, "triggered", ival)
stash_stats[name] = struct {
Type string
Count int
}{Type: mtype, Count: ival}
default:
log.Debugf("unknown: %+v", fam.Name)
continue
}
}
}
if formatType == "human" {
acquisStatsTable(out, acquis_stats)
bucketStatsTable(out, buckets_stats)
parserStatsTable(out, parsers_stats)
lapiStatsTable(out, lapi_stats)
lapiMachineStatsTable(out, lapi_machine_stats)
lapiBouncerStatsTable(out, lapi_bouncer_stats)
lapiDecisionStatsTable(out, lapi_decisions_stats)
decisionStatsTable(out, decisions_stats)
alertStatsTable(out, alerts_stats)
stashStatsTable(out, stash_stats)
} else if formatType == "json" {
for _, val := range []interface{}{acquis_stats, parsers_stats, buckets_stats, lapi_stats, lapi_bouncer_stats, lapi_machine_stats, lapi_decisions_stats, decisions_stats, alerts_stats, stash_stats} {
x, err := json.MarshalIndent(val, "", " ")
if err != nil {
return fmt.Errorf("failed to unmarshal metrics : %v", err)
}
out.Write(x)
}
return nil
} else if formatType == "raw" {
for _, val := range []interface{}{acquis_stats, parsers_stats, buckets_stats, lapi_stats, lapi_bouncer_stats, lapi_machine_stats, lapi_decisions_stats, decisions_stats, alerts_stats, stash_stats} {
x, err := yaml.Marshal(val)
if err != nil {
return fmt.Errorf("failed to unmarshal metrics : %v", err)
}
out.Write(x)
}
return nil
}
return nil
}
type cliMetrics struct {
cfg configGetter
}
var noUnit bool
func NewCLIMetrics(cfg configGetter) *cliMetrics {
return &cliMetrics{
cfg: cfg,
}
}
func (ms metricStore) Format(out io.Writer, sections []string, formatType string, noUnit bool) error {
// copy only the sections we want
want := map[string]metricSection{}
// if explicitly asking for sections, we want to show empty tables
showEmpty := len(sections) > 0
// if no sections are specified, we want all of them
if len(sections) == 0 {
sections = maptools.SortedKeys(ms)
}
for _, section := range sections {
want[section] = ms[section]
}
switch formatType {
case "human":
for _, section := range maptools.SortedKeys(want) {
want[section].Table(out, noUnit, showEmpty)
}
case "json":
x, err := json.MarshalIndent(want, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal metrics: %w", err)
}
out.Write(x)
case "raw":
x, err := yaml.Marshal(want)
if err != nil {
return fmt.Errorf("failed to marshal metrics: %w", err)
}
out.Write(x)
default:
return fmt.Errorf("unknown format type %s", formatType)
}
return nil
}
func (cli *cliMetrics) show(sections []string, url string, noUnit bool) error {
cfg := cli.cfg()
if url != "" {
cfg.Cscli.PrometheusUrl = url
}
if cfg.Prometheus == nil {
return ErrMissingConfig
}
if !cfg.Prometheus.Enabled {
return ErrMetricsDisabled
}
ms := NewMetricStore()
if err := ms.Fetch(cfg.Cscli.PrometheusUrl); err != nil {
return err
}
// any section that we don't have in the store is an error
for _, section := range sections {
if _, ok := ms[section]; !ok {
return fmt.Errorf("unknown metrics type: %s", section)
}
}
if err := ms.Format(color.Output, sections, cfg.Cscli.Output, noUnit); err != nil {
return err
}
return nil
}
func (cli *cliMetrics) NewCommand() *cobra.Command {
var (
url string
noUnit bool
)
cmd := &cobra.Command{
Use: "metrics",
Short: "Display crowdsec prometheus metrics.",
Long: `Fetch metrics from a Local API server and display them`,
Example: `# Show all Metrics, skip empty tables (same as "cecli metrics show")
cscli metrics
# Show only some metrics, connect to a different url
cscli metrics --url http://lapi.local:6060/metrics show acquisition parsers
# List available metric types
cscli metrics list`,
func NewMetricsCmd() *cobra.Command {
var cmdMetrics = &cobra.Command{
Use: "metrics",
Short: "Display crowdsec prometheus metrics.",
Long: `Fetch metrics from the prometheus server and display them in a human-friendly way`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.show(nil, url, noUnit)
Run: func(cmd *cobra.Command, args []string) {
if err := csConfig.LoadPrometheus(); err != nil {
log.Fatal(err)
}
if !csConfig.Prometheus.Enabled {
log.Warning("Prometheus is not enabled, can't show metrics")
os.Exit(1)
}
if prometheusURL == "" {
prometheusURL = csConfig.Cscli.PrometheusUrl
}
if prometheusURL == "" {
log.Errorf("No prometheus url, please specify in %s or via -u", *csConfig.FilePath)
os.Exit(1)
}
err := FormatPrometheusMetrics(color.Output, prometheusURL+"/metrics", csConfig.Cscli.Output)
if err != nil {
log.Fatalf("could not fetch prometheus metrics: %s", err)
}
},
}
cmdMetrics.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
cmdMetrics.PersistentFlags().BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
flags := cmd.Flags()
flags.StringVarP(&url, "url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
flags.BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
cmd.AddCommand(cli.newShowCmd())
cmd.AddCommand(cli.newListCmd())
return cmd
}
// expandAlias returns a list of sections. The input can be a list of sections or alias.
func (cli *cliMetrics) expandAlias(args []string) []string {
ret := []string{}
for _, section := range args {
switch section {
case "engine":
ret = append(ret, "acquisition", "parsers", "scenarios", "stash", "whitelists")
case "lapi":
ret = append(ret, "alerts", "decisions", "lapi", "lapi-bouncer", "lapi-decisions", "lapi-machine")
case "appsec":
ret = append(ret, "appsec-engine", "appsec-rule")
default:
ret = append(ret, section)
}
}
return ret
}
func (cli *cliMetrics) newShowCmd() *cobra.Command {
var (
url string
noUnit bool
)
cmd := &cobra.Command{
Use: "show [type]...",
Short: "Display all or part of the available metrics.",
Long: `Fetch metrics from a Local API server and display them, optionally filtering on specific types.`,
Example: `# Show all Metrics, skip empty tables
cscli metrics show
# Use an alias: "engine", "lapi" or "appsec" to show a group of metrics
cscli metrics show engine
# Show some specific metrics, show empty tables, connect to a different url
cscli metrics show acquisition parsers scenarios stash --url http://lapi.local:6060/metrics
# To list available metric types, use "cscli metrics list"
cscli metrics list; cscli metrics list -o json
# Show metrics in json format
cscli metrics show acquisition parsers scenarios stash -o json`,
// Positional args are optional
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
args = cli.expandAlias(args)
return cli.show(args, url, noUnit)
},
}
flags := cmd.Flags()
flags.StringVarP(&url, "url", "u", "", "Metrics url (http://<ip>:<port>/metrics)")
flags.BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
return cmd
}
func (cli *cliMetrics) list() error {
type metricType struct {
Type string `json:"type" yaml:"type"`
Title string `json:"title" yaml:"title"`
Description string `json:"description" yaml:"description"`
}
var allMetrics []metricType
ms := NewMetricStore()
for _, section := range maptools.SortedKeys(ms) {
title, description := ms[section].Description()
allMetrics = append(allMetrics, metricType{
Type: section,
Title: title,
Description: description,
})
}
switch cli.cfg().Cscli.Output {
case "human":
t := newTable(color.Output)
t.SetRowLines(true)
t.SetHeaders("Type", "Title", "Description")
for _, metric := range allMetrics {
t.AddRow(metric.Type, metric.Title, metric.Description)
}
t.Render()
case "json":
x, err := json.MarshalIndent(allMetrics, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal metric types: %w", err)
}
fmt.Println(string(x))
case "raw":
x, err := yaml.Marshal(allMetrics)
if err != nil {
return fmt.Errorf("failed to marshal metric types: %w", err)
}
fmt.Println(string(x))
}
return nil
}
func (cli *cliMetrics) newListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "list",
Short: "List available types of metrics.",
Long: `List available types of metrics.`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
return cli.list()
},
}
return cmd
return cmdMetrics
}

View file

@ -1,33 +1,25 @@
package main
import (
"errors"
"fmt"
"io"
"sort"
"strconv"
"github.com/aquasecurity/table"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/maptools"
)
// ErrNilTable means a nil pointer was passed instead of a table instance. This is a programming error.
var ErrNilTable = errors.New("nil table")
func lapiMetricsToTable(t *table.Table, stats map[string]map[string]map[string]int) int {
// stats: machine -> route -> method -> count
// sort keys to keep consistent order when printing
machineKeys := []string{}
for k := range stats {
machineKeys = append(machineKeys, k)
}
sort.Strings(machineKeys)
numRows := 0
for _, machine := range machineKeys {
// oneRow: route -> method -> count
machineRow := stats[machine]
@ -39,79 +31,41 @@ func lapiMetricsToTable(t *table.Table, stats map[string]map[string]map[string]i
methodName,
}
if count != 0 {
row = append(row, strconv.Itoa(count))
row = append(row, fmt.Sprintf("%d", count))
} else {
row = append(row, "-")
}
t.AddRow(row...)
numRows++
}
}
}
return numRows
}
func wlMetricsToTable(t *table.Table, stats map[string]map[string]map[string]int, noUnit bool) (int, error) {
func metricsToTable(t *table.Table, stats map[string]map[string]int, keys []string) (int, error) {
if t == nil {
return 0, ErrNilTable
return 0, fmt.Errorf("nil table")
}
// sort keys to keep consistent order when printing
sortedKeys := []string{}
for k := range stats {
sortedKeys = append(sortedKeys, k)
}
sort.Strings(sortedKeys)
numRows := 0
for _, name := range maptools.SortedKeys(stats) {
for _, reason := range maptools.SortedKeys(stats[name]) {
row := []string{
name,
reason,
"-",
"-",
}
for _, action := range maptools.SortedKeys(stats[name][reason]) {
value := stats[name][reason][action]
switch action {
case "whitelisted":
row[3] = strconv.Itoa(value)
case "hits":
row[2] = strconv.Itoa(value)
default:
log.Debugf("unexpected counter '%s' for whitelists = %d", action, value)
}
}
t.AddRow(row...)
numRows++
}
}
return numRows, nil
}
func metricsToTable(t *table.Table, stats map[string]map[string]int, keys []string, noUnit bool) (int, error) {
if t == nil {
return 0, ErrNilTable
}
numRows := 0
for _, alabel := range maptools.SortedKeys(stats) {
for _, alabel := range sortedKeys {
astats, ok := stats[alabel]
if !ok {
continue
}
row := []string{
alabel,
}
for _, sl := range keys {
if v, ok := astats[sl]; ok && v != 0 {
numberToShow := strconv.Itoa(v)
numberToShow := fmt.Sprintf("%d", v)
if !noUnit {
numberToShow = formatNumber(v)
}
@ -121,192 +75,45 @@ func metricsToTable(t *table.Table, stats map[string]map[string]int, keys []stri
row = append(row, "-")
}
}
t.AddRow(row...)
numRows++
}
return numRows, nil
}
func (s statBucket) Description() (string, string) {
return "Scenario Metrics",
`Measure events in different scenarios. Current count is the number of buckets during metrics collection. ` +
`Overflows are past event-producing buckets, while Expired are the ones that didnt receive enough events to Overflow.`
}
func (s statBucket) Process(bucket, metric string, val int) {
if _, ok := s[bucket]; !ok {
s[bucket] = make(map[string]int)
}
s[bucket][metric] += val
}
func (s statBucket) Table(out io.Writer, noUnit bool, showEmpty bool) {
func bucketStatsTable(out io.Writer, stats map[string]map[string]int) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Scenario", "Current Count", "Overflows", "Instantiated", "Poured", "Expired")
t.SetHeaders("Bucket", "Current Count", "Overflows", "Instantiated", "Poured", "Expired")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
keys := []string{"curr_count", "overflow", "instantiation", "pour", "underflow"}
if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
log.Warningf("while collecting scenario stats: %s", err)
} else if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
if numRows, err := metricsToTable(t, stats, keys); err != nil {
log.Warningf("while collecting acquis stats: %s", err)
} else if numRows > 0 {
renderTableTitle(out, "\nBucket Metrics:")
t.Render()
}
}
func (s statAcquis) Description() (string, string) {
return "Acquisition Metrics",
`Measures the lines read, parsed, and unparsed per datasource. ` +
`Zero read lines indicate a misconfigured or inactive datasource. ` +
`Zero parsed lines mean the parser(s) failed. ` +
`Non-zero parsed lines are fine as crowdsec selects relevant lines.`
}
func (s statAcquis) Process(source, metric string, val int) {
if _, ok := s[source]; !ok {
s[source] = make(map[string]int)
}
s[source][metric] += val
}
func (s statAcquis) Table(out io.Writer, noUnit bool, showEmpty bool) {
func acquisStatsTable(out io.Writer, stats map[string]map[string]int) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Source", "Lines read", "Lines parsed", "Lines unparsed", "Lines poured to bucket", "Lines whitelisted")
t.SetHeaders("Source", "Lines read", "Lines parsed", "Lines unparsed", "Lines poured to bucket")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
keys := []string{"reads", "parsed", "unparsed", "pour", "whitelisted"}
keys := []string{"reads", "parsed", "unparsed", "pour"}
if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
if numRows, err := metricsToTable(t, stats, keys); err != nil {
log.Warningf("while collecting acquis stats: %s", err)
} else if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
} else if numRows > 0 {
renderTableTitle(out, "\nAcquisition Metrics:")
t.Render()
}
}
func (s statAppsecEngine) Description() (string, string) {
return "Appsec Metrics",
`Measures the number of parsed and blocked requests by the AppSec Component.`
}
func (s statAppsecEngine) Process(appsecEngine, metric string, val int) {
if _, ok := s[appsecEngine]; !ok {
s[appsecEngine] = make(map[string]int)
}
s[appsecEngine][metric] += val
}
func (s statAppsecEngine) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Appsec Engine", "Processed", "Blocked")
t.SetAlignment(table.AlignLeft, table.AlignLeft)
keys := []string{"processed", "blocked"}
if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
log.Warningf("while collecting appsec stats: %s", err)
} else if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func (s statAppsecRule) Description() (string, string) {
return "Appsec Rule Metrics",
`Provides “per AppSec Component” information about the number of matches for loaded AppSec Rules.`
}
func (s statAppsecRule) Process(appsecEngine, appsecRule string, metric string, val int) {
if _, ok := s[appsecEngine]; !ok {
s[appsecEngine] = make(map[string]map[string]int)
}
if _, ok := s[appsecEngine][appsecRule]; !ok {
s[appsecEngine][appsecRule] = make(map[string]int)
}
s[appsecEngine][appsecRule][metric] += val
}
func (s statAppsecRule) Table(out io.Writer, noUnit bool, showEmpty bool) {
for appsecEngine, appsecEngineRulesStats := range s {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Rule ID", "Triggered")
t.SetAlignment(table.AlignLeft, table.AlignLeft)
keys := []string{"triggered"}
if numRows, err := metricsToTable(t, appsecEngineRulesStats, keys, noUnit); err != nil {
log.Warningf("while collecting appsec rules stats: %s", err)
} else if numRows > 0 || showEmpty {
renderTableTitle(out, fmt.Sprintf("\nAppsec '%s' Rules Metrics:", appsecEngine))
t.Render()
}
}
}
func (s statWhitelist) Description() (string, string) {
return "Whitelist Metrics",
`Tracks the number of events processed and possibly whitelisted by each parser whitelist.`
}
func (s statWhitelist) Process(whitelist, reason, metric string, val int) {
if _, ok := s[whitelist]; !ok {
s[whitelist] = make(map[string]map[string]int)
}
if _, ok := s[whitelist][reason]; !ok {
s[whitelist][reason] = make(map[string]int)
}
s[whitelist][reason][metric] += val
}
func (s statWhitelist) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Whitelist", "Reason", "Hits", "Whitelisted")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
if numRows, err := wlMetricsToTable(t, s, noUnit); err != nil {
log.Warningf("while collecting parsers stats: %s", err)
} else if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
t.Render()
}
}
func (s statParser) Description() (string, string) {
return "Parser Metrics",
`Tracks the number of events processed by each parser and indicates success of failure. ` +
`Zero parsed lines means the parer(s) failed. ` +
`Non-zero unparsed lines are fine as crowdsec select relevant lines.`
}
func (s statParser) Process(parser, metric string, val int) {
if _, ok := s[parser]; !ok {
s[parser] = make(map[string]int)
}
s[parser][metric] += val
}
func (s statParser) Table(out io.Writer, noUnit bool, showEmpty bool) {
func parserStatsTable(out io.Writer, stats map[string]map[string]int) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Parsers", "Hits", "Parsed", "Unparsed")
@ -314,302 +121,187 @@ func (s statParser) Table(out io.Writer, noUnit bool, showEmpty bool) {
keys := []string{"hits", "parsed", "unparsed"}
if numRows, err := metricsToTable(t, s, keys, noUnit); err != nil {
log.Warningf("while collecting parsers stats: %s", err)
} else if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
if numRows, err := metricsToTable(t, stats, keys); err != nil {
log.Warningf("while collecting acquis stats: %s", err)
} else if numRows > 0 {
renderTableTitle(out, "\nParser Metrics:")
t.Render()
}
}
func (s statStash) Description() (string, string) {
return "Parser Stash Metrics",
`Tracks the status of stashes that might be created by various parsers and scenarios.`
}
func stashStatsTable(out io.Writer, stats map[string]struct {
Type string
Count int
}) {
func (s statStash) Process(name, mtype string, val int) {
s[name] = struct {
Type string
Count int
}{
Type: mtype,
Count: val,
}
}
func (s statStash) Table(out io.Writer, noUnit bool, showEmpty bool) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Name", "Type", "Items")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
// unfortunately, we can't reuse metricsToTable as the structure is too different :/
numRows := 0
sortedKeys := []string{}
for k := range stats {
sortedKeys = append(sortedKeys, k)
}
sort.Strings(sortedKeys)
for _, alabel := range maptools.SortedKeys(s) {
astats := s[alabel]
numRows := 0
for _, alabel := range sortedKeys {
astats := stats[alabel]
row := []string{
alabel,
astats.Type,
strconv.Itoa(astats.Count),
fmt.Sprintf("%d", astats.Count),
}
t.AddRow(row...)
numRows++
}
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
if numRows > 0 {
renderTableTitle(out, "\nParser Stash Metrics:")
t.Render()
}
}
func (s statLapi) Description() (string, string) {
return "Local API Metrics",
`Monitors the requests made to local API routes.`
}
func (s statLapi) Process(route, method string, val int) {
if _, ok := s[route]; !ok {
s[route] = make(map[string]int)
}
s[route][method] += val
}
func (s statLapi) Table(out io.Writer, noUnit bool, showEmpty bool) {
func lapiStatsTable(out io.Writer, stats map[string]map[string]int) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Route", "Method", "Hits")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
// unfortunately, we can't reuse metricsToTable as the structure is too different :/
numRows := 0
sortedKeys := []string{}
for k := range stats {
sortedKeys = append(sortedKeys, k)
}
sort.Strings(sortedKeys)
for _, alabel := range maptools.SortedKeys(s) {
astats := s[alabel]
numRows := 0
for _, alabel := range sortedKeys {
astats := stats[alabel]
subKeys := []string{}
for skey := range astats {
subKeys = append(subKeys, skey)
}
sort.Strings(subKeys)
for _, sl := range subKeys {
row := []string{
alabel,
sl,
strconv.Itoa(astats[sl]),
fmt.Sprintf("%d", astats[sl]),
}
t.AddRow(row...)
numRows++
}
}
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
if numRows > 0 {
renderTableTitle(out, "\nLocal Api Metrics:")
t.Render()
}
}
func (s statLapiMachine) Description() (string, string) {
return "Local API Machines Metrics",
`Tracks the number of calls to the local API from each registered machine.`
}
func (s statLapiMachine) Process(machine, route, method string, val int) {
if _, ok := s[machine]; !ok {
s[machine] = make(map[string]map[string]int)
}
if _, ok := s[machine][route]; !ok {
s[machine][route] = make(map[string]int)
}
s[machine][route][method] += val
}
func (s statLapiMachine) Table(out io.Writer, noUnit bool, showEmpty bool) {
func lapiMachineStatsTable(out io.Writer, stats map[string]map[string]map[string]int) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Machine", "Route", "Method", "Hits")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
numRows := lapiMetricsToTable(t, s)
numRows := lapiMetricsToTable(t, stats)
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
if numRows > 0 {
renderTableTitle(out, "\nLocal Api Machines Metrics:")
t.Render()
}
}
func (s statLapiBouncer) Description() (string, string) {
return "Local API Bouncers Metrics",
`Tracks total hits to remediation component related API routes.`
}
func (s statLapiBouncer) Process(bouncer, route, method string, val int) {
if _, ok := s[bouncer]; !ok {
s[bouncer] = make(map[string]map[string]int)
}
if _, ok := s[bouncer][route]; !ok {
s[bouncer][route] = make(map[string]int)
}
s[bouncer][route][method] += val
}
func (s statLapiBouncer) Table(out io.Writer, noUnit bool, showEmpty bool) {
func lapiBouncerStatsTable(out io.Writer, stats map[string]map[string]map[string]int) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Bouncer", "Route", "Method", "Hits")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
numRows := lapiMetricsToTable(t, s)
numRows := lapiMetricsToTable(t, stats)
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
if numRows > 0 {
renderTableTitle(out, "\nLocal Api Bouncers Metrics:")
t.Render()
}
}
func (s statLapiDecision) Description() (string, string) {
return "Local API Bouncers Decisions",
`Tracks the number of empty/non-empty answers from LAPI to bouncers that are working in "live" mode.`
}
func (s statLapiDecision) Process(bouncer, fam string, val int) {
if _, ok := s[bouncer]; !ok {
s[bouncer] = struct {
NonEmpty int
Empty int
}{}
}
x := s[bouncer]
switch fam {
case "cs_lapi_decisions_ko_total":
x.Empty += val
case "cs_lapi_decisions_ok_total":
x.NonEmpty += val
}
s[bouncer] = x
}
func (s statLapiDecision) Table(out io.Writer, noUnit bool, showEmpty bool) {
func lapiDecisionStatsTable(out io.Writer, stats map[string]struct {
NonEmpty int
Empty int
},
) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Bouncer", "Empty answers", "Non-empty answers")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
numRows := 0
for bouncer, hits := range s {
for bouncer, hits := range stats {
t.AddRow(
bouncer,
strconv.Itoa(hits.Empty),
strconv.Itoa(hits.NonEmpty),
fmt.Sprintf("%d", hits.Empty),
fmt.Sprintf("%d", hits.NonEmpty),
)
numRows++
}
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
if numRows > 0 {
renderTableTitle(out, "\nLocal Api Bouncers Decisions:")
t.Render()
}
}
func (s statDecision) Description() (string, string) {
return "Local API Decisions",
`Provides information about all currently active decisions. ` +
`Includes both local (crowdsec) and global decisions (CAPI), and lists subscriptions (lists).`
}
func (s statDecision) Process(reason, origin, action string, val int) {
if _, ok := s[reason]; !ok {
s[reason] = make(map[string]map[string]int)
}
if _, ok := s[reason][origin]; !ok {
s[reason][origin] = make(map[string]int)
}
s[reason][origin][action] += val
}
func (s statDecision) Table(out io.Writer, noUnit bool, showEmpty bool) {
func decisionStatsTable(out io.Writer, stats map[string]map[string]map[string]int) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Reason", "Origin", "Action", "Count")
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
numRows := 0
for reason, origins := range s {
for reason, origins := range stats {
for origin, actions := range origins {
for action, hits := range actions {
t.AddRow(
reason,
origin,
action,
strconv.Itoa(hits),
fmt.Sprintf("%d", hits),
)
numRows++
}
}
}
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
if numRows > 0 {
renderTableTitle(out, "\nLocal Api Decisions:")
t.Render()
}
}
func (s statAlert) Description() (string, string) {
return "Local API Alerts",
`Tracks the total number of past and present alerts for the installed scenarios.`
}
func (s statAlert) Process(reason string, val int) {
s[reason] += val
}
func (s statAlert) Table(out io.Writer, noUnit bool, showEmpty bool) {
func alertStatsTable(out io.Writer, stats map[string]int) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Reason", "Count")
t.SetAlignment(table.AlignLeft, table.AlignLeft)
numRows := 0
for scenario, hits := range s {
for scenario, hits := range stats {
t.AddRow(
scenario,
strconv.Itoa(hits),
fmt.Sprintf("%d", hits),
)
numRows++
}
if numRows > 0 || showEmpty {
title, _ := s.Description()
renderTableTitle(out, "\n"+title+":")
if numRows > 0 {
renderTableTitle(out, "\nLocal Api Alerts:")
t.Render()
}
}

View file

@ -4,7 +4,6 @@ import (
"context"
"encoding/csv"
"encoding/json"
"errors"
"fmt"
"io/fs"
"net/url"
@ -16,171 +15,147 @@ import (
"github.com/fatih/color"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/csplugin"
"github.com/crowdsecurity/crowdsec/pkg/csprofiles"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
)
type NotificationsCfg struct {
Config csplugin.PluginConfig `json:"plugin_config"`
Profiles []*csconfig.ProfileCfg `json:"associated_profiles"`
ids []uint
}
type cliNotifications struct {
cfg configGetter
}
func NewCLINotifications(cfg configGetter) *cliNotifications {
return &cliNotifications{
cfg: cfg,
}
}
func (cli *cliNotifications) NewCommand() *cobra.Command {
cmd := &cobra.Command{
func NewNotificationsCmd() *cobra.Command {
var cmdNotifications = &cobra.Command{
Use: "notifications [action]",
Short: "Helper for notification plugin configuration",
Long: "To list/inspect/test notification template",
Args: cobra.MinimumNArgs(1),
Aliases: []string{"notifications", "notification"},
DisableAutoGenTag: true,
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := require.LAPI(cfg); err != nil {
return err
PersistentPreRun: func(cmd *cobra.Command, args []string) {
var (
err error
)
if err = csConfig.API.Server.LoadProfiles(); err != nil {
log.Fatal(err)
}
if err := cfg.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
if csConfig.ConfigPaths.NotificationDir == "" {
log.Fatalf("config_paths.notification_dir is not set in crowdsec config")
}
if err := require.Notifications(cfg); err != nil {
return err
}
return nil
},
}
cmd.AddCommand(cli.NewListCmd())
cmd.AddCommand(cli.NewInspectCmd())
cmd.AddCommand(cli.NewReinjectCmd())
cmd.AddCommand(cli.NewTestCmd())
return cmd
cmdNotifications.AddCommand(NewNotificationsListCmd())
cmdNotifications.AddCommand(NewNotificationsInspectCmd())
cmdNotifications.AddCommand(NewNotificationsReinjectCmd())
return cmdNotifications
}
func (cli *cliNotifications) getPluginConfigs() (map[string]csplugin.PluginConfig, error) {
cfg := cli.cfg()
func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
pcfgs := map[string]csplugin.PluginConfig{}
wf := func(path string, info fs.FileInfo, err error) error {
if info == nil {
return fmt.Errorf("error while traversing directory %s: %w", path, err)
return errors.Wrapf(err, "error while traversing directory %s", path)
}
name := filepath.Join(cfg.ConfigPaths.NotificationDir, info.Name()) // Avoid calling info.Name() twice
name := filepath.Join(csConfig.ConfigPaths.NotificationDir, info.Name()) //Avoid calling info.Name() twice
if (strings.HasSuffix(name, "yaml") || strings.HasSuffix(name, "yml")) && !(info.IsDir()) {
ts, err := csplugin.ParsePluginConfigFile(name)
if err != nil {
return fmt.Errorf("loading notifification plugin configuration with %s: %w", name, err)
return errors.Wrapf(err, "Loading notifification plugin configuration with %s", name)
}
for _, t := range ts {
csplugin.SetRequiredFields(&t)
pcfgs[t.Name] = t
}
}
return nil
}
if err := filepath.Walk(cfg.ConfigPaths.NotificationDir, wf); err != nil {
return nil, fmt.Errorf("while loading notifification plugin configuration: %w", err)
if err := filepath.Walk(csConfig.ConfigPaths.NotificationDir, wf); err != nil {
return nil, errors.Wrap(err, "Loading notifification plugin configuration")
}
return pcfgs, nil
}
func (cli *cliNotifications) getProfilesConfigs() (map[string]NotificationsCfg, error) {
cfg := cli.cfg()
// A bit of a tricky stuf now: reconcile profiles and notification plugins
pcfgs, err := cli.getPluginConfigs()
if err != nil {
return nil, err
}
ncfgs := map[string]NotificationsCfg{}
for _, pc := range pcfgs {
ncfgs[pc.Name] = NotificationsCfg{
Config: pc,
}
}
profiles, err := csprofiles.NewProfile(cfg.API.Server.Profiles)
profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
if err != nil {
return nil, fmt.Errorf("while extracting profiles from configuration: %w", err)
return nil, errors.Wrap(err, "Cannot extract profiles from configuration")
}
for profileID, profile := range profiles {
loop:
for _, notif := range profile.Cfg.Notifications {
pc, ok := pcfgs[notif]
if !ok {
return nil, fmt.Errorf("notification plugin '%s' does not exist", notif)
for name, pc := range pcfgs {
if notif == name {
if _, ok := ncfgs[pc.Name]; !ok {
ncfgs[pc.Name] = NotificationsCfg{
Config: pc,
Profiles: []*csconfig.ProfileCfg{profile.Cfg},
ids: []uint{uint(profileID)},
}
continue loop
}
tmp := ncfgs[pc.Name]
for _, pr := range tmp.Profiles {
var profiles []*csconfig.ProfileCfg
if pr.Name == profile.Cfg.Name {
continue
}
profiles = append(tmp.Profiles, profile.Cfg)
ids := append(tmp.ids, uint(profileID))
ncfgs[pc.Name] = NotificationsCfg{
Config: tmp.Config,
Profiles: profiles,
ids: ids,
}
}
}
}
tmp, ok := ncfgs[pc.Name]
if !ok {
return nil, fmt.Errorf("notification plugin '%s' does not exist", pc.Name)
}
tmp.Profiles = append(tmp.Profiles, profile.Cfg)
tmp.ids = append(tmp.ids, uint(profileID))
ncfgs[pc.Name] = tmp
}
}
return ncfgs, nil
}
func (cli *cliNotifications) NewListCmd() *cobra.Command {
cmd := &cobra.Command{
func NewNotificationsListCmd() *cobra.Command {
var cmdNotificationsList = &cobra.Command{
Use: "list",
Short: "list active notifications plugins",
Long: `list active notifications plugins`,
Short: "List active notifications plugins",
Long: `List active notifications plugins`,
Example: `cscli notifications list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
ncfgs, err := cli.getProfilesConfigs()
RunE: func(cmd *cobra.Command, arg []string) error {
ncfgs, err := getNotificationsConfiguration()
if err != nil {
return fmt.Errorf("can't build profiles configuration: %w", err)
return errors.Wrap(err, "Can't build profiles configuration")
}
if cfg.Cscli.Output == "human" {
if csConfig.Cscli.Output == "human" {
notificationListTable(color.Output, ncfgs)
} else if cfg.Cscli.Output == "json" {
} else if csConfig.Cscli.Output == "json" {
x, err := json.MarshalIndent(ncfgs, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal notification configuration: %w", err)
return errors.New("failed to marshal notification configuration")
}
fmt.Printf("%s", string(x))
} else if cfg.Cscli.Output == "raw" {
} else if csConfig.Cscli.Output == "raw" {
csvwriter := csv.NewWriter(os.Stdout)
err := csvwriter.Write([]string{"Name", "Type", "Profile name"})
if err != nil {
return fmt.Errorf("failed to write raw header: %w", err)
return errors.Wrap(err, "failed to write raw header")
}
for _, b := range ncfgs {
profilesList := []string{}
@ -189,193 +164,140 @@ func (cli *cliNotifications) NewListCmd() *cobra.Command {
}
err := csvwriter.Write([]string{b.Config.Name, b.Config.Type, strings.Join(profilesList, ", ")})
if err != nil {
return fmt.Errorf("failed to write raw content: %w", err)
return errors.Wrap(err, "failed to write raw content")
}
}
csvwriter.Flush()
}
return nil
},
}
return cmd
return cmdNotificationsList
}
func (cli *cliNotifications) NewInspectCmd() *cobra.Command {
cmd := &cobra.Command{
func NewNotificationsInspectCmd() *cobra.Command {
var cmdNotificationsInspect = &cobra.Command{
Use: "inspect",
Short: "Inspect active notifications plugin configuration",
Long: `Inspect active notifications plugin and show configuration`,
Example: `cscli notifications inspect <plugin_name>`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, args []string) error {
cfg := cli.cfg()
ncfgs, err := cli.getProfilesConfigs()
RunE: func(cmd *cobra.Command, arg []string) error {
var (
cfg NotificationsCfg
ok bool
)
pluginName := arg[0]
if pluginName == "" {
errors.New("Please provide a plugin name to inspect")
}
ncfgs, err := getNotificationsConfiguration()
if err != nil {
return fmt.Errorf("can't build profiles configuration: %w", err)
return errors.Wrap(err, "Can't build profiles configuration")
}
ncfg, ok := ncfgs[args[0]]
if !ok {
return fmt.Errorf("plugin '%s' does not exist or is not active", args[0])
if cfg, ok = ncfgs[pluginName]; !ok {
return errors.New("The provided plugin name doesn't exist or isn't active")
}
if cfg.Cscli.Output == "human" || cfg.Cscli.Output == "raw" {
fmt.Printf(" - %15s: %15s\n", "Type", ncfg.Config.Type)
fmt.Printf(" - %15s: %15s\n", "Name", ncfg.Config.Name)
fmt.Printf(" - %15s: %15s\n", "Timeout", ncfg.Config.TimeOut)
fmt.Printf(" - %15s: %15s\n", "Format", ncfg.Config.Format)
for k, v := range ncfg.Config.Config {
if csConfig.Cscli.Output == "human" || csConfig.Cscli.Output == "raw" {
fmt.Printf(" - %15s: %15s\n", "Type", cfg.Config.Type)
fmt.Printf(" - %15s: %15s\n", "Name", cfg.Config.Name)
fmt.Printf(" - %15s: %15s\n", "Timeout", cfg.Config.TimeOut)
fmt.Printf(" - %15s: %15s\n", "Format", cfg.Config.Format)
for k, v := range cfg.Config.Config {
fmt.Printf(" - %15s: %15v\n", k, v)
}
} else if cfg.Cscli.Output == "json" {
} else if csConfig.Cscli.Output == "json" {
x, err := json.MarshalIndent(cfg, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal notification configuration: %w", err)
return errors.New("failed to marshal notification configuration")
}
fmt.Printf("%s", string(x))
}
return nil
},
}
return cmd
return cmdNotificationsInspect
}
func (cli *cliNotifications) NewTestCmd() *cobra.Command {
var (
pluginBroker csplugin.PluginBroker
pluginTomb tomb.Tomb
alertOverride string
)
cmd := &cobra.Command{
Use: "test [plugin name]",
Short: "send a generic test alert to notification plugin",
Long: `send a generic test alert to a notification plugin to test configuration even if is not active`,
Example: `cscli notifications test [plugin_name]`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
PreRunE: func(_ *cobra.Command, args []string) error {
cfg := cli.cfg()
pconfigs, err := cli.getPluginConfigs()
if err != nil {
return fmt.Errorf("can't build profiles configuration: %w", err)
}
pcfg, ok := pconfigs[args[0]]
if !ok {
return fmt.Errorf("plugin name: '%s' does not exist", args[0])
}
// Create a single profile with plugin name as notification name
return pluginBroker.Init(cfg.PluginConfig, []*csconfig.ProfileCfg{
{
Notifications: []string{
pcfg.Name,
},
},
}, cfg.ConfigPaths)
},
RunE: func(_ *cobra.Command, _ []string) error {
pluginTomb.Go(func() error {
pluginBroker.Run(&pluginTomb)
return nil
})
alert := &models.Alert{
Capacity: ptr.Of(int32(0)),
Decisions: []*models.Decision{{
Duration: ptr.Of("4h"),
Scope: ptr.Of("Ip"),
Value: ptr.Of("10.10.10.10"),
Type: ptr.Of("ban"),
Scenario: ptr.Of("test alert"),
Origin: ptr.Of(types.CscliOrigin),
}},
Events: []*models.Event{},
EventsCount: ptr.Of(int32(1)),
Leakspeed: ptr.Of("0"),
Message: ptr.Of("test alert"),
ScenarioHash: ptr.Of(""),
Scenario: ptr.Of("test alert"),
ScenarioVersion: ptr.Of(""),
Simulated: ptr.Of(false),
Source: &models.Source{
AsName: "",
AsNumber: "",
Cn: "",
IP: "10.10.10.10",
Range: "",
Scope: ptr.Of("Ip"),
Value: ptr.Of("10.10.10.10"),
},
StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
CreatedAt: time.Now().UTC().Format(time.RFC3339),
}
if err := yaml.Unmarshal([]byte(alertOverride), alert); err != nil {
return fmt.Errorf("failed to unmarshal alert override: %w", err)
}
func NewNotificationsReinjectCmd() *cobra.Command {
var remediation bool
var alertOverride string
pluginBroker.PluginChannel <- csplugin.ProfileAlert{
ProfileID: uint(0),
Alert: alert,
}
// time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
pluginTomb.Kill(errors.New("terminating"))
pluginTomb.Wait()
return nil
},
}
cmd.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the generic alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
return cmd
}
func (cli *cliNotifications) NewReinjectCmd() *cobra.Command {
var (
alertOverride string
alert *models.Alert
)
cmd := &cobra.Command{
var cmdNotificationsReinject = &cobra.Command{
Use: "reinject",
Short: "reinject an alert into profiles to trigger notifications",
Long: `reinject an alert into profiles to be evaluated by the filter and sent to matched notifications plugins`,
Short: "reinject alert into notifications system",
Long: `Reinject alert into notifications system`,
Example: `
cscli notifications reinject <alert_id>
cscli notifications reinject <alert_id> -a '{"remediation": false,"scenario":"notification/test"}'
cscli notifications reinject <alert_id> --remediation
cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"notification/test"}'
`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
PreRunE: func(_ *cobra.Command, args []string) error {
var err error
alert, err = cli.fetchAlertFromArgString(args[0])
if err != nil {
return err
}
return nil
},
RunE: func(_ *cobra.Command, _ []string) error {
RunE: func(cmd *cobra.Command, args []string) error {
var (
pluginBroker csplugin.PluginBroker
pluginTomb tomb.Tomb
)
cfg := cli.cfg()
if alertOverride != "" {
if err := json.Unmarshal([]byte(alertOverride), alert); err != nil {
return fmt.Errorf("can't unmarshal data in the alert flag: %w", err)
}
if len(args) != 1 {
printHelp(cmd)
return errors.New("Wrong number of argument: there should be one argument")
}
err := pluginBroker.Init(cfg.PluginConfig, cfg.API.Server.Profiles, cfg.ConfigPaths)
//first: get the alert
id, err := strconv.Atoi(args[0])
if err != nil {
return fmt.Errorf("can't initialize plugins: %w", err)
return errors.New(fmt.Sprintf("bad alert id %s", args[0]))
}
if err := csConfig.LoadAPIClient(); err != nil {
return errors.Wrapf(err, "loading api client")
}
if csConfig.API.Client == nil {
return errors.New("There is no configuration on 'api_client:'")
}
if csConfig.API.Client.Credentials == nil {
return errors.New(fmt.Sprintf("Please provide credentials for the API in '%s'", csConfig.API.Client.CredentialsFilePath))
}
apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
if err != nil {
return errors.Wrapf(err, "error parsing the URL of the API")
}
client, err := apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
Password: strfmt.Password(csConfig.API.Client.Credentials.Password),
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
URL: apiURL,
VersionPrefix: "v1",
})
if err != nil {
return errors.Wrapf(err, "error creating the client for the API")
}
alert, _, err := client.Alerts.GetByID(context.Background(), id)
if err != nil {
return errors.Wrapf(err, fmt.Sprintf("can't find alert with id %s", args[0]))
}
if alertOverride != "" {
if err = json.Unmarshal([]byte(alertOverride), alert); err != nil {
return errors.Wrapf(err, "Can't unmarshal the data given in the alert flag")
}
}
if !remediation {
alert.Remediation = true
}
// second we start plugins
err = pluginBroker.Init(csConfig.PluginConfig, csConfig.API.Server.Profiles, csConfig.ConfigPaths)
if err != nil {
return errors.Wrapf(err, "Can't initialize plugins")
}
pluginTomb.Go(func() error {
@ -383,15 +305,17 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
return nil
})
profiles, err := csprofiles.NewProfile(cfg.API.Server.Profiles)
//third: get the profile(s), and process the whole stuff
profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
if err != nil {
return fmt.Errorf("cannot extract profiles from configuration: %w", err)
return errors.Wrap(err, "Cannot extract profiles from configuration")
}
for id, profile := range profiles {
_, matched, err := profile.EvaluateProfile(alert)
if err != nil {
return fmt.Errorf("can't evaluate profile %s: %w", profile.Cfg.Name, err)
return errors.Wrapf(err, "can't evaluate profile %s", profile.Cfg.Name)
}
if !matched {
log.Infof("The profile %s didn't match", profile.Cfg.Name)
@ -409,54 +333,23 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
default:
time.Sleep(50 * time.Millisecond)
log.Info("sleeping\n")
}
}
if profile.Cfg.OnSuccess == "break" {
log.Infof("The profile %s contains a 'on_success: break' so bailing out", profile.Cfg.Name)
break
}
}
// time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
// time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
pluginTomb.Kill(errors.New("terminating"))
pluginTomb.Wait()
return nil
},
}
cmd.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the reinjected alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
cmdNotificationsReinject.Flags().BoolVarP(&remediation, "remediation", "r", false, "Set Alert.Remediation to false in the reinjected alert (see your profile filter configuration)")
cmdNotificationsReinject.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the reinjected alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
return cmd
}
func (cli *cliNotifications) fetchAlertFromArgString(toParse string) (*models.Alert, error) {
cfg := cli.cfg()
id, err := strconv.Atoi(toParse)
if err != nil {
return nil, fmt.Errorf("bad alert id %s", toParse)
}
apiURL, err := url.Parse(cfg.API.Client.Credentials.URL)
if err != nil {
return nil, fmt.Errorf("error parsing the URL of the API: %w", err)
}
client, err := apiclient.NewClient(&apiclient.Config{
MachineID: cfg.API.Client.Credentials.Login,
Password: strfmt.Password(cfg.API.Client.Credentials.Password),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
})
if err != nil {
return nil, fmt.Errorf("error creating the client for the API: %w", err)
}
alert, _, err := client.Alerts.GetByID(context.Background(), id)
if err != nil {
return nil, fmt.Errorf("can't find alert with id %d: %w", id, err)
}
return alert, nil
return cmdNotificationsReinject
}

View file

@ -2,43 +2,23 @@ package main
import (
"io"
"sort"
"strings"
"github.com/aquasecurity/table"
"github.com/crowdsecurity/crowdsec/pkg/emoji"
)
func notificationListTable(out io.Writer, ncfgs map[string]NotificationsCfg) {
t := newLightTable(out)
t.SetHeaders("Active", "Name", "Type", "Profile name")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetHeaders("Name", "Type", "Profile name")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
keys := make([]string, 0, len(ncfgs))
for k := range ncfgs {
keys = append(keys, k)
}
sort.Slice(keys, func(i, j int) bool {
return len(ncfgs[keys[i]].Profiles) > len(ncfgs[keys[j]].Profiles)
})
for _, k := range keys {
b := ncfgs[k]
for _, b := range ncfgs {
profilesList := []string{}
for _, p := range b.Profiles {
profilesList = append(profilesList, p.Name)
}
active := emoji.CheckMark
if len(profilesList) == 0 {
active = emoji.Prohibited
}
t.AddRow(active, b.Config.Name, b.Config.Type, strings.Join(profilesList, ", "))
t.AddRow(b.Config.Name, b.Config.Type, strings.Join(profilesList, ", "))
}
t.Render()

View file

@ -1,90 +1,77 @@
package main
import (
"fmt"
"time"
"github.com/crowdsecurity/crowdsec/pkg/apiserver"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/tomb.v2"
"github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiserver"
"github.com/crowdsecurity/crowdsec/pkg/database"
)
type cliPapi struct {
cfg configGetter
}
func NewCLIPapi(cfg configGetter) *cliPapi {
return &cliPapi{
cfg: cfg,
}
}
func (cli *cliPapi) NewCommand() *cobra.Command {
cmd := &cobra.Command{
func NewPapiCmd() *cobra.Command {
var cmdLapi = &cobra.Command{
Use: "papi [action]",
Short: "Manage interaction with Polling API (PAPI)",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
cfg := cli.cfg()
if err := require.LAPI(cfg); err != nil {
return err
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
return errors.Wrap(err, "Local API is disabled, please run this command on the local API machine")
}
if err := require.CAPI(cfg); err != nil {
return err
if csConfig.API.Server.OnlineClient == nil {
log.Fatalf("no configuration for Central API in '%s'", *csConfig.FilePath)
}
if err := require.PAPI(cfg); err != nil {
return err
if csConfig.API.Server.OnlineClient.Credentials.PapiURL == "" {
log.Fatalf("no PAPI URL in configuration")
}
return nil
},
}
cmd.AddCommand(cli.NewStatusCmd())
cmd.AddCommand(cli.NewSyncCmd())
cmdLapi.AddCommand(NewPapiStatusCmd())
cmdLapi.AddCommand(NewPapiSyncCmd())
return cmd
return cmdLapi
}
func (cli *cliPapi) NewStatusCmd() *cobra.Command {
cmd := &cobra.Command{
func NewPapiStatusCmd() *cobra.Command {
cmdCapiStatus := &cobra.Command{
Use: "status",
Short: "Get status of the Polling API",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
Run: func(cmd *cobra.Command, args []string) {
var err error
cfg := cli.cfg()
dbClient, err = database.NewClient(cfg.DbConfig)
dbClient, err = database.NewClient(csConfig.DbConfig)
if err != nil {
return fmt.Errorf("unable to initialize database client: %w", err)
log.Fatalf("unable to initialize database client : %s", err)
}
apic, err := apiserver.NewAPIC(cfg.API.Server.OnlineClient, dbClient, cfg.API.Server.ConsoleConfig, cfg.API.Server.CapiWhitelists)
apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig, csConfig.API.Server.CapiWhitelists)
if err != nil {
return fmt.Errorf("unable to initialize API client: %w", err)
log.Fatalf("unable to initialize API client : %s", err)
}
papi, err := apiserver.NewPAPI(apic, dbClient, cfg.API.Server.ConsoleConfig, log.GetLevel())
papi, err := apiserver.NewPAPI(apic, dbClient, csConfig.API.Server.ConsoleConfig, log.GetLevel())
if err != nil {
return fmt.Errorf("unable to initialize PAPI client: %w", err)
log.Fatalf("unable to initialize PAPI client : %s", err)
}
perms, err := papi.GetPermissions()
if err != nil {
return fmt.Errorf("unable to get PAPI permissions: %w", err)
log.Fatalf("unable to get PAPI permissions: %s", err)
}
var lastTimestampStr *string
lastTimestampStr, err = dbClient.GetConfigItem(apiserver.PapiPullKey)
if err != nil {
lastTimestampStr = ptr.Of("never")
lastTimestampStr = types.StrPtr("never")
}
log.Infof("You can successfully interact with Polling API (PAPI)")
log.Infof("Console plan: %s", perms.Plan)
@ -94,47 +81,45 @@ func (cli *cliPapi) NewStatusCmd() *cobra.Command {
for _, sub := range perms.Categories {
log.Infof(" - %s", sub)
}
return nil
},
}
return cmd
return cmdCapiStatus
}
func (cli *cliPapi) NewSyncCmd() *cobra.Command {
cmd := &cobra.Command{
func NewPapiSyncCmd() *cobra.Command {
cmdCapiSync := &cobra.Command{
Use: "sync",
Short: "Sync with the Polling API, pulling all non-expired orders for the instance",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
Run: func(cmd *cobra.Command, args []string) {
var err error
cfg := cli.cfg()
t := tomb.Tomb{}
dbClient, err = database.NewClient(cfg.DbConfig)
dbClient, err = database.NewClient(csConfig.DbConfig)
if err != nil {
return fmt.Errorf("unable to initialize database client: %w", err)
log.Fatalf("unable to initialize database client : %s", err)
}
apic, err := apiserver.NewAPIC(cfg.API.Server.OnlineClient, dbClient, cfg.API.Server.ConsoleConfig, cfg.API.Server.CapiWhitelists)
apic, err := apiserver.NewAPIC(csConfig.API.Server.OnlineClient, dbClient, csConfig.API.Server.ConsoleConfig, csConfig.API.Server.CapiWhitelists)
if err != nil {
return fmt.Errorf("unable to initialize API client: %w", err)
log.Fatalf("unable to initialize API client : %s", err)
}
t.Go(apic.Push)
papi, err := apiserver.NewPAPI(apic, dbClient, cfg.API.Server.ConsoleConfig, log.GetLevel())
if err != nil {
return fmt.Errorf("unable to initialize PAPI client: %w", err)
}
papi, err := apiserver.NewPAPI(apic, dbClient, csConfig.API.Server.ConsoleConfig, log.GetLevel())
if err != nil {
log.Fatalf("unable to initialize PAPI client : %s", err)
}
t.Go(papi.SyncDecisions)
err = papi.PullOnce(time.Time{}, true)
if err != nil {
return fmt.Errorf("unable to sync decisions: %w", err)
log.Fatalf("unable to sync decisions: %s", err)
}
log.Infof("Sending acknowledgements to CAPI")
@ -144,9 +129,8 @@ func (cli *cliPapi) NewSyncCmd() *cobra.Command {
t.Wait()
time.Sleep(5 * time.Second) //FIXME: the push done by apic.Push is run inside a sub goroutine, sleep to make sure it's done
return nil
},
}
return cmd
return cmdCapiSync
}

202
cmd/crowdsec-cli/parsers.go Normal file
View file

@ -0,0 +1,202 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewParsersCmd() *cobra.Command {
var cmdParsers = &cobra.Command{
Use: "parsers [action] [config]",
Short: "Install/Remove/Upgrade/Inspect parser(s) from hub",
Example: `cscli parsers install crowdsecurity/sshd-logs
cscli parsers inspect crowdsecurity/sshd-logs
cscli parsers upgrade crowdsecurity/sshd-logs
cscli parsers list
cscli parsers remove crowdsecurity/sshd-logs
`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"parser"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if csConfig.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
cmdParsers.AddCommand(NewParsersInstallCmd())
cmdParsers.AddCommand(NewParsersRemoveCmd())
cmdParsers.AddCommand(NewParsersUpgradeCmd())
cmdParsers.AddCommand(NewParsersInspectCmd())
cmdParsers.AddCommand(NewParsersListCmd())
return cmdParsers
}
func NewParsersInstallCmd() *cobra.Command {
var ignoreError bool
var cmdParsersInstall = &cobra.Command{
Use: "install [config]",
Short: "Install given parser(s)",
Long: `Fetch and install given parser(s) from hub`,
Example: `cscli parsers install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.PARSERS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
for _, name := range args {
t := cwhub.GetItem(cwhub.PARSERS, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.PARSERS, name)
Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
if ignoreError {
log.Errorf("Error while installing '%s': %s", name, err)
} else {
log.Fatalf("Error while installing '%s': %s", name, err)
}
}
}
},
}
cmdParsersInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdParsersInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdParsersInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple parsers")
return cmdParsersInstall
}
func NewParsersRemoveCmd() *cobra.Command {
var cmdParsersRemove = &cobra.Command{
Use: "remove [config]",
Short: "Remove given parser(s)",
Long: `Remove given parse(s) from hub`,
Aliases: []string{"delete"},
Example: `cscli parsers remove crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
if all {
cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, forceAction)
return
}
if len(args) == 0 {
log.Fatalf("Specify at least one parser to remove or '--all' flag.")
}
for _, name := range args {
cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, forceAction)
}
},
}
cmdParsersRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdParsersRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdParsersRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the parsers")
return cmdParsersRemove
}
func NewParsersUpgradeCmd() *cobra.Command {
var cmdParsersUpgrade = &cobra.Command{
Use: "upgrade [config]",
Short: "Upgrade given parser(s)",
Long: `Fetch and upgrade given parser(s) from hub`,
Example: `cscli parsers upgrade crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
} else {
if len(args) == 0 {
log.Fatalf("no target parser to upgrade")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, forceAction)
}
}
},
}
cmdParsersUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the parsers")
cmdParsersUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
return cmdParsersUpgrade
}
func NewParsersInspectCmd() *cobra.Command {
var cmdParsersInspect = &cobra.Command{
Use: "inspect [name]",
Short: "Inspect given parser",
Long: `Inspect given parser`,
Example: `cscli parsers inspect crowdsec/xxx`,
DisableAutoGenTag: true,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
InspectItem(args[0], cwhub.PARSERS)
},
}
cmdParsersInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
return cmdParsersInspect
}
func NewParsersListCmd() *cobra.Command {
var cmdParsersList = &cobra.Command{
Use: "list [name]",
Short: "List all parsers or given one",
Long: `List all parsers or given one`,
Example: `cscli parsers list
cscli parser list crowdsecurity/xxx`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all)
},
}
cmdParsersList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmdParsersList
}

View file

@ -0,0 +1,200 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewPostOverflowsInstallCmd() *cobra.Command {
var ignoreError bool
cmdPostOverflowsInstall := &cobra.Command{
Use: "install [config]",
Short: "Install given postoverflow(s)",
Long: `Fetch and install given postoverflow(s) from hub`,
Example: `cscli postoverflows install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
for _, name := range args {
t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.PARSERS_OVFLW, name)
Suggest(cwhub.PARSERS_OVFLW, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil {
if ignoreError {
log.Errorf("Error while installing '%s': %s", name, err)
} else {
log.Fatalf("Error while installing '%s': %s", name, err)
}
}
}
},
}
cmdPostOverflowsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdPostOverflowsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdPostOverflowsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple postoverflows")
return cmdPostOverflowsInstall
}
func NewPostOverflowsRemoveCmd() *cobra.Command {
cmdPostOverflowsRemove := &cobra.Command{
Use: "remove [config]",
Short: "Remove given postoverflow(s)",
Long: `remove given postoverflow(s)`,
Example: `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
Aliases: []string{"delete"},
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
if all {
cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, forceAction)
return
}
if len(args) == 0 {
log.Fatalf("Specify at least one postoverflow to remove or '--all' flag.")
}
for _, name := range args {
cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, forceAction)
}
},
}
cmdPostOverflowsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdPostOverflowsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdPostOverflowsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the postoverflows")
return cmdPostOverflowsRemove
}
func NewPostOverflowsUpgradeCmd() *cobra.Command {
cmdPostOverflowsUpgrade := &cobra.Command{
Use: "upgrade [config]",
Short: "Upgrade given postoverflow(s)",
Long: `Fetch and Upgrade given postoverflow(s) from hub`,
Example: `cscli postoverflows upgrade crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
} else {
if len(args) == 0 {
log.Fatalf("no target postoverflow to upgrade")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, forceAction)
}
}
},
}
cmdPostOverflowsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the postoverflows")
cmdPostOverflowsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
return cmdPostOverflowsUpgrade
}
func NewPostOverflowsInspectCmd() *cobra.Command {
cmdPostOverflowsInspect := &cobra.Command{
Use: "inspect [config]",
Short: "Inspect given postoverflow",
Long: `Inspect given postoverflow`,
Example: `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
InspectItem(args[0], cwhub.PARSERS_OVFLW)
},
}
return cmdPostOverflowsInspect
}
func NewPostOverflowsListCmd() *cobra.Command {
cmdPostOverflowsList := &cobra.Command{
Use: "list [config]",
Short: "List all postoverflows or given one",
Long: `List all postoverflows or given one`,
Example: `cscli postoverflows list
cscli postoverflows list crowdsecurity/xxx`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all)
},
}
cmdPostOverflowsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmdPostOverflowsList
}
func NewPostOverflowsCmd() *cobra.Command {
cmdPostOverflows := &cobra.Command{
Use: "postoverflows [action] [config]",
Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub",
Example: `cscli postoverflows install crowdsecurity/cdn-whitelist
cscli postoverflows inspect crowdsecurity/cdn-whitelist
cscli postoverflows upgrade crowdsecurity/cdn-whitelist
cscli postoverflows list
cscli postoverflows remove crowdsecurity/cdn-whitelist`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"postoverflow"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if csConfig.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsListCmd())
return cmdPostOverflows
}

View file

@ -1,62 +0,0 @@
package require
// Set the appropriate hub branch according to config settings and crowdsec version
import (
log "github.com/sirupsen/logrus"
"golang.org/x/mod/semver"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
func chooseBranch(cfg *csconfig.Config) string {
// this was set from config.yaml or flag
if cfg.Cscli.HubBranch != "" {
log.Debugf("Hub override from config: branch '%s'", cfg.Cscli.HubBranch)
return cfg.Cscli.HubBranch
}
latest, err := cwversion.Latest()
if err != nil {
log.Warningf("Unable to retrieve latest crowdsec version: %s, using hub branch 'master'", err)
return "master"
}
csVersion := cwversion.VersionStrip()
if csVersion == latest {
log.Debugf("Latest crowdsec version (%s), using hub branch 'master'", csVersion)
return "master"
}
// if current version is greater than the latest we are in pre-release
if semver.Compare(csVersion, latest) == 1 {
log.Debugf("Your current crowdsec version seems to be a pre-release (%s), using hub branch 'master'", csVersion)
return "master"
}
if csVersion == "" {
log.Warning("Crowdsec version is not set, using hub branch 'master'")
return "master"
}
log.Warnf("A new CrowdSec release is available (%s). "+
"Your version is '%s'. Please update it to use new parsers/scenarios/collections.",
latest, csVersion)
return csVersion
}
// HubBranch sets the branch (in cscli config) and returns its value
// It can be "master", or the branch corresponding to the current crowdsec version, or the value overridden in config/flag
func HubBranch(cfg *csconfig.Config) string {
branch := chooseBranch(cfg)
cfg.Cscli.HubBranch = branch
return branch
}
func HubURLTemplate(cfg *csconfig.Config) string {
return cfg.Cscli.HubURLTemplate
}

View file

@ -1,100 +0,0 @@
package require
import (
"errors"
"fmt"
"io"
"github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func LAPI(c *csconfig.Config) error {
if err := c.LoadAPIServer(true); err != nil {
return fmt.Errorf("failed to load Local API: %w", err)
}
if c.DisableAPI {
return errors.New("local API is disabled -- this command must be run on the local API machine")
}
return nil
}
func CAPI(c *csconfig.Config) error {
if c.API.Server.OnlineClient == nil {
return fmt.Errorf("no configuration for Central API (CAPI) in '%s'", *c.FilePath)
}
return nil
}
func PAPI(c *csconfig.Config) error {
if c.API.Server.OnlineClient.Credentials.PapiURL == "" {
return errors.New("no PAPI URL in configuration")
}
return nil
}
func CAPIRegistered(c *csconfig.Config) error {
if c.API.Server.OnlineClient.Credentials == nil {
return errors.New("the Central API (CAPI) must be configured with 'cscli capi register'")
}
return nil
}
func DB(c *csconfig.Config) error {
if err := c.LoadDBConfig(true); err != nil {
return fmt.Errorf("this command requires direct database access (must be run on the local API machine): %w", err)
}
return nil
}
func Notifications(c *csconfig.Config) error {
if c.ConfigPaths.NotificationDir == "" {
return errors.New("config_paths.notification_dir is not set in crowdsec config")
}
return nil
}
// RemoteHub returns the configuration required to download hub index and items: url, branch, etc.
func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg {
// set branch in config, and log if necessary
branch := HubBranch(c)
urlTemplate := HubURLTemplate(c)
remote := &cwhub.RemoteHubCfg{
Branch: branch,
URLTemplate: urlTemplate,
IndexPath: ".index.json",
}
return remote
}
// Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items.
// If no remote parameter is provided, the hub can only be used for local operations.
func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg, logger *logrus.Logger) (*cwhub.Hub, error) {
local := c.Hub
if local == nil {
return nil, errors.New("you must configure cli before interacting with hub")
}
if logger == nil {
logger = logrus.New()
logger.SetOutput(io.Discard)
}
hub, err := cwhub.NewHub(local, remote, false, logger)
if err != nil {
return nil, fmt.Errorf("failed to read Hub index: %w. Run 'sudo cscli hub update' to download the index again", err)
}
return hub, nil
}

View file

@ -0,0 +1,198 @@
package main
import (
"fmt"
"github.com/fatih/color"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewScenariosCmd() *cobra.Command {
var cmdScenarios = &cobra.Command{
Use: "scenarios [action] [config]",
Short: "Install/Remove/Upgrade/Inspect scenario(s) from hub",
Example: `cscli scenarios list [-a]
cscli scenarios install crowdsecurity/ssh-bf
cscli scenarios inspect crowdsecurity/ssh-bf
cscli scenarios upgrade crowdsecurity/ssh-bf
cscli scenarios remove crowdsecurity/ssh-bf
`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"scenario"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if csConfig.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return errors.Wrap(err, "while setting hub branch")
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
cmdScenarios.AddCommand(NewCmdScenariosInstall())
cmdScenarios.AddCommand(NewCmdScenariosRemove())
cmdScenarios.AddCommand(NewCmdScenariosUpgrade())
cmdScenarios.AddCommand(NewCmdScenariosInspect())
cmdScenarios.AddCommand(NewCmdScenariosList())
return cmdScenarios
}
func NewCmdScenariosInstall() *cobra.Command {
var ignoreError bool
var cmdScenariosInstall = &cobra.Command{
Use: "install [config]",
Short: "Install given scenario(s)",
Long: `Fetch and install given scenario(s) from hub`,
Example: `cscli scenarios install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
for _, name := range args {
t := cwhub.GetItem(cwhub.SCENARIOS, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.SCENARIOS, name)
Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
if ignoreError {
log.Errorf("Error while installing '%s': %s", name, err)
} else {
log.Fatalf("Error while installing '%s': %s", name, err)
}
}
}
},
}
cmdScenariosInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdScenariosInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdScenariosInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple scenarios")
return cmdScenariosInstall
}
func NewCmdScenariosRemove() *cobra.Command {
var cmdScenariosRemove = &cobra.Command{
Use: "remove [config]",
Short: "Remove given scenario(s)",
Long: `remove given scenario(s)`,
Example: `cscli scenarios remove crowdsec/xxx crowdsec/xyz`,
Aliases: []string{"delete"},
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if all {
cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, forceAction)
return
}
if len(args) == 0 {
log.Fatalf("Specify at least one scenario to remove or '--all' flag.")
}
for _, name := range args {
cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, forceAction)
}
},
}
cmdScenariosRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdScenariosRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdScenariosRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the scenarios")
return cmdScenariosRemove
}
func NewCmdScenariosUpgrade() *cobra.Command {
var cmdScenariosUpgrade = &cobra.Command{
Use: "upgrade [config]",
Short: "Upgrade given scenario(s)",
Long: `Fetch and Upgrade given scenario(s) from hub`,
Example: `cscli scenarios upgrade crowdsec/xxx crowdsec/xyz`,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
} else {
if len(args) == 0 {
log.Fatalf("no target scenario to upgrade")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, forceAction)
}
}
},
}
cmdScenariosUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the scenarios")
cmdScenariosUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
return cmdScenariosUpgrade
}
func NewCmdScenariosInspect() *cobra.Command {
var cmdScenariosInspect = &cobra.Command{
Use: "inspect [config]",
Short: "Inspect given scenario",
Long: `Inspect given scenario`,
Example: `cscli scenarios inspect crowdsec/xxx`,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
InspectItem(args[0], cwhub.SCENARIOS)
},
}
cmdScenariosInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
return cmdScenariosInspect
}
func NewCmdScenariosList() *cobra.Command {
var cmdScenariosList = &cobra.Command{
Use: "list [config]",
Short: "List all scenario(s) or given one",
Long: `List all scenario(s) or given one`,
Example: `cscli scenarios list
cscli scenarios list crowdsecurity/xxx`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all)
},
}
cmdScenariosList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmdScenariosList
}

View file

@ -2,17 +2,15 @@ package main
import (
"bytes"
"errors"
"fmt"
"os"
"os/exec"
goccyyaml "github.com/goccy/go-yaml"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
goccyyaml "github.com/goccy/go-yaml"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/setup"
)
@ -114,22 +112,6 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
return err
}
var detectReader *os.File
switch detectConfigFile {
case "-":
log.Tracef("Reading detection rules from stdin")
detectReader = os.Stdin
default:
log.Tracef("Reading detection rules: %s", detectConfigFile)
detectReader, err = os.Open(detectConfigFile)
if err != nil {
return err
}
}
listSupportedServices, err := flags.GetBool("list-supported-services")
if err != nil {
return err
@ -174,7 +156,6 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
_, err := exec.LookPath("systemctl")
if err != nil {
log.Debug("systemctl not available: snubbing systemd")
snubSystemd = true
}
}
@ -186,12 +167,11 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
if forcedOSFamily == "" && forcedOSID != "" {
log.Debug("force-os-id is set: force-os-family defaults to 'linux'")
forcedOSFamily = "linux"
}
if listSupportedServices {
supported, err := setup.ListSupported(detectReader)
supported, err := setup.ListSupported(detectConfigFile)
if err != nil {
return err
}
@ -215,7 +195,7 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
SnubSystemd: snubSystemd,
}
hubSetup, err := setup.Detect(detectReader, opts)
hubSetup, err := setup.Detect(detectConfigFile, opts)
if err != nil {
return fmt.Errorf("detecting services: %w", err)
}
@ -224,7 +204,6 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
fmt.Println(setup)
return nil
@ -310,12 +289,7 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error {
return fmt.Errorf("while reading file %s: %w", fromFile, err)
}
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig), log.StandardLogger())
if err != nil {
return err
}
if err = setup.InstallHubItems(hub, input, dryRun); err != nil {
if err = setup.InstallHubItems(csConfig, input, dryRun); err != nil {
return err
}
@ -324,7 +298,6 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error {
func runSetupValidate(cmd *cobra.Command, args []string) error {
fromFile := args[0]
input, err := os.ReadFile(fromFile)
if err != nil {
return fmt.Errorf("while reading stdin: %w", err)
@ -332,7 +305,7 @@ func runSetupValidate(cmd *cobra.Command, args []string) error {
if err = setup.Validate(input); err != nil {
fmt.Printf("%v\n", err)
return errors.New("invalid setup file")
return fmt.Errorf("invalid setup file")
}
return nil

View file

@ -1,209 +1,39 @@
package main
import (
"errors"
"fmt"
"os"
"slices"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
type cliSimulation struct {
cfg configGetter
func addToExclusion(name string) error {
csConfig.Cscli.SimulationConfig.Exclusions = append(csConfig.Cscli.SimulationConfig.Exclusions, name)
return nil
}
func NewCLISimulation(cfg configGetter) *cliSimulation {
return &cliSimulation{
cfg: cfg,
}
}
func (cli *cliSimulation) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "simulation [command]",
Short: "Manage simulation status of scenarios",
Example: `cscli simulation status
cscli simulation enable crowdsecurity/ssh-bf
cscli simulation disable crowdsecurity/ssh-bf`,
DisableAutoGenTag: true,
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
if err := cli.cfg().LoadSimulation(); err != nil {
return err
}
if cli.cfg().Cscli.SimulationConfig == nil {
return errors.New("no simulation configured")
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, _ []string) {
if cmd.Name() != "status" {
log.Infof(ReloadMessage())
}
},
}
cmd.Flags().SortFlags = false
cmd.PersistentFlags().SortFlags = false
cmd.AddCommand(cli.NewEnableCmd())
cmd.AddCommand(cli.NewDisableCmd())
cmd.AddCommand(cli.NewStatusCmd())
return cmd
}
func (cli *cliSimulation) NewEnableCmd() *cobra.Command {
var forceGlobalSimulation bool
cmd := &cobra.Command{
Use: "enable [scenario] [-global]",
Short: "Enable the simulation, globally or on specified scenarios",
Example: `cscli simulation enable`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
hub, err := require.Hub(cli.cfg(), nil, nil)
if err != nil {
return err
}
if len(args) > 0 {
for _, scenario := range args {
item := hub.GetItem(cwhub.SCENARIOS, scenario)
if item == nil {
log.Errorf("'%s' doesn't exist or is not a scenario", scenario)
continue
}
if !item.State.Installed {
log.Warningf("'%s' isn't enabled", scenario)
}
isExcluded := slices.Contains(cli.cfg().Cscli.SimulationConfig.Exclusions, scenario)
if *cli.cfg().Cscli.SimulationConfig.Simulation && !isExcluded {
log.Warning("global simulation is already enabled")
continue
}
if !*cli.cfg().Cscli.SimulationConfig.Simulation && isExcluded {
log.Warningf("simulation for '%s' already enabled", scenario)
continue
}
if *cli.cfg().Cscli.SimulationConfig.Simulation && isExcluded {
cli.removeFromExclusion(scenario)
log.Printf("simulation enabled for '%s'", scenario)
continue
}
cli.addToExclusion(scenario)
log.Printf("simulation mode for '%s' enabled", scenario)
}
if err := cli.dumpSimulationFile(); err != nil {
return fmt.Errorf("simulation enable: %w", err)
}
} else if forceGlobalSimulation {
if err := cli.enableGlobalSimulation(); err != nil {
return fmt.Errorf("unable to enable global simulation mode: %w", err)
}
} else {
printHelp(cmd)
}
return nil
},
}
cmd.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Enable global simulation (reverse mode)")
return cmd
}
func (cli *cliSimulation) NewDisableCmd() *cobra.Command {
var forceGlobalSimulation bool
cmd := &cobra.Command{
Use: "disable [scenario]",
Short: "Disable the simulation mode. Disable only specified scenarios",
Example: `cscli simulation disable`,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if len(args) > 0 {
for _, scenario := range args {
isExcluded := slices.Contains(cli.cfg().Cscli.SimulationConfig.Exclusions, scenario)
if !*cli.cfg().Cscli.SimulationConfig.Simulation && !isExcluded {
log.Warningf("%s isn't in simulation mode", scenario)
continue
}
if !*cli.cfg().Cscli.SimulationConfig.Simulation && isExcluded {
cli.removeFromExclusion(scenario)
log.Printf("simulation mode for '%s' disabled", scenario)
continue
}
if isExcluded {
log.Warningf("simulation mode is enabled but is already disable for '%s'", scenario)
continue
}
cli.addToExclusion(scenario)
log.Printf("simulation mode for '%s' disabled", scenario)
}
if err := cli.dumpSimulationFile(); err != nil {
return fmt.Errorf("simulation disable: %w", err)
}
} else if forceGlobalSimulation {
if err := cli.disableGlobalSimulation(); err != nil {
return fmt.Errorf("unable to disable global simulation mode: %w", err)
}
} else {
printHelp(cmd)
}
return nil
},
}
cmd.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Disable global simulation (reverse mode)")
return cmd
}
func (cli *cliSimulation) NewStatusCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "status",
Short: "Show simulation mode status",
Example: `cscli simulation status`,
DisableAutoGenTag: true,
Run: func(_ *cobra.Command, _ []string) {
cli.status()
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
},
}
return cmd
}
func (cli *cliSimulation) addToExclusion(name string) {
cfg := cli.cfg()
cfg.Cscli.SimulationConfig.Exclusions = append(cfg.Cscli.SimulationConfig.Exclusions, name)
}
func (cli *cliSimulation) removeFromExclusion(name string) {
cfg := cli.cfg()
index := slices.Index(cfg.Cscli.SimulationConfig.Exclusions, name)
func removeFromExclusion(name string) error {
index := indexOf(name, csConfig.Cscli.SimulationConfig.Exclusions)
// Remove element from the slice
cfg.Cscli.SimulationConfig.Exclusions[index] = cfg.Cscli.SimulationConfig.Exclusions[len(cfg.Cscli.SimulationConfig.Exclusions)-1]
cfg.Cscli.SimulationConfig.Exclusions[len(cfg.Cscli.SimulationConfig.Exclusions)-1] = ""
cfg.Cscli.SimulationConfig.Exclusions = cfg.Cscli.SimulationConfig.Exclusions[:len(cfg.Cscli.SimulationConfig.Exclusions)-1]
csConfig.Cscli.SimulationConfig.Exclusions[index] = csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1] = ""
csConfig.Cscli.SimulationConfig.Exclusions = csConfig.Cscli.SimulationConfig.Exclusions[:len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
return nil
}
func (cli *cliSimulation) enableGlobalSimulation() error {
cfg := cli.cfg()
cfg.Cscli.SimulationConfig.Simulation = new(bool)
*cfg.Cscli.SimulationConfig.Simulation = true
cfg.Cscli.SimulationConfig.Exclusions = []string{}
func enableGlobalSimulation() error {
csConfig.Cscli.SimulationConfig.Simulation = new(bool)
*csConfig.Cscli.SimulationConfig.Simulation = true
csConfig.Cscli.SimulationConfig.Exclusions = []string{}
if err := cli.dumpSimulationFile(); err != nil {
return fmt.Errorf("unable to dump simulation file: %w", err)
if err := dumpSimulationFile(); err != nil {
log.Fatalf("unable to dump simulation file: %s", err)
}
log.Printf("global simulation: enabled")
@ -211,72 +41,227 @@ func (cli *cliSimulation) enableGlobalSimulation() error {
return nil
}
func (cli *cliSimulation) dumpSimulationFile() error {
cfg := cli.cfg()
newConfigSim, err := yaml.Marshal(cfg.Cscli.SimulationConfig)
func dumpSimulationFile() error {
newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
if err != nil {
return fmt.Errorf("unable to marshal simulation configuration: %w", err)
return fmt.Errorf("unable to marshal simulation configuration: %s", err)
}
err = os.WriteFile(cfg.ConfigPaths.SimulationFilePath, newConfigSim, 0o644)
err = os.WriteFile(csConfig.ConfigPaths.SimulationFilePath, newConfigSim, 0644)
if err != nil {
return fmt.Errorf("write simulation config in '%s' failed: %w", cfg.ConfigPaths.SimulationFilePath, err)
return fmt.Errorf("write simulation config in '%s' failed: %s", csConfig.ConfigPaths.SimulationFilePath, err)
}
log.Debugf("updated simulation file %s", cfg.ConfigPaths.SimulationFilePath)
log.Debugf("updated simulation file %s", csConfig.ConfigPaths.SimulationFilePath)
return nil
}
func (cli *cliSimulation) disableGlobalSimulation() error {
cfg := cli.cfg()
cfg.Cscli.SimulationConfig.Simulation = new(bool)
*cfg.Cscli.SimulationConfig.Simulation = false
func disableGlobalSimulation() error {
csConfig.Cscli.SimulationConfig.Simulation = new(bool)
*csConfig.Cscli.SimulationConfig.Simulation = false
cfg.Cscli.SimulationConfig.Exclusions = []string{}
newConfigSim, err := yaml.Marshal(cfg.Cscli.SimulationConfig)
csConfig.Cscli.SimulationConfig.Exclusions = []string{}
newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
if err != nil {
return fmt.Errorf("unable to marshal new simulation configuration: %w", err)
return fmt.Errorf("unable to marshal new simulation configuration: %s", err)
}
err = os.WriteFile(cfg.ConfigPaths.SimulationFilePath, newConfigSim, 0o644)
err = os.WriteFile(csConfig.ConfigPaths.SimulationFilePath, newConfigSim, 0644)
if err != nil {
return fmt.Errorf("unable to write new simulation config in '%s': %w", cfg.ConfigPaths.SimulationFilePath, err)
return fmt.Errorf("unable to write new simulation config in '%s' : %s", csConfig.ConfigPaths.SimulationFilePath, err)
}
log.Printf("global simulation: disabled")
return nil
}
func (cli *cliSimulation) status() {
cfg := cli.cfg()
if cfg.Cscli.SimulationConfig == nil {
func simulationStatus() error {
if csConfig.Cscli.SimulationConfig == nil {
log.Printf("global simulation: disabled (configuration file is missing)")
return
return nil
}
if *cfg.Cscli.SimulationConfig.Simulation {
if *csConfig.Cscli.SimulationConfig.Simulation {
log.Println("global simulation: enabled")
if len(cfg.Cscli.SimulationConfig.Exclusions) > 0 {
if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
log.Println("Scenarios not in simulation mode :")
for _, scenario := range cfg.Cscli.SimulationConfig.Exclusions {
for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
log.Printf(" - %s", scenario)
}
}
} else {
log.Println("global simulation: disabled")
if len(cfg.Cscli.SimulationConfig.Exclusions) > 0 {
if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
log.Println("Scenarios in simulation mode :")
for _, scenario := range cfg.Cscli.SimulationConfig.Exclusions {
for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
log.Printf(" - %s", scenario)
}
}
}
return nil
}
func NewSimulationCmds() *cobra.Command {
var cmdSimulation = &cobra.Command{
Use: "simulation [command]",
Short: "Manage simulation status of scenarios",
Example: `cscli simulation status
cscli simulation enable crowdsecurity/ssh-bf
cscli simulation disable crowdsecurity/ssh-bf`,
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadSimulation(); err != nil {
log.Fatal(err)
}
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before using simulation")
}
if csConfig.Cscli.SimulationConfig == nil {
return fmt.Errorf("no simulation configured")
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() != "status" {
log.Infof(ReloadMessage())
}
},
}
cmdSimulation.Flags().SortFlags = false
cmdSimulation.PersistentFlags().SortFlags = false
cmdSimulation.AddCommand(NewSimulationEnableCmd())
cmdSimulation.AddCommand(NewSimulationDisableCmd())
cmdSimulation.AddCommand(NewSimulationStatusCmd())
return cmdSimulation
}
func NewSimulationEnableCmd() *cobra.Command {
var forceGlobalSimulation bool
var cmdSimulationEnable = &cobra.Command{
Use: "enable [scenario] [-global]",
Short: "Enable the simulation, globally or on specified scenarios",
Example: `cscli simulation enable`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
}
if len(args) > 0 {
for _, scenario := range args {
var item = cwhub.GetItem(cwhub.SCENARIOS, scenario)
if item == nil {
log.Errorf("'%s' doesn't exist or is not a scenario", scenario)
continue
}
if !item.Installed {
log.Warningf("'%s' isn't enabled", scenario)
}
isExcluded := inSlice(scenario, csConfig.Cscli.SimulationConfig.Exclusions)
if *csConfig.Cscli.SimulationConfig.Simulation && !isExcluded {
log.Warning("global simulation is already enabled")
continue
}
if !*csConfig.Cscli.SimulationConfig.Simulation && isExcluded {
log.Warningf("simulation for '%s' already enabled", scenario)
continue
}
if *csConfig.Cscli.SimulationConfig.Simulation && isExcluded {
if err := removeFromExclusion(scenario); err != nil {
log.Fatal(err)
}
log.Printf("simulation enabled for '%s'", scenario)
continue
}
if err := addToExclusion(scenario); err != nil {
log.Fatal(err)
}
log.Printf("simulation mode for '%s' enabled", scenario)
}
if err := dumpSimulationFile(); err != nil {
log.Fatalf("simulation enable: %s", err)
}
} else if forceGlobalSimulation {
if err := enableGlobalSimulation(); err != nil {
log.Fatalf("unable to enable global simulation mode : %s", err)
}
} else {
printHelp(cmd)
}
},
}
cmdSimulationEnable.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Enable global simulation (reverse mode)")
return cmdSimulationEnable
}
func NewSimulationDisableCmd() *cobra.Command {
var forceGlobalSimulation bool
var cmdSimulationDisable = &cobra.Command{
Use: "disable [scenario]",
Short: "Disable the simulation mode. Disable only specified scenarios",
Example: `cscli simulation disable`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if len(args) > 0 {
for _, scenario := range args {
isExcluded := inSlice(scenario, csConfig.Cscli.SimulationConfig.Exclusions)
if !*csConfig.Cscli.SimulationConfig.Simulation && !isExcluded {
log.Warningf("%s isn't in simulation mode", scenario)
continue
}
if !*csConfig.Cscli.SimulationConfig.Simulation && isExcluded {
if err := removeFromExclusion(scenario); err != nil {
log.Fatal(err)
}
log.Printf("simulation mode for '%s' disabled", scenario)
continue
}
if isExcluded {
log.Warningf("simulation mode is enabled but is already disable for '%s'", scenario)
continue
}
if err := addToExclusion(scenario); err != nil {
log.Fatal(err)
}
log.Printf("simulation mode for '%s' disabled", scenario)
}
if err := dumpSimulationFile(); err != nil {
log.Fatalf("simulation disable: %s", err)
}
} else if forceGlobalSimulation {
if err := disableGlobalSimulation(); err != nil {
log.Fatalf("unable to disable global simulation mode : %s", err)
}
} else {
printHelp(cmd)
}
},
}
cmdSimulationDisable.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Disable global simulation (reverse mode)")
return cmdSimulationDisable
}
func NewSimulationStatusCmd() *cobra.Command {
var cmdSimulationStatus = &cobra.Command{
Use: "status",
Short: "Show simulation mode status",
Example: `cscli simulation status`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if err := simulationStatus(); err != nil {
log.Fatal(err)
}
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
},
}
return cmdSimulationStatus
}

View file

@ -4,7 +4,6 @@ import (
"archive/zip"
"bytes"
"context"
"errors"
"fmt"
"io"
"net/http"
@ -13,23 +12,19 @@ import (
"path/filepath"
"regexp"
"strings"
"time"
"github.com/blackfireio/osinfo"
"github.com/go-openapi/strfmt"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
const (
@ -40,7 +35,6 @@ const (
SUPPORT_OS_INFO_PATH = "osinfo.txt"
SUPPORT_PARSERS_PATH = "hub/parsers.txt"
SUPPORT_SCENARIOS_PATH = "hub/scenarios.txt"
SUPPORT_CONTEXTS_PATH = "hub/scenarios.txt"
SUPPORT_COLLECTIONS_PATH = "hub/collections.txt"
SUPPORT_POSTOVERFLOWS_PATH = "hub/postoverflows.txt"
SUPPORT_BOUNCERS_PATH = "lapi/bouncers.txt"
@ -50,54 +44,43 @@ const (
SUPPORT_CAPI_STATUS_PATH = "capi_status.txt"
SUPPORT_ACQUISITION_CONFIG_BASE_PATH = "config/acquis/"
SUPPORT_CROWDSEC_PROFILE_PATH = "config/profiles.yaml"
SUPPORT_CRASH_PATH = "crash/"
)
// from https://github.com/acarl005/stripansi
var reStripAnsi = regexp.MustCompile("[\u001B\u009B][[\\]()#;?]*(?:(?:(?:[a-zA-Z\\d]*(?:;[a-zA-Z\\d]*)*)?\u0007)|(?:(?:\\d{1,4}(?:;\\d{0,4})*)?[\\dA-PRZcf-ntqry=><~]))")
func stripAnsiString(str string) string {
// the byte version doesn't strip correctly
return reStripAnsi.ReplaceAllString(str, "")
}
func collectMetrics() ([]byte, []byte, error) {
log.Info("Collecting prometheus metrics")
if csConfig.Cscli.PrometheusUrl == "" {
log.Warn("No Prometheus URL configured, metrics will not be collected")
return nil, nil, errors.New("prometheus_uri is not set")
}
humanMetrics := bytes.NewBuffer(nil)
ms := NewMetricStore()
if err := ms.Fetch(csConfig.Cscli.PrometheusUrl); err != nil {
return nil, nil, fmt.Errorf("could not fetch prometheus metrics: %w", err)
}
if err := ms.Format(humanMetrics, nil, "human", false); err != nil {
err := csConfig.LoadPrometheus()
if err != nil {
return nil, nil, err
}
req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl, nil)
if err != nil {
return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %w", err)
if csConfig.Cscli.PrometheusUrl == "" {
log.Warn("No Prometheus URL configured, metrics will not be collected")
return nil, nil, fmt.Errorf("prometheus_uri is not set")
}
client := &http.Client{}
humanMetrics := bytes.NewBuffer(nil)
err = FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl+"/metrics", "human")
resp, err := client.Do(req)
if err != nil {
return nil, nil, fmt.Errorf("could not get metrics from prometheus endpoint: %w", err)
return nil, nil, fmt.Errorf("could not fetch promtheus metrics: %s", err)
}
req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl+"/metrics", nil)
if err != nil {
return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err)
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, nil, fmt.Errorf("could not get metrics from prometheus endpoint: %s", err)
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, nil, fmt.Errorf("could not read metrics from prometheus endpoint: %w", err)
return nil, nil, fmt.Errorf("could not read metrics from prometheus endpoint: %s", err)
}
return humanMetrics.Bytes(), body, nil
@ -110,108 +93,100 @@ func collectVersion() []byte {
func collectFeatures() []byte {
log.Info("Collecting feature flags")
enabledFeatures := fflag.Crowdsec.GetEnabledFeatures()
w := bytes.NewBuffer(nil)
for _, k := range enabledFeatures {
fmt.Fprintf(w, "%s\n", k)
}
return w.Bytes()
}
func collectOSInfo() ([]byte, error) {
log.Info("Collecting OS info")
info, err := osinfo.GetOSInfo()
if err != nil {
return nil, err
}
w := bytes.NewBuffer(nil)
fmt.Fprintf(w, "Architecture: %s\n", info.Architecture)
fmt.Fprintf(w, "Family: %s\n", info.Family)
fmt.Fprintf(w, "ID: %s\n", info.ID)
fmt.Fprintf(w, "Name: %s\n", info.Name)
fmt.Fprintf(w, "Codename: %s\n", info.Codename)
fmt.Fprintf(w, "Version: %s\n", info.Version)
fmt.Fprintf(w, "Build: %s\n", info.Build)
w.WriteString(fmt.Sprintf("Architecture: %s\n", info.Architecture))
w.WriteString(fmt.Sprintf("Family: %s\n", info.Family))
w.WriteString(fmt.Sprintf("ID: %s\n", info.ID))
w.WriteString(fmt.Sprintf("Name: %s\n", info.Name))
w.WriteString(fmt.Sprintf("Codename: %s\n", info.Codename))
w.WriteString(fmt.Sprintf("Version: %s\n", info.Version))
w.WriteString(fmt.Sprintf("Build: %s\n", info.Build))
return w.Bytes(), nil
}
func collectHubItems(hub *cwhub.Hub, itemType string) []byte {
var err error
func initHub() error {
if err := csConfig.LoadHub(); err != nil {
return fmt.Errorf("cannot load hub: %s", err)
}
if csConfig.Hub == nil {
return fmt.Errorf("hub not configured")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("cannot set hub branch: %s", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
return fmt.Errorf("no hub index found: %s", err)
}
return nil
}
func collectHubItems(itemType string) []byte {
out := bytes.NewBuffer(nil)
log.Infof("Collecting %s list", itemType)
items := make(map[string][]*cwhub.Item)
if items[itemType], err = selectItems(hub, itemType, nil, true); err != nil {
log.Warnf("could not collect %s list: %s", itemType, err)
}
if err := listItems(out, []string{itemType}, items, false); err != nil {
log.Warnf("could not collect %s list: %s", itemType, err)
}
ListItems(out, []string{itemType}, []string{}, false, true, all)
return out.Bytes()
}
func collectBouncers(dbClient *database.Client) ([]byte, error) {
out := bytes.NewBuffer(nil)
bouncers, err := dbClient.ListBouncers()
err := getBouncers(out, dbClient)
if err != nil {
return nil, fmt.Errorf("unable to list bouncers: %w", err)
return nil, err
}
getBouncersTable(out, bouncers)
return out.Bytes(), nil
}
func collectAgents(dbClient *database.Client) ([]byte, error) {
out := bytes.NewBuffer(nil)
machines, err := dbClient.ListMachines()
err := getAgents(out, dbClient)
if err != nil {
return nil, fmt.Errorf("unable to list machines: %w", err)
return nil, err
}
getAgentsTable(out, machines)
return out.Bytes(), nil
}
func collectAPIStatus(login string, password string, endpoint string, prefix string, hub *cwhub.Hub) []byte {
func collectAPIStatus(login string, password string, endpoint string, prefix string) []byte {
if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil {
return []byte("No agent credentials found, are we LAPI ?")
}
pwd := strfmt.Password(password)
apiurl, err := url.Parse(endpoint)
if err != nil {
return []byte(fmt.Sprintf("cannot parse API URL: %s", err))
}
scenarios, err := hub.GetInstalledNamesByType(cwhub.SCENARIOS)
scenarios, err := cwhub.GetInstalledScenariosAsString()
if err != nil {
return []byte(fmt.Sprintf("could not collect scenarios: %s", err))
}
Client, err = apiclient.NewDefaultClient(apiurl,
prefix,
fmt.Sprintf("crowdsec/%s", version.String()),
fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
nil)
if err != nil {
return []byte(fmt.Sprintf("could not init client: %s", err))
}
t := models.WatcherAuthRequest{
MachineID: &login,
Password: &pwd,
@ -228,7 +203,6 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
func collectCrowdsecConfig() []byte {
log.Info("Collecting crowdsec config")
config, err := os.ReadFile(*csConfig.FilePath)
if err != nil {
return []byte(fmt.Sprintf("could not read config file: %s", err))
@ -241,18 +215,15 @@ func collectCrowdsecConfig() []byte {
func collectCrowdsecProfile() []byte {
log.Info("Collecting crowdsec profile")
config, err := os.ReadFile(csConfig.API.Server.ProfilesPath)
if err != nil {
return []byte(fmt.Sprintf("could not read profile file: %s", err))
}
return config
}
func collectAcquisitionConfig() map[string][]byte {
log.Info("Collecting acquisition config")
ret := make(map[string][]byte)
for _, filename := range csConfig.Crowdsec.AcquisitionFiles {
@ -267,19 +238,8 @@ func collectAcquisitionConfig() map[string][]byte {
return ret
}
func collectCrash() ([]string, error) {
log.Info("Collecting crash dumps")
return trace.List()
}
type cliSupport struct{}
func NewCLISupport() *cliSupport {
return &cliSupport{}
}
func (cli cliSupport) NewCommand() *cobra.Command {
cmd := &cobra.Command{
func NewSupportCmd() *cobra.Command {
var cmdSupport = &cobra.Command{
Use: "support [action]",
Short: "Provide commands to help during support",
Args: cobra.MinimumNArgs(1),
@ -289,15 +249,9 @@ func (cli cliSupport) NewCommand() *cobra.Command {
},
}
cmd.AddCommand(cli.NewDumpCmd())
return cmd
}
func (cli cliSupport) NewDumpCmd() *cobra.Command {
var outFile string
cmd := &cobra.Command{
cmdDump := &cobra.Command{
Use: "dump",
Short: "Dump all your configuration to a zip file for easier support",
Long: `Dump the following informations:
@ -307,7 +261,6 @@ func (cli cliSupport) NewDumpCmd() *cobra.Command {
- Installed parsers list
- Installed scenarios list
- Installed postoverflows list
- Installed context list
- Bouncers list
- Machines list
- CAPI status
@ -319,7 +272,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
`,
Args: cobra.NoArgs,
DisableAutoGenTag: true,
RunE: func(_ *cobra.Command, _ []string) error {
Run: func(cmd *cobra.Command, args []string) {
var err error
var skipHub, skipDB, skipCAPI, skipLAPI, skipAgent bool
infos := map[string][]byte{
@ -339,25 +292,24 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_AGENTS_PATH] = []byte(err.Error())
}
if err = csConfig.LoadAPIServer(true); err != nil {
if err := csConfig.LoadAPIServer(); err != nil {
log.Warnf("could not load LAPI, skipping CAPI check")
skipLAPI = true
infos[SUPPORT_CAPI_STATUS_PATH] = []byte(err.Error())
}
if err = csConfig.LoadCrowdsec(); err != nil {
if err := csConfig.LoadCrowdsec(); err != nil {
log.Warnf("could not load agent config, skipping crowdsec config check")
skipAgent = true
}
hub, err := require.Hub(csConfig, nil, nil)
err = initHub()
if err != nil {
log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
skipHub = true
infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())
infos[SUPPORT_SCENARIOS_PATH] = []byte(err.Error())
infos[SUPPORT_POSTOVERFLOWS_PATH] = []byte(err.Error())
infos[SUPPORT_CONTEXTS_PATH] = []byte(err.Error())
infos[SUPPORT_COLLECTIONS_PATH] = []byte(err.Error())
}
@ -390,11 +342,10 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_CROWDSEC_CONFIG_PATH] = collectCrowdsecConfig()
if !skipHub {
infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS)
infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS)
infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
infos[SUPPORT_CONTEXTS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS)
infos[SUPPORT_PARSERS_PATH] = collectHubItems(cwhub.PARSERS)
infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(cwhub.SCENARIOS)
infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.PARSERS_OVFLW)
infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(cwhub.COLLECTIONS)
}
if !skipDB {
@ -416,8 +367,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_CAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Server.OnlineClient.Credentials.Login,
csConfig.API.Server.OnlineClient.Credentials.Password,
csConfig.API.Server.OnlineClient.Credentials.URL,
CAPIURLPrefix,
hub)
CAPIURLPrefix)
}
if !skipLAPI {
@ -425,12 +375,12 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_LAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Client.Credentials.Login,
csConfig.API.Client.Credentials.Password,
csConfig.API.Client.Credentials.URL,
LAPIURLPrefix,
hub)
LAPIURLPrefix)
infos[SUPPORT_CROWDSEC_PROFILE_PATH] = collectCrowdsecProfile()
}
if !skipAgent {
acquis := collectAcquisitionConfig()
for filename, content := range acquis {
@ -439,57 +389,33 @@ cscli support dump -f /tmp/crowdsec-support.zip
}
}
crash, err := collectCrash()
if err != nil {
log.Errorf("could not collect crash dumps: %s", err)
}
for _, filename := range crash {
content, err := os.ReadFile(filename)
if err != nil {
log.Errorf("could not read crash dump %s: %s", filename, err)
}
infos[SUPPORT_CRASH_PATH+filepath.Base(filename)] = content
}
w := bytes.NewBuffer(nil)
zipWriter := zip.NewWriter(w)
for filename, data := range infos {
header := &zip.FileHeader{
Name: filename,
Method: zip.Deflate,
// TODO: retain mtime where possible (esp. trace)
Modified: time.Now(),
}
fw, err := zipWriter.CreateHeader(header)
fw, err := zipWriter.Create(filename)
if err != nil {
log.Errorf("Could not add zip entry for %s: %s", filename, err)
continue
}
fw.Write([]byte(stripAnsiString(string(data))))
fw.Write([]byte(types.StripAnsiString(string(data))))
}
err = zipWriter.Close()
if err != nil {
return fmt.Errorf("could not finalize zip file: %s", err)
log.Fatalf("could not finalize zip file: %s", err)
}
if outFile == "-" {
_, err = os.Stdout.Write(w.Bytes())
return err
}
err = os.WriteFile(outFile, w.Bytes(), 0o600)
err = os.WriteFile(outFile, w.Bytes(), 0600)
if err != nil {
return fmt.Errorf("could not write zip file to %s: %s", outFile, err)
log.Fatalf("could not write zip file to %s: %s", outFile, err)
}
log.Infof("Written zip file to %s", outFile)
return nil
},
}
cmdDump.Flags().StringVarP(&outFile, "outFile", "f", "", "File to dump the information to")
cmdSupport.AddCommand(cmdDump)
cmd.Flags().StringVarP(&outFile, "outFile", "f", "", "File to dump the information to")
return cmd
return cmdSupport
}

View file

@ -1,23 +1,283 @@
package main
import (
"encoding/csv"
"encoding/json"
"fmt"
"io"
"math"
"net"
"net/http"
"os"
"strconv"
"strings"
"time"
"github.com/fatih/color"
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/prom2json"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/texttheater/golang-levenshtein/levenshtein"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
const MaxDistance = 7
func printHelp(cmd *cobra.Command) {
if err := cmd.Help(); err != nil {
err := cmd.Help()
if err != nil {
log.Fatalf("unable to print help(): %s", err)
}
}
func inSlice(s string, slice []string) bool {
for _, str := range slice {
if s == str {
return true
}
}
return false
}
func indexOf(s string, slice []string) int {
for i, elem := range slice {
if s == elem {
return i
}
}
return -1
}
func LoadHub() error {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if csConfig.Hub == nil {
return fmt.Errorf("unable to load hub")
}
if err := cwhub.SetHubBranch(); err != nil {
log.Warningf("unable to set hub branch (%s), default to master", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
return fmt.Errorf("Failed to get Hub index : '%w'. Run 'sudo cscli hub update' to get the hub index", err)
}
return nil
}
func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
errMsg := ""
if score < MaxDistance {
errMsg = fmt.Sprintf("unable to find %s '%s', did you mean %s ?", itemType, baseItem, suggestItem)
} else {
errMsg = fmt.Sprintf("unable to find %s '%s'", itemType, baseItem)
}
if ignoreErr {
log.Error(errMsg)
} else {
log.Fatalf(errMsg)
}
}
func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
allItems := make([]string, 0)
nearestScore := 100
nearestItem := &cwhub.Item{}
hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
for _, item := range hubItems {
allItems = append(allItems, item.Name)
}
for _, s := range allItems {
d := levenshtein.DistanceForStrings([]rune(itemName), []rune(s), levenshtein.DefaultOptions)
if d < nearestScore {
nearestScore = d
nearestItem = cwhub.GetItem(itemType, s)
}
}
return nearestItem, nearestScore
}
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := LoadHub(); err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
for _, item := range hubItems {
if !inSlice(item.Name, args) && strings.Contains(item.Name, toComplete) {
comp = append(comp, item.Name)
}
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := LoadHub(); err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
var items []string
var err error
switch itemType {
case cwhub.PARSERS:
items, err = cwhub.GetInstalledParsersAsString()
case cwhub.SCENARIOS:
items, err = cwhub.GetInstalledScenariosAsString()
case cwhub.PARSERS_OVFLW:
items, err = cwhub.GetInstalledPostOverflowsAsString()
case cwhub.COLLECTIONS:
items, err = cwhub.GetInstalledCollectionsAsString()
default:
return nil, cobra.ShellCompDirectiveDefault
}
if err != nil {
cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
if toComplete != "" {
for _, item := range items {
if strings.Contains(item, toComplete) {
comp = append(comp, item)
}
}
} else {
comp = items
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}
func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) {
var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus)
for _, itemType := range itemTypes {
itemName := ""
if len(args) == 1 {
itemName = args[0]
}
hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all)
}
if csConfig.Cscli.Output == "human" {
for _, itemType := range itemTypes {
var statuses []cwhub.ItemHubStatus
var ok bool
if statuses, ok = hubStatusByItemType[itemType]; !ok {
log.Errorf("unknown item type: %s", itemType)
continue
}
listHubItemTable(out, "\n"+strings.ToUpper(itemType), statuses)
}
} else if csConfig.Cscli.Output == "json" {
x, err := json.MarshalIndent(hubStatusByItemType, "", " ")
if err != nil {
log.Fatalf("failed to unmarshal")
}
out.Write(x)
} else if csConfig.Cscli.Output == "raw" {
csvwriter := csv.NewWriter(out)
if showHeader {
header := []string{"name", "status", "version", "description"}
if showType {
header = append(header, "type")
}
err := csvwriter.Write(header)
if err != nil {
log.Fatalf("failed to write header: %s", err)
}
}
for _, itemType := range itemTypes {
var statuses []cwhub.ItemHubStatus
var ok bool
if statuses, ok = hubStatusByItemType[itemType]; !ok {
log.Errorf("unknown item type: %s", itemType)
continue
}
for _, status := range statuses {
if status.LocalVersion == "" {
status.LocalVersion = "n/a"
}
row := []string{
status.Name,
status.Status,
status.LocalVersion,
status.Description,
}
if showType {
row = append(row, itemType)
}
err := csvwriter.Write(row)
if err != nil {
log.Fatalf("failed to write raw output : %s", err)
}
}
}
csvwriter.Flush()
}
}
func InspectItem(name string, objecitemType string) {
hubItem := cwhub.GetItem(objecitemType, name)
if hubItem == nil {
log.Fatalf("unable to retrieve item.")
}
var b []byte
var err error
switch csConfig.Cscli.Output {
case "human", "raw":
b, err = yaml.Marshal(*hubItem)
if err != nil {
log.Fatalf("unable to marshal item : %s", err)
}
case "json":
b, err = json.MarshalIndent(*hubItem, "", " ")
if err != nil {
log.Fatalf("unable to marshal item : %s", err)
}
}
fmt.Printf("%s", string(b))
if csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
return
}
if prometheusURL == "" {
//This is technically wrong to do this, as the prometheus section contains a listen address, not an URL to query prometheus
//But for ease of use, we will use the listen address as the prometheus URL because it will be 127.0.0.1 in the default case
listenAddr := csConfig.Prometheus.ListenAddr
if listenAddr == "" {
listenAddr = "127.0.0.1"
}
listenPort := csConfig.Prometheus.ListenPort
if listenPort == 0 {
listenPort = 6060
}
prometheusURL = fmt.Sprintf("http://%s:%d/metrics", listenAddr, listenPort)
log.Debugf("No prometheus URL provided using: %s", prometheusURL)
}
fmt.Printf("\nCurrent metrics : \n")
ShowMetrics(hubItem)
}
func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
/*if a range is provided, change the scope*/
if *ipRange != "" {
_, _, err := net.ParseCIDR(*ipRange)
@ -25,7 +285,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
return fmt.Errorf("%s isn't a valid range", *ipRange)
}
}
if *ip != "" {
ipRepr := net.ParseIP(*ip)
if ipRepr == nil {
@ -33,7 +292,7 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
}
}
// avoid confusion on scope (ip vs Ip and range vs Range)
//avoid confusion on scope (ip vs Ip and range vs Range)
switch strings.ToLower(*scope) {
case "ip":
*scope = types.Ip
@ -44,10 +303,450 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
case "as":
*scope = types.AS
}
return nil
}
func ShowMetrics(hubItem *cwhub.Item) {
switch hubItem.Type {
case cwhub.PARSERS:
metrics := GetParserMetric(prometheusURL, hubItem.Name)
parserMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.SCENARIOS:
metrics := GetScenarioMetric(prometheusURL, hubItem.Name)
scenarioMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.COLLECTIONS:
for _, item := range hubItem.Parsers {
metrics := GetParserMetric(prometheusURL, item)
parserMetricsTable(color.Output, item, metrics)
}
for _, item := range hubItem.Scenarios {
metrics := GetScenarioMetric(prometheusURL, item)
scenarioMetricsTable(color.Output, item, metrics)
}
for _, item := range hubItem.Collections {
hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
if hubItem == nil {
log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
}
ShowMetrics(hubItem)
}
default:
log.Errorf("item of type '%s' is unknown", hubItem.Type)
}
}
// GetParserMetric is a complete rip from prom2json
func GetParserMetric(url string, itemName string) map[string]map[string]int {
stats := make(map[string]map[string]int)
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
source, ok := metric.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v", metric.Labels)
} else {
if srctype, ok := metric.Labels["type"]; ok {
source = srctype + ":" + source
}
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_reader_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
stats[source]["parsed"] = 0
stats[source]["reads"] = 0
stats[source]["unparsed"] = 0
stats[source]["hits"] = 0
}
stats[source]["reads"] += ival
case "cs_parser_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_parser_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
case "cs_node_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["hits"] += ival
case "cs_node_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_node_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
default:
continue
}
}
}
return stats
}
func GetScenarioMetric(url string, itemName string) map[string]int {
stats := make(map[string]int)
stats["instantiation"] = 0
stats["curr_count"] = 0
stats["overflow"] = 0
stats["pour"] = 0
stats["underflow"] = 0
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_bucket_created_total":
stats["instantiation"] += ival
case "cs_buckets":
stats["curr_count"] += ival
case "cs_bucket_overflowed_total":
stats["overflow"] += ival
case "cs_bucket_poured_total":
stats["pour"] += ival
case "cs_bucket_underflowed_total":
stats["underflow"] += ival
default:
continue
}
}
}
return stats
}
// it's a rip of the cli version, but in silent-mode
func silenceInstallItem(name string, obtype string) (string, error) {
var item = cwhub.GetItem(obtype, name)
if item == nil {
return "", fmt.Errorf("error retrieving item")
}
it := *item
if downloadOnly && it.Downloaded && it.UpToDate {
return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
}
it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction, false)
if err != nil {
return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
}
if err := cwhub.AddItem(obtype, it); err != nil {
return "", err
}
if downloadOnly {
return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
}
it, err = cwhub.EnableItem(csConfig.Hub, it)
if err != nil {
return "", fmt.Errorf("error while enabling %s : %v", it.Name, err)
}
if err := cwhub.AddItem(obtype, it); err != nil {
return "", err
}
return fmt.Sprintf("Enabled %s", it.Name), nil
}
func GetPrometheusMetric(url string) []*prom2json.Family {
mfChan := make(chan *dto.MetricFamily, 1024)
// Start with the DefaultTransport for sane defaults.
transport := http.DefaultTransport.(*http.Transport).Clone()
// Conservatively disable HTTP keep-alives as this program will only
// ever need a single HTTP request.
transport.DisableKeepAlives = true
// Timeout early if the server doesn't even return the headers.
transport.ResponseHeaderTimeout = time.Minute
go func() {
defer types.CatchPanic("crowdsec/GetPrometheusMetric")
err := prom2json.FetchMetricFamilies(url, mfChan, transport)
if err != nil {
log.Fatalf("failed to fetch prometheus metrics : %v", err)
}
}()
result := []*prom2json.Family{}
for mf := range mfChan {
result = append(result, prom2json.NewFamily(mf))
}
log.Debugf("Finished reading prometheus output, %d entries", len(result))
return result
}
func RestoreHub(dirPath string) error {
var err error
if err := csConfig.LoadHub(); err != nil {
return err
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
for _, itype := range cwhub.ItemTypes {
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
if _, err = os.Stat(itemDirectory); err != nil {
log.Infof("no %s in backup", itype)
continue
}
/*restore the upstream items*/
upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
file, err := os.ReadFile(upstreamListFN)
if err != nil {
return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
}
var upstreamList []string
err = json.Unmarshal(file, &upstreamList)
if err != nil {
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
}
for _, toinstall := range upstreamList {
label, err := silenceInstallItem(toinstall, itype)
if err != nil {
log.Errorf("Error while installing %s : %s", toinstall, err)
} else if label != "" {
log.Infof("Installed %s : %s", toinstall, label)
} else {
log.Printf("Installed %s : ok", toinstall)
}
}
/*restore the local and tainted items*/
files, err := os.ReadDir(itemDirectory)
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
}
for _, file := range files {
//this was the upstream data
if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
continue
}
if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
//we expect a stage here
if !file.IsDir() {
continue
}
stage := file.Name()
stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
}
/*find items*/
ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
}
//finally copy item
for _, tfile := range ifiles {
log.Infof("Going to restore local/tainted [%s]", tfile.Name())
sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
if err = types.CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
} else {
log.Infof("Going to restore local/tainted [%s]", file.Name())
sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
if err = types.CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
}
}
return nil
}
func BackupHub(dirPath string) error {
var err error
var itemDirectory string
var upstreamParsers []string
for _, itemType := range cwhub.ItemTypes {
clog := log.WithFields(log.Fields{
"type": itemType,
})
itemMap := cwhub.GetItemMap(itemType)
if itemMap == nil {
clog.Infof("No %s to backup.", itemType)
continue
}
itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
}
upstreamParsers = []string{}
for k, v := range itemMap {
clog = clog.WithFields(log.Fields{
"file": v.Name,
})
if !v.Installed { //only backup installed ones
clog.Debugf("[%s] : not installed", k)
continue
}
//for the local/tainted ones, we backup the full file
if v.Tainted || v.Local || !v.UpToDate {
//we need to backup stages for parsers
if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
}
}
clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
if err = types.CopyFile(v.LocalPath, tfile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
}
clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
continue
}
clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
upstreamParsers = append(upstreamParsers, v.Name)
}
//write the upstream items
upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
if err != nil {
return fmt.Errorf("failed marshaling upstream parsers : %s", err)
}
err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
if err != nil {
return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
}
clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
}
return nil
}
type unit struct {
value int64
symbol string
}
var ranges = []unit{
{
value: 1e18,
symbol: "E",
},
{
value: 1e15,
symbol: "P",
},
{
value: 1e12,
symbol: "T",
},
{
value: 1e6,
symbol: "M",
},
{
value: 1e3,
symbol: "k",
},
{
value: 1,
symbol: "",
},
}
func formatNumber(num int) string {
goodUnit := unit{}
for _, u := range ranges {
if int64(num) >= u.value {
goodUnit = u
break
}
}
if goodUnit.value == 1 {
return fmt.Sprintf("%d%s", num, goodUnit.symbol)
}
res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
}
func getDBClient() (*database.Client, error) {
var err error
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
return nil, err
}
ret, err := database.NewClient(csConfig.DbConfig)
if err != nil {
return nil, err
}
return ret, nil
}
func removeFromSlice(val string, slice []string) []string {
var i int
var value string
@ -69,4 +768,5 @@ func removeFromSlice(val string, slice []string) []string {
}
return slice
}

View file

@ -3,56 +3,39 @@ package main
import (
"fmt"
"io"
"strconv"
"github.com/aquasecurity/table"
"github.com/enescakir/emoji"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/emoji"
)
func listHubItemTable(out io.Writer, title string, items []*cwhub.Item) {
func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatus) {
t := newLightTable(out)
t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
for _, item := range items {
status := fmt.Sprintf("%v %s", item.State.Emoji(), item.State.Text())
t.AddRow(item.Name, status, item.State.LocalVersion, item.State.LocalPath)
for _, status := range statuses {
t.AddRow(status.Name, status.UTF8_Status, status.LocalVersion, status.LocalPath)
}
renderTableTitle(out, title)
t.Render()
}
func appsecMetricsTable(out io.Writer, itemName string, metrics map[string]int) {
t := newTable(out)
t.SetHeaders("Inband Hits", "Outband Hits")
t.AddRow(
strconv.Itoa(metrics["inband_hits"]),
strconv.Itoa(metrics["outband_hits"]),
)
renderTableTitle(out, fmt.Sprintf("\n - (AppSec Rule) %s:", itemName))
t.Render()
}
func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int) {
if metrics["instantiation"] == 0 {
return
}
t := newTable(out)
t.SetHeaders("Current Count", "Overflows", "Instantiated", "Poured", "Expired")
t.AddRow(
strconv.Itoa(metrics["curr_count"]),
strconv.Itoa(metrics["overflow"]),
strconv.Itoa(metrics["instantiation"]),
strconv.Itoa(metrics["pour"]),
strconv.Itoa(metrics["underflow"]),
fmt.Sprintf("%d", metrics["curr_count"]),
fmt.Sprintf("%d", metrics["overflow"]),
fmt.Sprintf("%d", metrics["instantiation"]),
fmt.Sprintf("%d", metrics["pour"]),
fmt.Sprintf("%d", metrics["underflow"]),
)
renderTableTitle(out, fmt.Sprintf("\n - (Scenario) %s:", itemName))
@ -60,26 +43,23 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
}
func parserMetricsTable(out io.Writer, itemName string, metrics map[string]map[string]int) {
skip := true
t := newTable(out)
t.SetHeaders("Parsers", "Hits", "Parsed", "Unparsed")
// don't show table if no hits
showTable := false
for source, stats := range metrics {
if stats["hits"] > 0 {
t.AddRow(
source,
strconv.Itoa(stats["hits"]),
strconv.Itoa(stats["parsed"]),
strconv.Itoa(stats["unparsed"]),
fmt.Sprintf("%d", stats["hits"]),
fmt.Sprintf("%d", stats["parsed"]),
fmt.Sprintf("%d", stats["unparsed"]),
)
showTable = true
skip = false
}
}
if showTable {
if !skip {
renderTableTitle(out, fmt.Sprintf("\n - (Parser) %s:", itemName))
t.Render()
}

View file

@ -1,27 +0,0 @@
package main
import (
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
)
type cliVersion struct{}
func NewCLIVersion() *cliVersion {
return &cliVersion{}
}
func (cli cliVersion) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "version",
Short: "Display version",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
Run: func(_ *cobra.Command, _ []string) {
cwversion.Show()
},
}
return cmd
}

View file

@ -4,9 +4,12 @@ ifeq ($(OS), Windows_NT)
EXT = .exe
endif
GO = go
GOBUILD = $(GO) build
GOTEST = $(GO) test
# Go parameters
GOCMD = go
GOBUILD = $(GOCMD) build
GOCLEAN = $(GOCMD) clean
GOTEST = $(GOCMD) test
GOGET = $(GOCMD) get
CROWDSEC_BIN = crowdsec$(EXT)
# names longer than 15 chars break 'pgrep'
@ -22,13 +25,13 @@ SYSTEMD_PATH_FILE = "/etc/systemd/system/crowdsec.service"
all: clean test build
build: clean
$(GOBUILD) $(LD_OPTS) -o $(CROWDSEC_BIN)
$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(CROWDSEC_BIN)
test:
$(GOTEST) $(LD_OPTS) -v ./...
clean:
@$(RM) $(CROWDSEC_BIN) $(WIN_IGNORE_ERR)
@$(RM) $(CROWDSEC_BIN) $(CROWDSEC_BIN).test $(WIN_IGNORE_ERR)
.PHONY: install
install: install-conf install-bin
@ -56,7 +59,7 @@ install-conf:
install-bin:
install -v -m 755 -D "$(CROWDSEC_BIN)" "$(BIN_PREFIX)/$(CROWDSEC_BIN)" || exit
.PHONY: systemd
.PHONY: systemd"$(BIN_PREFI"$(BIN_PREFIX)/$(CROWDSEC_BIN)""$(BIN_PREFIX)/$(CROWDSEC_BIN)"X)/$(CROWDSEC_BIN)"
systemd: install
CFG=$(CFG_PREFIX) PID=$(PID_DIR) BIN=$(BIN_PREFIX)"/"$(CROWDSEC_BIN) envsubst < ../../config/crowdsec.service > "$(SYSTEMD_PATH_FILE)"
systemctl daemon-reload

View file

@ -1,17 +1,15 @@
package main
import (
"errors"
"fmt"
"runtime"
"time"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/pkg/apiserver"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) {
@ -21,7 +19,7 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) {
apiServer, err := apiserver.NewServer(cConfig.API.Server)
if err != nil {
return nil, fmt.Errorf("unable to run local API: %w", err)
return nil, errors.Wrap(err, "unable to run local API")
}
if hasPlugins(cConfig.API.Server.Profiles) {
@ -30,38 +28,33 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) {
if cConfig.PluginConfig == nil && runtime.GOOS != "windows" {
return nil, errors.New("plugins are enabled, but the plugin_config section is missing in the configuration")
}
if cConfig.ConfigPaths.NotificationDir == "" {
return nil, errors.New("plugins are enabled, but config_paths.notification_dir is not defined")
}
if cConfig.ConfigPaths.PluginDir == "" {
return nil, errors.New("plugins are enabled, but config_paths.plugin_dir is not defined")
}
err = pluginBroker.Init(cConfig.PluginConfig, cConfig.API.Server.Profiles, cConfig.ConfigPaths)
if err != nil {
return nil, fmt.Errorf("unable to run plugin broker: %w", err)
return nil, errors.Wrap(err, "unable to run local API")
}
log.Info("initiated plugin broker")
apiServer.AttachPluginBroker(&pluginBroker)
}
err = apiServer.InitController()
if err != nil {
return nil, fmt.Errorf("unable to run local API: %w", err)
return nil, errors.Wrap(err, "unable to run local API")
}
return apiServer, nil
}
func serveAPIServer(apiServer *apiserver.APIServer) {
apiReady := make(chan bool, 1)
func serveAPIServer(apiServer *apiserver.APIServer, apiReady chan bool) {
apiTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/serveAPIServer")
defer types.CatchPanic("crowdsec/serveAPIServer")
go func() {
defer trace.CatchPanic("crowdsec/runAPIServer")
defer types.CatchPanic("crowdsec/runAPIServer")
log.Debugf("serving API after %s ms", time.Since(crowdsecT0))
if err := apiServer.Run(apiReady); err != nil {
log.Fatal(err)
@ -81,7 +74,6 @@ func serveAPIServer(apiServer *apiserver.APIServer) {
}
return nil
})
<-apiReady
}
func hasPlugins(profiles []*csconfig.ProfileCfg) bool {

View file

@ -1,183 +1,144 @@
package main
import (
"context"
"fmt"
"os"
"path/filepath"
"sync"
"time"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/go-cs-lib/trace"
"path/filepath"
"github.com/crowdsecurity/crowdsec/pkg/acquisition"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
"github.com/crowdsecurity/crowdsec/pkg/appsec"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
)
// initCrowdsec prepares the log processor service
func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, []acquisition.DataSource, error) {
func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
var err error
if err = alertcontext.LoadConsoleContext(cConfig, hub); err != nil {
return nil, nil, fmt.Errorf("while loading context: %w", err)
// Populate cwhub package tools
if err := cwhub.GetHubIdx(cConfig.Hub); err != nil {
return &parser.Parsers{}, fmt.Errorf("Failed to load hub index : %s", err)
}
// Start loading configs
csParsers := parser.NewParsers(hub)
csParsers := parser.NewParsers()
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
return nil, nil, fmt.Errorf("while loading parsers: %w", err)
return &parser.Parsers{}, fmt.Errorf("Failed to load parsers: %s", err)
}
if err := LoadBuckets(cConfig, hub); err != nil {
return nil, nil, fmt.Errorf("while loading scenarios: %w", err)
if err := LoadBuckets(cConfig); err != nil {
return &parser.Parsers{}, fmt.Errorf("Failed to load scenarios: %s", err)
}
if err := appsec.LoadAppsecRules(hub); err != nil {
return nil, nil, fmt.Errorf("while loading appsec rules: %w", err)
if err := LoadAcquisition(cConfig); err != nil {
return &parser.Parsers{}, fmt.Errorf("Error while loading acquisition config : %s", err)
}
datasources, err := LoadAcquisition(cConfig)
if err != nil {
return nil, nil, fmt.Errorf("while loading acquisition config: %w", err)
}
return csParsers, datasources, nil
return csParsers, nil
}
// runCrowdsec starts the log processor service
func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.Hub, datasources []acquisition.DataSource) error {
func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
inputEventChan = make(chan types.Event)
inputLineChan = make(chan types.Event)
// start go-routines for parsing, buckets pour and outputs.
//start go-routines for parsing, buckets pour and outputs.
parserWg := &sync.WaitGroup{}
parsersTomb.Go(func() error {
parserWg.Add(1)
for i := 0; i < cConfig.Crowdsec.ParserRoutinesCount; i++ {
parsersTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/runParse")
if err := runParse(inputLineChan, inputEventChan, *parsers.Ctx, parsers.Nodes); err != nil {
// this error will never happen as parser.Parse is not able to return errors
defer types.CatchPanic("crowdsec/runParse")
if err := runParse(inputLineChan, inputEventChan, *parsers.Ctx, parsers.Nodes); err != nil { //this error will never happen as parser.Parse is not able to return errors
log.Fatalf("starting parse error : %s", err)
return err
}
return nil
})
}
parserWg.Done()
return nil
})
parserWg.Wait()
bucketWg := &sync.WaitGroup{}
bucketsTomb.Go(func() error {
bucketWg.Add(1)
/*restore previous state as well if present*/
if cConfig.Crowdsec.BucketStateFile != "" {
log.Warningf("Restoring buckets state from %s", cConfig.Crowdsec.BucketStateFile)
if err := leaky.LoadBucketsState(cConfig.Crowdsec.BucketStateFile, buckets, holders); err != nil {
return fmt.Errorf("unable to restore buckets: %w", err)
return fmt.Errorf("unable to restore buckets : %s", err)
}
}
for i := 0; i < cConfig.Crowdsec.BucketsRoutinesCount; i++ {
bucketsTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/runPour")
defer types.CatchPanic("crowdsec/runPour")
if err := runPour(inputEventChan, holders, buckets, cConfig); err != nil {
log.Fatalf("starting pour error : %s", err)
return err
}
return nil
})
}
bucketWg.Done()
return nil
})
bucketWg.Wait()
apiClient, err := AuthenticatedLAPIClient(*cConfig.API.Client.Credentials, hub)
if err != nil {
return err
}
log.Debugf("Starting HeartBeat service")
apiClient.HeartBeat.StartHeartBeat(context.Background(), &outputsTomb)
outputWg := &sync.WaitGroup{}
outputsTomb.Go(func() error {
outputWg.Add(1)
for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ {
outputsTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/runOutput")
if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, apiClient); err != nil {
defer types.CatchPanic("crowdsec/runOutput")
if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials); err != nil {
log.Fatalf("starting outputs error : %s", err)
return err
}
return nil
})
}
outputWg.Done()
return nil
})
outputWg.Wait()
if cConfig.Prometheus != nil && cConfig.Prometheus.Enabled {
aggregated := false
if cConfig.Prometheus.Level == configuration.CFG_METRICS_AGGREGATE {
if cConfig.Prometheus.Level == "aggregated" {
aggregated = true
}
if err := acquisition.GetMetrics(dataSources, aggregated); err != nil {
return fmt.Errorf("while fetching prometheus metrics for datasources: %w", err)
return errors.Wrap(err, "while fetching prometheus metrics for datasources.")
}
}
}
log.Info("Starting processing data")
if err := acquisition.StartAcquisition(dataSources, inputLineChan, &acquisTomb); err != nil {
return fmt.Errorf("starting acquisition error: %w", err)
log.Fatalf("starting acquisition error : %s", err)
return err
}
return nil
}
// serveCrowdsec wraps the log processor service
func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub.Hub, datasources []acquisition.DataSource, agentReady chan bool) {
func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady chan bool) {
crowdsecTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/serveCrowdsec")
defer types.CatchPanic("crowdsec/serveCrowdsec")
go func() {
defer trace.CatchPanic("crowdsec/runCrowdsec")
defer types.CatchPanic("crowdsec/runCrowdsec")
// this logs every time, even at config reload
log.Debugf("running agent after %s ms", time.Since(crowdsecT0))
agentReady <- true
if err := runCrowdsec(cConfig, parsers, hub, datasources); err != nil {
if err := runCrowdsec(cConfig, parsers); err != nil {
log.Fatalf("unable to start crowdsec routines: %s", err)
}
}()
@ -188,88 +149,74 @@ func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub
*/
waitOnTomb()
log.Debugf("Shutting down crowdsec routines")
if err := ShutdownCrowdsecRoutines(); err != nil {
log.Fatalf("unable to shutdown crowdsec routines: %s", err)
}
log.Debugf("everything is dead, return crowdsecTomb")
if dumpStates {
dumpParserState()
dumpOverflowState()
dumpBucketsPour()
os.Exit(0)
}
return nil
})
}
func dumpBucketsPour() {
fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "bucketpour-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o666)
fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "bucketpour-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
if err != nil {
log.Fatalf("open: %s", err)
}
out, err := yaml.Marshal(leaky.BucketPourCache)
if err != nil {
log.Fatalf("marshal: %s", err)
}
b, err := fd.Write(out)
if err != nil {
log.Fatalf("write: %s", err)
}
log.Tracef("wrote %d bytes", b)
if err := fd.Close(); err != nil {
log.Fatalf(" close: %s", err)
}
}
func dumpParserState() {
fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "parser-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o666)
fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "parser-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
if err != nil {
log.Fatalf("open: %s", err)
}
out, err := yaml.Marshal(parser.StageParseCache)
if err != nil {
log.Fatalf("marshal: %s", err)
}
b, err := fd.Write(out)
if err != nil {
log.Fatalf("write: %s", err)
}
log.Tracef("wrote %d bytes", b)
if err := fd.Close(); err != nil {
log.Fatalf(" close: %s", err)
}
}
func dumpOverflowState() {
fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "bucket-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o666)
fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "bucket-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
if err != nil {
log.Fatalf("open: %s", err)
}
out, err := yaml.Marshal(bucketOverflows)
if err != nil {
log.Fatalf("marshal: %s", err)
}
b, err := fd.Write(out)
if err != nil {
log.Fatalf("write: %s", err)
}
log.Tracef("wrote %d bytes", b)
if err := fd.Close(); err != nil {
log.Fatalf(" close: %s", err)
}
@ -281,7 +228,7 @@ func waitOnTomb() {
case <-acquisTomb.Dead():
/*if it's acquisition dying it means that we were in "cat" mode.
while shutting down, we need to give time for all buckets to process in flight data*/
log.Info("Acquisition is finished, shutting down")
log.Warning("Acquisition is finished, shutting down")
/*
While it might make sense to want to shut-down parser/buckets/etc. as soon as acquisition is finished,
we might have some pending buckets: buckets that overflowed, but whose LeakRoutine are still alive because they

View file

@ -1,28 +0,0 @@
package main
import (
"io"
log "github.com/sirupsen/logrus"
)
// FatalHook is used to log fatal messages to stderr when the rest goes to a file
type FatalHook struct {
Writer io.Writer
LogLevels []log.Level
}
func (hook *FatalHook) Fire(entry *log.Entry) error {
line, err := entry.String()
if err != nil {
return err
}
_, err = hook.Writer.Write([]byte(line))
return err
}
func (hook *FatalHook) Levels() []log.Level {
return hook.LogLevels
}

View file

@ -1,92 +0,0 @@
package main
import (
"context"
"fmt"
"net/url"
"time"
"github.com/go-openapi/strfmt"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/models"
)
func AuthenticatedLAPIClient(credentials csconfig.ApiCredentialsCfg, hub *cwhub.Hub) (*apiclient.ApiClient, error) {
scenarios, err := hub.GetInstalledNamesByType(cwhub.SCENARIOS)
if err != nil {
return nil, fmt.Errorf("loading list of installed hub scenarios: %w", err)
}
appsecRules, err := hub.GetInstalledNamesByType(cwhub.APPSEC_RULES)
if err != nil {
return nil, fmt.Errorf("loading list of installed hub appsec rules: %w", err)
}
installedScenariosAndAppsecRules := make([]string, 0, len(scenarios)+len(appsecRules))
installedScenariosAndAppsecRules = append(installedScenariosAndAppsecRules, scenarios...)
installedScenariosAndAppsecRules = append(installedScenariosAndAppsecRules, appsecRules...)
apiURL, err := url.Parse(credentials.URL)
if err != nil {
return nil, fmt.Errorf("parsing api url ('%s'): %w", credentials.URL, err)
}
papiURL, err := url.Parse(credentials.PapiURL)
if err != nil {
return nil, fmt.Errorf("parsing polling api url ('%s'): %w", credentials.PapiURL, err)
}
password := strfmt.Password(credentials.Password)
client, err := apiclient.NewClient(&apiclient.Config{
MachineID: credentials.Login,
Password: password,
Scenarios: installedScenariosAndAppsecRules,
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
PapiURL: papiURL,
VersionPrefix: "v1",
UpdateScenario: func() ([]string, error) {
scenarios, err := hub.GetInstalledNamesByType(cwhub.SCENARIOS)
if err != nil {
return nil, err
}
appsecRules, err := hub.GetInstalledNamesByType(cwhub.APPSEC_RULES)
if err != nil {
return nil, err
}
ret := make([]string, 0, len(scenarios)+len(appsecRules))
ret = append(ret, scenarios...)
ret = append(ret, appsecRules...)
return ret, nil
},
})
if err != nil {
return nil, fmt.Errorf("new client api: %w", err)
}
authResp, _, err := client.Auth.AuthenticateWatcher(context.Background(), models.WatcherAuthRequest{
MachineID: &credentials.Login,
Password: &password,
Scenarios: installedScenariosAndAppsecRules,
})
if err != nil {
return nil, fmt.Errorf("authenticate watcher (%s): %w", credentials.Login, err)
}
var expiration time.Time
if err := expiration.UnmarshalText([]byte(authResp.Expire)); err != nil {
return nil, fmt.Errorf("unable to parse jwt expiration: %w", err)
}
client.GetClient().Transport.(*apiclient.JWTTransport).Token = authResp.Token
client.GetClient().Transport.(*apiclient.JWTTransport).Expiration = expiration
return client, nil
}

View file

@ -1,22 +1,18 @@
package main
import (
"errors"
"flag"
"fmt"
_ "net/http/pprof"
"os"
"path/filepath"
"runtime"
"runtime/pprof"
"strings"
"time"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/pkg/acquisition"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/csplugin"
@ -55,15 +51,12 @@ var (
)
type Flags struct {
ConfigFile string
LogLevelTrace bool
LogLevelDebug bool
LogLevelInfo bool
LogLevelWarn bool
LogLevelError bool
LogLevelFatal bool
ConfigFile string
TraceLevel bool
DebugLevel bool
InfoLevel bool
WarnLevel bool
ErrorLevel bool
PrintVersion bool
SingleFileType string
Labels map[string]string
@ -74,35 +67,27 @@ type Flags struct {
WinSvc string
DisableCAPI bool
Transform string
OrderEvent bool
CPUProfile string
}
func (f *Flags) haveTimeMachine() bool {
return f.OneShotDSN != ""
}
type labelsMap map[string]string
func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error {
func LoadBuckets(cConfig *csconfig.Config) error {
var (
err error
files []string
)
for _, hubScenarioItem := range hub.GetItemMap(cwhub.SCENARIOS) {
if hubScenarioItem.State.Installed {
files = append(files, hubScenarioItem.State.LocalPath)
for _, hubScenarioItem := range cwhub.GetItemMap(cwhub.SCENARIOS) {
if hubScenarioItem.Installed {
files = append(files, hubScenarioItem.LocalPath)
}
}
buckets = leakybucket.NewBuckets()
log.Infof("Loading %d scenario files", len(files))
holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, files, &bucketsTomb, buckets)
holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, hub, files, &bucketsTomb, buckets, flags.OrderEvent)
if err != nil {
return fmt.Errorf("scenario loading failed: %w", err)
return fmt.Errorf("scenario loading failed: %v", err)
}
if cConfig.Prometheus != nil && cConfig.Prometheus.Enabled {
@ -110,11 +95,10 @@ func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error {
holders[holderIndex].Profiling = true
}
}
return nil
}
func LoadAcquisition(cConfig *csconfig.Config) ([]acquisition.DataSource, error) {
func LoadAcquisition(cConfig *csconfig.Config) error {
var err error
if flags.SingleFileType != "" && flags.OneShotDSN != "" {
@ -123,20 +107,16 @@ func LoadAcquisition(cConfig *csconfig.Config) ([]acquisition.DataSource, error)
dataSources, err = acquisition.LoadAcquisitionFromDSN(flags.OneShotDSN, flags.Labels, flags.Transform)
if err != nil {
return nil, fmt.Errorf("failed to configure datasource for %s: %w", flags.OneShotDSN, err)
return errors.Wrapf(err, "failed to configure datasource for %s", flags.OneShotDSN)
}
} else {
dataSources, err = acquisition.LoadAcquisitionFromFile(cConfig.Crowdsec, cConfig.Prometheus)
dataSources, err = acquisition.LoadAcquisitionFromFile(cConfig.Crowdsec)
if err != nil {
return nil, err
return err
}
}
if len(dataSources) == 0 {
return nil, errors.New("no datasource enabled")
}
return dataSources, nil
return nil
}
var (
@ -150,28 +130,21 @@ func (l *labelsMap) String() string {
}
func (l labelsMap) Set(label string) error {
for _, pair := range strings.Split(label, ",") {
split := strings.Split(pair, ":")
if len(split) != 2 {
return fmt.Errorf("invalid format for label '%s', must be key:value", pair)
}
l[split[0]] = split[1]
split := strings.Split(label, ":")
if len(split) != 2 {
return errors.Wrapf(errors.New("Bad Format"), "for Label '%s'", label)
}
l[split[0]] = split[1]
return nil
}
func (f *Flags) Parse() {
flag.StringVar(&f.ConfigFile, "c", csconfig.DefaultConfigPath("config.yaml"), "configuration file")
flag.BoolVar(&f.LogLevelTrace, "trace", false, "set log level to 'trace' (VERY verbose)")
flag.BoolVar(&f.LogLevelDebug, "debug", false, "set log level to 'debug'")
flag.BoolVar(&f.LogLevelInfo, "info", false, "set log level to 'info'")
flag.BoolVar(&f.LogLevelWarn, "warning", false, "set log level to 'warning'")
flag.BoolVar(&f.LogLevelError, "error", false, "set log level to 'error'")
flag.BoolVar(&f.LogLevelFatal, "fatal", false, "set log level to 'fatal'")
flag.BoolVar(&f.TraceLevel, "trace", false, "VERY verbose")
flag.BoolVar(&f.DebugLevel, "debug", false, "print debug-level on stderr")
flag.BoolVar(&f.InfoLevel, "info", false, "print info-level on stderr")
flag.BoolVar(&f.WarnLevel, "warning", false, "print warning-level on stderr")
flag.BoolVar(&f.ErrorLevel, "error", false, "print error-level on stderr")
flag.BoolVar(&f.PrintVersion, "version", false, "display version")
flag.StringVar(&f.OneShotDSN, "dsn", "", "Process a single data source in time-machine")
flag.StringVar(&f.Transform, "transform", "", "expr to apply on the event after acquisition")
@ -181,14 +154,8 @@ func (f *Flags) Parse() {
flag.BoolVar(&f.DisableAgent, "no-cs", false, "disable crowdsec agent")
flag.BoolVar(&f.DisableAPI, "no-api", false, "disable local API")
flag.BoolVar(&f.DisableCAPI, "no-capi", false, "disable communication with Central API")
flag.BoolVar(&f.OrderEvent, "order-event", false, "enforce event ordering with significant performance cost")
if runtime.GOOS == "windows" {
flag.StringVar(&f.WinSvc, "winsvc", "", "Windows service Action: Install, Remove etc..")
}
flag.StringVar(&f.WinSvc, "winsvc", "", "Windows service Action : Install, Remove etc..")
flag.StringVar(&dumpFolder, "dump-data", "", "dump parsers/buckets raw outputs")
flag.StringVar(&f.CPUProfile, "cpu-profile", "", "write cpu profile to file")
flag.Parse()
}
@ -203,18 +170,16 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
// override from flags
switch {
case f.LogLevelTrace:
case f.TraceLevel:
ret = log.TraceLevel
case f.LogLevelDebug:
case f.DebugLevel:
ret = log.DebugLevel
case f.LogLevelInfo:
case f.InfoLevel:
ret = log.InfoLevel
case f.LogLevelWarn:
case f.WarnLevel:
ret = log.WarnLevel
case f.LogLevelError:
case f.ErrorLevel:
ret = log.ErrorLevel
case f.LogLevelFatal:
ret = log.FatalLevel
default:
}
@ -222,19 +187,18 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
// avoid returning a new ptr to the same value
return curLevelPtr
}
return &ret
}
// LoadConfig returns a configuration parsed from configuration file
func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*csconfig.Config, error) {
cConfig, _, err := csconfig.NewConfig(configFile, disableAgent, disableAPI, quiet)
cConfig, err := csconfig.NewConfig(configFile, disableAgent, disableAPI, quiet)
if err != nil {
return nil, fmt.Errorf("while loading configuration file: %w", err)
return nil, err
}
if err := trace.Init(filepath.Join(cConfig.ConfigPaths.DataDir, "trace")); err != nil {
return nil, fmt.Errorf("while setting up trace directory: %w", err)
if (cConfig.Common == nil || *cConfig.Common == csconfig.CommonCfg{}) {
return nil, fmt.Errorf("unable to load configuration: common section is empty")
}
cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags)
@ -246,6 +210,11 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
dumpStates = true
}
// Configuration paths are dependency to load crowdsec configuration
if err := cConfig.LoadConfigurationPaths(); err != nil {
return nil, err
}
if flags.SingleFileType != "" && flags.OneShotDSN != "" {
// if we're in time-machine mode, we don't want to log to file
cConfig.Common.LogMedia = "stdout"
@ -260,25 +229,18 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
return nil, err
}
if cConfig.Common.LogMedia != "stdout" {
log.AddHook(&FatalHook{
Writer: os.Stderr,
LogLevels: []log.Level{log.FatalLevel, log.PanicLevel},
})
}
if err := csconfig.LoadFeatureFlagsFile(configFile, log.StandardLogger()); err != nil {
return nil, err
}
if !cConfig.DisableAgent {
if !flags.DisableAgent {
if err := cConfig.LoadCrowdsec(); err != nil {
return nil, err
}
}
if !cConfig.DisableAPI {
if err := cConfig.LoadAPIServer(false); err != nil {
if !flags.DisableAPI {
if err := cConfig.LoadAPIServer(); err != nil {
return nil, err
}
}
@ -288,7 +250,11 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
}
if cConfig.DisableAPI && cConfig.DisableAgent {
return nil, errors.New("you must run at least the API Server or crowdsec")
return nil, errors.New("You must run at least the API Server or crowdsec")
}
if flags.TestMode && !cConfig.DisableAgent {
cConfig.Crowdsec.LintOnly = true
}
if flags.OneShotDSN != "" && flags.SingleFileType == "" {
@ -308,10 +274,9 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
cConfig.API.Server.OnlineClient = nil
}
/*if the api is disabled as well, just read file and exit, don't daemonize*/
if cConfig.DisableAPI {
if flags.DisableAPI {
cConfig.Common.Daemonize = false
}
log.Infof("single file mode : log_media=%s daemonize=%t", cConfig.Common.LogMedia, cConfig.Common.Daemonize)
}
@ -321,7 +286,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
if cConfig.Common.Daemonize && runtime.GOOS == "windows" {
log.Debug("Daemonization is not supported on Windows, disabling")
cConfig.Common.Daemonize = false
}
@ -339,16 +303,12 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
var crowdsecT0 time.Time
func main() {
// The initial log level is INFO, even if the user provided an -error or -warning flag
// because we need feature flags before parsing cli flags
log.SetFormatter(&log.TextFormatter{TimestampFormat: time.RFC3339, FullTimestamp: true})
if err := fflag.RegisterAllFeatures(); err != nil {
log.Fatalf("failed to register features: %s", err)
}
// some features can require configuration or command-line options,
// so we need to parse them asap. we'll load from feature.yaml later.
// so wwe need to parse them asap. we'll load from feature.yaml later.
if err := csconfig.LoadFeatureFlagsEnv(log.StandardLogger()); err != nil {
log.Fatalf("failed to set feature flags from environment: %s", err)
}
@ -373,28 +333,9 @@ func main() {
os.Exit(0)
}
if flags.CPUProfile != "" {
f, err := os.Create(flags.CPUProfile)
if err != nil {
log.Fatalf("could not create CPU profile: %s", err)
}
log.Infof("CPU profile will be written to %s", flags.CPUProfile)
if err := pprof.StartCPUProfile(f); err != nil {
f.Close()
log.Fatalf("could not start CPU profile: %s", err)
}
defer f.Close()
defer pprof.StopCPUProfile()
}
err := StartRunSvc()
if err != nil {
pprof.StopCPUProfile()
log.Fatal(err) //nolint:gocritic // Disable warning for the defer pprof.StopCPUProfile() call
log.Fatal(err)
}
os.Exit(0)
}

View file

@ -3,26 +3,24 @@ package main
import (
"fmt"
"net/http"
"time"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
v1 "github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers/v1"
"github.com/crowdsecurity/crowdsec/pkg/cache"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
// Prometheus
/*prometheus*/
var globalParserHits = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_parser_hits_total",
@ -30,7 +28,6 @@ var globalParserHits = prometheus.NewCounterVec(
},
[]string{"source", "type"},
)
var globalParserHitsOk = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_parser_hits_ok_total",
@ -38,7 +35,6 @@ var globalParserHitsOk = prometheus.NewCounterVec(
},
[]string{"source", "type"},
)
var globalParserHitsKo = prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "cs_parser_hits_ko_total",
@ -65,7 +61,7 @@ var globalCsInfo = prometheus.NewGauge(
prometheus.GaugeOpts{
Name: "cs_info",
Help: "Information about Crowdsec.",
ConstLabels: prometheus.Labels{"version": version.String()},
ConstLabels: prometheus.Labels{"version": cwversion.VersionStr()},
},
)
@ -105,29 +101,25 @@ var globalPourHistogram = prometheus.NewHistogramVec(
func computeDynamicMetrics(next http.Handler, dbClient *database.Client) http.HandlerFunc {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// catch panics here because they are not handled by servePrometheus
defer trace.CatchPanic("crowdsec/computeDynamicMetrics")
// update cache metrics (stash)
//update cache metrics (stash)
cache.UpdateCacheMetrics()
// update cache metrics (regexp)
//update cache metrics (regexp)
exprhelpers.UpdateRegexpCacheMetrics()
// decision metrics are only relevant for LAPI
//decision metrics are only relevant for LAPI
if dbClient == nil {
next.ServeHTTP(w, r)
return
}
decisions, err := dbClient.QueryDecisionCountByScenario()
decisionsFilters := make(map[string][]string, 0)
decisions, err := dbClient.QueryDecisionCountByScenario(decisionsFilters)
if err != nil {
log.Errorf("Error querying decisions for metrics: %v", err)
next.ServeHTTP(w, r)
return
}
globalActiveDecisions.Reset()
for _, d := range decisions {
globalActiveDecisions.With(prometheus.Labels{"reason": d.Scenario, "origin": d.Origin, "action": d.Type}).Set(float64(d.Count))
}
@ -139,10 +131,10 @@ func computeDynamicMetrics(next http.Handler, dbClient *database.Client) http.Ha
}
alerts, err := dbClient.AlertsCountPerScenario(alertsFilter)
if err != nil {
log.Errorf("Error querying alerts for metrics: %v", err)
next.ServeHTTP(w, r)
return
}
@ -158,18 +150,25 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
if !config.Enabled {
return
}
if config.ListenAddr == "" {
log.Warning("prometheus is enabled, but the listen address is empty, using '127.0.0.1'")
config.ListenAddr = "127.0.0.1"
}
if config.ListenPort == 0 {
log.Warning("prometheus is enabled, but the listen port is empty, using '6060'")
config.ListenPort = 6060
}
// Registering prometheus
// If in aggregated mode, do not register events associated with a source, to keep the cardinality low
if config.Level == configuration.CFG_METRICS_AGGREGATE {
if config.Level == "aggregated" {
log.Infof("Loading aggregated prometheus collectors")
prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo,
globalCsInfo, globalParsingHistogram, globalPourHistogram,
leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow,
v1.LapiRouteHits,
leaky.BucketsCurrentCount,
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, parser.NodesWlHitsOk, parser.NodesWlHits,
)
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics)
} else {
log.Infof("Loading prometheus collectors")
prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo,
@ -177,27 +176,24 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
globalCsInfo, globalParsingHistogram, globalPourHistogram,
v1.LapiRouteHits, v1.LapiMachineHits, v1.LapiBouncerHits, v1.LapiNilDecisions, v1.LapiNonNilDecisions, v1.LapiResponseTime,
leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, leaky.BucketsCurrentCount,
globalActiveDecisions, globalAlerts, parser.NodesWlHitsOk, parser.NodesWlHits,
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics,
)
globalActiveDecisions, globalAlerts,
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics)
}
}
func servePrometheus(config *csconfig.PrometheusCfg, dbClient *database.Client, agentReady chan bool) {
<-agentReady
func servePrometheus(config *csconfig.PrometheusCfg, dbClient *database.Client, apiReady chan bool, agentReady chan bool) {
if !config.Enabled {
return
}
defer trace.CatchPanic("crowdsec/servePrometheus")
defer types.CatchPanic("crowdsec/servePrometheus")
http.Handle("/metrics", computeDynamicMetrics(promhttp.Handler(), dbClient))
<-apiReady
<-agentReady
log.Debugf("serving metrics after %s ms", time.Since(crowdsecT0))
if err := http.ListenAndServe(fmt.Sprintf("%s:%d", config.ListenAddr, config.ListenPort), nil); err != nil {
// in time machine, we most likely have the LAPI using the port
if !flags.haveTimeMachine() {
log.Warningf("prometheus: %s", err)
}
log.Warningf("prometheus: %s", err)
}
}

View file

@ -3,19 +3,25 @@ package main
import (
"context"
"fmt"
"net/url"
"sync"
"time"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/go-openapi/strfmt"
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
)
func dedupAlerts(alerts []types.RuntimeAlert) ([]*models.Alert, error) {
var dedupCache []*models.Alert
for idx, alert := range alerts {
@ -25,21 +31,16 @@ func dedupAlerts(alerts []types.RuntimeAlert) ([]*models.Alert, error) {
dedupCache = append(dedupCache, alert.Alert)
continue
}
for k, src := range alert.Sources {
refsrc := *alert.Alert // copy
refsrc := *alert.Alert //copy
log.Tracef("source[%s]", k)
refsrc.Source = &src
dedupCache = append(dedupCache, &refsrc)
}
}
if len(dedupCache) != len(alerts) {
log.Tracef("went from %d to %d alerts", len(alerts), len(dedupCache))
}
return dedupCache, nil
}
@ -48,27 +49,72 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error
alertsToPush, err := dedupAlerts(alerts)
if err != nil {
return fmt.Errorf("failed to transform alerts for api: %w", err)
return errors.Wrap(err, "failed to transform alerts for api")
}
_, _, err = client.Alerts.Add(ctx, alertsToPush)
if err != nil {
return fmt.Errorf("failed sending alert to LAPI: %w", err)
return errors.Wrap(err, "failed sending alert to LAPI")
}
return nil
}
var bucketOverflows []types.Event
func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets, postOverflowCTX parser.UnixParserCtx,
postOverflowNodes []parser.Node, client *apiclient.ApiClient) error {
var (
cache []types.RuntimeAlert
cacheMutex sync.Mutex
)
func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets,
postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node, apiConfig csconfig.ApiCredentialsCfg) error {
var err error
ticker := time.NewTicker(1 * time.Second)
var cache []types.RuntimeAlert
var cacheMutex sync.Mutex
scenarios, err := cwhub.GetInstalledScenariosAsString()
if err != nil {
return errors.Wrapf(err, "loading list of installed hub scenarios: %s", err)
}
apiURL, err := url.Parse(apiConfig.URL)
if err != nil {
return errors.Wrapf(err, "parsing api url ('%s'): %s", apiConfig.URL, err)
}
papiURL, err := url.Parse(apiConfig.PapiURL)
if err != nil {
return errors.Wrapf(err, "parsing polling api url ('%s'): %s", apiConfig.PapiURL, err)
}
password := strfmt.Password(apiConfig.Password)
Client, err := apiclient.NewClient(&apiclient.Config{
MachineID: apiConfig.Login,
Password: password,
Scenarios: scenarios,
UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()),
URL: apiURL,
PapiURL: papiURL,
VersionPrefix: "v1",
UpdateScenario: cwhub.GetInstalledScenariosAsString,
})
if err != nil {
return errors.Wrapf(err, "new client api: %s", err)
}
authResp, _, err := Client.Auth.AuthenticateWatcher(context.Background(), models.WatcherAuthRequest{
MachineID: &apiConfig.Login,
Password: &password,
Scenarios: scenarios,
})
if err != nil {
return errors.Wrapf(err, "authenticate watcher (%s)", apiConfig.Login)
}
if err := Client.GetClient().Transport.(*apiclient.JWTTransport).Expiration.UnmarshalText([]byte(authResp.Expire)); err != nil {
return errors.Wrap(err, "unable to parse jwt expiration")
}
Client.GetClient().Transport.(*apiclient.JWTTransport).Token = authResp.Token
//start the heartbeat service
log.Debugf("Starting HeartBeat service")
Client.HeartBeat.StartHeartBeat(context.Background(), &outputsTomb)
LOOP:
for {
select {
@ -79,9 +125,9 @@ LOOP:
newcache := make([]types.RuntimeAlert, 0)
cache = newcache
cacheMutex.Unlock()
if err := PushAlerts(cachecopy, client); err != nil {
if err := PushAlerts(cachecopy, Client); err != nil {
log.Errorf("while pushing to api : %s", err)
// just push back the events to the queue
//just push back the events to the queue
cacheMutex.Lock()
cache = append(cache, cachecopy...)
cacheMutex.Unlock()
@ -92,13 +138,19 @@ LOOP:
cacheMutex.Lock()
cachecopy := cache
cacheMutex.Unlock()
if err := PushAlerts(cachecopy, client); err != nil {
if err := PushAlerts(cachecopy, Client); err != nil {
log.Errorf("while pushing leftovers to api : %s", err)
}
}
break LOOP
case event := <-overflow:
//if the Alert is nil, it's to signal bucket is ready for GC, don't track this
if dumpStates && event.Overflow.Alert != nil {
if bucketOverflows == nil {
bucketOverflows = make([]types.Event, 0)
}
bucketOverflows = append(bucketOverflows, event)
}
/*if alert is empty and mapKey is present, the overflow is just to cleanup bucket*/
if event.Overflow.Alert == nil && event.Overflow.Mapkey != "" {
buckets.Bucket_map.Delete(event.Overflow.Mapkey)
@ -107,17 +159,9 @@ LOOP:
/* process post overflow parser nodes */
event, err := parser.Parse(postOverflowCTX, event, postOverflowNodes)
if err != nil {
return fmt.Errorf("postoverflow failed: %w", err)
return fmt.Errorf("postoverflow failed : %s", err)
}
log.Printf("%s", *event.Overflow.Alert.Message)
// if the Alert is nil, it's to signal bucket is ready for GC, don't track this
// dump after postoveflow processing to avoid missing whitelist info
if dumpStates && event.Overflow.Alert != nil {
if bucketOverflows == nil {
bucketOverflows = make([]types.Event, 0)
}
bucketOverflows = append(bucketOverflows, event)
}
if event.Overflow.Whitelisted {
log.Printf("[%s] is whitelisted, skip.", *event.Overflow.Alert.Message)
continue
@ -137,6 +181,6 @@ LOOP:
}
ticker.Stop()
return nil
}

View file

@ -11,6 +11,7 @@ import (
)
func runParse(input chan types.Event, output chan types.Event, parserCTX parser.UnixParserCtx, nodes []parser.Node) error {
LOOP:
for {
select {
@ -21,13 +22,6 @@ LOOP:
if !event.Process {
continue
}
/*Application security engine is going to generate 2 events:
- one that is treated as a log and can go to scenarios
- another one that will go directly to LAPI*/
if event.Type == types.APPSEC {
outputEventChan <- event
continue
}
if event.Line.Module == "" {
log.Errorf("empty event.Line.Module field, the acquisition module must set it ! : %+v", event.Line)
continue
@ -55,6 +49,5 @@ LOOP:
output <- parsed
}
}
return nil
}

View file

@ -4,30 +4,29 @@ import (
"fmt"
"time"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
)
func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *leaky.Buckets, cConfig *csconfig.Config) error {
count := 0
var (
count int
)
for {
// bucket is now ready
//bucket is now ready
select {
case <-bucketsTomb.Dying():
log.Infof("Bucket routine exiting")
return nil
case parsed := <-input:
startTime := time.Now()
count++
if count%5000 == 0 {
log.Infof("%d existing buckets", leaky.LeakyRoutineCount)
// when in forensics mode, garbage collect buckets
//when in forensics mode, garbage collect buckets
if cConfig.Crowdsec.BucketsGCEnabled {
if parsed.MarshaledTime != "" {
z := &time.Time{}
@ -35,30 +34,26 @@ func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *lea
log.Warningf("Failed to unmarshal time from event '%s' : %s", parsed.MarshaledTime, err)
} else {
log.Warning("Starting buckets garbage collection ...")
if err = leaky.GarbageCollectBuckets(*z, buckets); err != nil {
return fmt.Errorf("failed to start bucket GC : %w", err)
return fmt.Errorf("failed to start bucket GC : %s", err)
}
}
}
}
}
// here we can bucketify with parsed
//here we can bucketify with parsed
poured, err := leaky.PourItemToHolders(parsed, holders, buckets)
if err != nil {
log.Errorf("bucketify failed for: %v", parsed)
continue
}
elapsed := time.Since(startTime)
globalPourHistogram.With(prometheus.Labels{"type": parsed.Line.Module, "source": parsed.Line.Src}).Observe(elapsed.Seconds())
if poured {
globalBucketPourOk.Inc()
} else {
globalBucketPourKo.Inc()
}
if len(parsed.MarshaledTime) != 0 {
if err := lastProcessedItem.UnmarshalText([]byte(parsed.MarshaledTime)); err != nil {
log.Warningf("failed to unmarshal time from event : %s", err)

View file

@ -1,18 +1,19 @@
//go:build !windows
//go:build linux || freebsd || netbsd || openbsd || solaris || !windows
// +build linux freebsd netbsd openbsd solaris !windows
package main
import (
"fmt"
"runtime/pprof"
"os"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/sirupsen/logrus/hooks/writer"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
func StartRunSvc() error {
@ -21,43 +22,41 @@ func StartRunSvc() error {
err error
)
defer trace.CatchPanic("crowdsec/StartRunSvc")
defer types.CatchPanic("crowdsec/StartRunSvc")
// Always try to stop CPU profiling to avoid passing flags around
// It's a noop if profiling is not enabled
defer pprof.StopCPUProfile()
// Set a default logger with level=fatal on stderr,
// in addition to the one we configure afterwards
log.AddHook(&writer.Hook{
Writer: os.Stderr,
LogLevels: []log.Level{
log.PanicLevel,
log.FatalLevel,
},
})
if cConfig, err = LoadConfig(flags.ConfigFile, flags.DisableAgent, flags.DisableAPI, false); err != nil {
return err
}
log.Infof("Crowdsec %s", version.String())
log.Infof("Crowdsec %s", cwversion.VersionStr())
apiReady := make(chan bool, 1)
agentReady := make(chan bool, 1)
// Enable profiling early
if cConfig.Prometheus != nil {
var dbClient *database.Client
var err error
if cConfig.DbConfig != nil {
dbClient, err = database.NewClient(cConfig.DbConfig)
if err != nil {
return fmt.Errorf("unable to create database client: %w", err)
return fmt.Errorf("unable to create database client: %s", err)
}
}
registerPrometheus(cConfig.Prometheus)
go servePrometheus(cConfig.Prometheus, dbClient, agentReady)
} else {
// avoid leaking the channel
go func() {
<-agentReady
}()
go servePrometheus(cConfig.Prometheus, dbClient, apiReady, agentReady)
}
return Serve(cConfig, agentReady)
return Serve(cConfig, apiReady, agentReady)
}

Some files were not shown because too many files have changed in this diff Show more