beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'master' into unix-socket

Browse source
This commit is contained in:
Marco Mariani 2023-12-11 15:21:14 +01:00
commit eaf8ad57c1
420 changed files with 20358 additions and 10243 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
.github/workflows/bats-hub.yml vendored
View file

@ -15,7 +15,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.21.5"]
name: "Build + tests"
runs-on: ubuntu-latest
@ -37,7 +37,6 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
- name: "Install bats dependencies"
env:

3
.github/workflows/bats-mysql.yml vendored
View file

@ -14,7 +14,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.21.5"]
name: "Build + tests"
runs-on: ubuntu-latest
@ -44,7 +44,6 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
- name: "Install bats dependencies"
env:

13
.github/workflows/bats-postgres.yml vendored
View file

@ -10,14 +10,14 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.21.5"]
name: "Build + tests"
runs-on: ubuntu-latest
timeout-minutes: 30
services:
database:
image: postgres:14
image: postgres:16
env:
POSTGRES_PASSWORD: "secret"
ports:
@ -30,6 +30,14 @@ jobs:
steps:
- name: "Install pg_dump v16"
# we can remove this when it's released on ubuntu-latest
run: |
sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list'
wget -qO- https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo tee /etc/apt/trusted.gpg.d/pgdg.asc &>/dev/null
sudo apt update
sudo apt -qq -y -o=Dpkg::Use-Pty=0 install postgresql-client-16
- name: "Force machineid"
run: |
sudo chmod +w /etc/machine-id
@ -45,7 +53,6 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
- name: "Install bats dependencies"
env:

3
.github/workflows/bats-sqlite-coverage.yml vendored
View file

@ -11,7 +11,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.21.5"]
name: "Build + tests"
runs-on: ubuntu-latest
@ -34,7 +34,6 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
- name: "Install bats dependencies"
env:

2
.github/workflows/bats.yml vendored
View file

@ -31,7 +31,7 @@ jobs:
# Jobs for Postgres (and sometimes MySQL) can have failing tests on GitHub
# CI, but they pass when run on devs' machines or in the release checks. We
# disable them here by default. Remove the if..false to enable them.
# disable them here by default. Remove if...false to enable them.
mariadb:
uses: ./.github/workflows/bats-mysql.yml

3
.github/workflows/ci-windows-build-msi.yml vendored
View file

@ -23,7 +23,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.21.5"]
name: Build
runs-on: windows-2019
@ -40,7 +40,6 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
- name: Build
run: make windows_installer BUILD_RE2_WASM=1

18
.github/workflows/codeql-analysis.yml vendored
View file

@ -45,6 +45,9 @@ jobs:
steps:
- name: Checkout repository
uses: actions/checkout@v3
with:
# required to pick up tags for BUILD_VERSION
fetch-depth: 0
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
@ -58,8 +61,8 @@ jobs:
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v2
# - name: Autobuild
# uses: github/codeql-action/autobuild@v2
# ℹ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
@ -68,9 +71,14 @@ jobs:
# and modify them (or add more) to build your code if your project
# uses a compiled language
#- run: |
# make bootstrap
# make release
- name: "Set up Go"
uses: actions/setup-go@v4
with:
go-version: "1.21.0"
cache-dependency-path: "**/go.sum"
- run: |
make clean build BUILD_RE2_WASM=1
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2

5
.github/workflows/go-tests-windows.yml vendored
View file

@ -22,7 +22,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.21.5"]
name: "Build + tests"
runs-on: windows-2022
@ -39,7 +39,6 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
- name: Build
run: |
@ -61,7 +60,7 @@ jobs:
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
version: v1.51
version: v1.54
args: --issues-exit-code=1 --timeout 10m
only-new-issues: false
# the cache is already managed above, enabling it here

22
.github/workflows/go-tests.yml vendored
View file

@ -34,7 +34,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.21.5"]
name: "Build + tests"
runs-on: ubuntu-latest
@ -108,6 +108,18 @@ jobs:
--health-timeout 10s
--health-retries 5
loki:
image: grafana/loki:2.8.0
ports:
- "3100:3100"
options: >-
--name=loki1
--health-cmd "wget -q -O - http://localhost:3100/ready | grep 'ready'"
--health-interval 30s
--health-timeout 10s
--health-retries 5
--health-start-period 30s
steps:
- name: Check out CrowdSec repository
@ -120,7 +132,6 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
- name: Build and run tests, static
run: |
@ -129,12 +140,13 @@ jobs:
go install github.com/kyoh86/richgo@v0.3.10
set -o pipefail
make build BUILD_STATIC=1
make go-acc | richgo testfilter
make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
- name: Run tests again, dynamic
run: |
make clean build
make go-acc | richgo testfilter
set -o pipefail
make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
- name: Upload unit coverage to Codecov
uses: codecov/codecov-action@v3
@ -145,7 +157,7 @@ jobs:
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
version: v1.51
version: v1.54
args: --issues-exit-code=1 --timeout 10m
only-new-issues: false
# the cache is already managed above, enabling it here

5
.github/workflows/release_publish-package.yml vendored
View file

@ -14,7 +14,7 @@ jobs:
build:
strategy:
matrix:
go-version: ["1.20.5"]
go-version: ["1.21.5"]
name: Build and upload binary package
runs-on: ubuntu-latest
@ -30,7 +30,6 @@ jobs:
uses: actions/setup-go@v4
with:
go-version: ${{ matrix.go-version }}
cache-dependency-path: "**/go.sum"
- name: Build the binaries
run: |
@ -42,4 +41,4 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
tag_name="${GITHUB_REF##*/}"
hub release edit -a crowdsec-release.tgz -a vendor.tgz -m "" "$tag_name"
gh release upload "$tag_name" crowdsec-release.tgz vendor.tgz *-vendor.tar.xz

4
.github/workflows/release_publish_docker-image.yml vendored
View file

@ -29,10 +29,10 @@ jobs:
VERSION=pr-${{ github.event.number }}
fi
TAGS="${DOCKER_IMAGE}:${VERSION},${GHCR_IMAGE}:${VERSION}"
TAGS_SLIM="${DOCKER_IMAGE}:${VERSION}-slim"
TAGS_SLIM="${DOCKER_IMAGE}:${VERSION}-slim,${GHCR_IMAGE}:${VERSION}-slim"
if [[ ${{ github.event.action }} == released ]]; then
TAGS=$TAGS,${DOCKER_IMAGE}:latest,${GHCR_IMAGE}:latest
TAGS_SLIM=$TAGS_SLIM,${DOCKER_IMAGE}:slim
TAGS_SLIM=$TAGS_SLIM,${DOCKER_IMAGE}:slim,${GHCR_IMAGE}:slim
fi
echo "version=${VERSION}" >> $GITHUB_OUTPUT
echo "tags=${TAGS}" >> $GITHUB_OUTPUT

6
.gitignore vendored
View file

@ -41,11 +41,7 @@ vendor.tgz
# crowdsec binaries
cmd/crowdsec-cli/cscli
cmd/crowdsec/crowdsec
plugins/notifications/http/notification-http
plugins/notifications/slack/notification-slack
plugins/notifications/splunk/notification-splunk
plugins/notifications/email/notification-email
plugins/notifications/dummy/notification-dummy
cmd/notification-*/notification-*
# Test cache (downloaded files)
.cache

15
.golangci.yml
View file

@ -105,6 +105,7 @@ linters:
# Recommended? (easy)
#
- depguard # Go linter that checks if package imports are in a list of acceptable packages
- dogsled # Checks assignments with too many blank identifiers (e.g. x, _, _, _, := f())
- errchkjson # Checks types passed to the json encoding functions. Reports unsupported types and optionally reports occations, where the check for the returned error can be omitted.
- errorlint # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13.
@ -121,10 +122,10 @@ linters:
- nosprintfhostport # Checks for misuse of Sprintf to construct a host with port in a URL.
- promlinter # Check Prometheus metrics naming via promlint
- revive # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint.
- tagalign # check that struct tags are well aligned [fast: true, auto-fix: true]
- thelper # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers
- wastedassign # wastedassign finds wasted assignment statements.
- wrapcheck # Checks that errors returned from external packages are wrapped
- depguard # Go linter that checks if package imports are in a list of acceptable packages
#
# Recommended? (requires some work)
@ -198,6 +199,18 @@ issues:
- govet
text: "shadow: declaration of \"err\" shadows declaration"
#
# typecheck
#
- linters:
- typecheck
text: "undefined: min"
- linters:
- typecheck
text: "undefined: max"
#
# errcheck
#

15
Dockerfile
View file

@ -1,7 +1,7 @@
# vim: set ft=dockerfile:
ARG GOVERSION=1.20.5
ARG GOVERSION=1.21.5
FROM golang:${GOVERSION}-alpine AS build
FROM golang:${GOVERSION}-alpine3.18 AS build
WORKDIR /go/src/crowdsec
@ -32,7 +32,7 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \
FROM alpine:latest as slim
RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata bash && \
RUN apk add --no-cache --repository=http://dl-cdn.alpinelinux.org/alpine/edge/community tzdata bash rsync && \
mkdir -p /staging/etc/crowdsec && \
mkdir -p /staging/etc/crowdsec/acquis.d && \
mkdir -p /staging/var/lib/crowdsec && \
@ -52,10 +52,11 @@ FROM slim as plugins
# Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
# The files are here for reference, as users will need to mount a new version to be actually able to use notifications
COPY --from=build /go/src/crowdsec/plugins/notifications/email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml /staging/etc/crowdsec/notifications/sentinel.yaml
COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
FROM slim as geoip

14
Dockerfile.debian
View file

@ -1,5 +1,5 @@
# vim: set ft=dockerfile:
ARG GOVERSION=1.20.5
ARG GOVERSION=1.21.5
FROM golang:${GOVERSION}-bookworm AS build
@ -47,7 +47,8 @@ RUN apt-get update && \
iproute2 \
ca-certificates \
bash \
tzdata && \
tzdata \
rsync && \
mkdir -p /staging/etc/crowdsec && \
mkdir -p /staging/etc/crowdsec/acquis.d && \
mkdir -p /staging/var/lib/crowdsec && \
@ -68,10 +69,11 @@ FROM slim as plugins
# Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp
# The files are here for reference, as users will need to mount a new version to be actually able to use notifications
COPY --from=build /go/src/crowdsec/plugins/notifications/email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
COPY --from=build /go/src/crowdsec/plugins/notifications/splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-email/email.yaml /staging/etc/crowdsec/notifications/email.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-http/http.yaml /staging/etc/crowdsec/notifications/http.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-slack/slack.yaml /staging/etc/crowdsec/notifications/slack.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-splunk/splunk.yaml /staging/etc/crowdsec/notifications/splunk.yaml
COPY --from=build /go/src/crowdsec/cmd/notification-sentinel/sentinel.yaml /staging/etc/crowdsec/notifications/sentinel.yaml
COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins
FROM slim as geoip

109
Makefile
View file

@ -23,26 +23,31 @@ BUILD_RE2_WASM ?= 0
BUILD_STATIC ?= 0
# List of plugins to build
PLUGINS ?= $(patsubst ./plugins/notifications/%,%,$(wildcard ./plugins/notifications/*))
PLUGINS ?= $(patsubst ./cmd/notification-%,%,$(wildcard ./cmd/notification-*))
# Can be overriden, if you can deal with the consequences
BUILD_REQUIRE_GO_MAJOR ?= 1
BUILD_REQUIRE_GO_MINOR ?= 20
BUILD_REQUIRE_GO_MINOR ?= 21
#--------------------------------------
GOCMD = go
GOTEST = $(GOCMD) test
GO = go
GOTEST = $(GO) test
BUILD_CODENAME ?= alphaga
CROWDSEC_FOLDER = ./cmd/crowdsec
CSCLI_FOLDER = ./cmd/crowdsec-cli/
PLUGINS_DIR = ./plugins/notifications
PLUGINS_DIR_PREFIX = ./cmd/notification-
CROWDSEC_BIN = crowdsec$(EXT)
CSCLI_BIN = cscli$(EXT)
# semver comparison to select the hub branch requires the version to start with "v"
ifneq ($(call substr,$(BUILD_VERSION),1,1),v)
$(error BUILD_VERSION "$(BUILD_VERSION)" should start with "v")
endif
# Directory for the release files
RELDIR = crowdsec-$(BUILD_VERSION)
@ -59,15 +64,15 @@ bool = $(if $(filter $(call lc, $1),1 yes true),1,0)
#--------------------------------------
#
# Define MAKE_FLAGS and LD_OPTS for the sub-makefiles in cmd/ and plugins/
# Define MAKE_FLAGS and LD_OPTS for the sub-makefiles in cmd/
#
MAKE_FLAGS = --no-print-directory GOARCH=$(GOARCH) GOOS=$(GOOS) RM="$(RM)" WIN_IGNORE_ERR="$(WIN_IGNORE_ERR)" CP="$(CP)" CPR="$(CPR)" MKDIR="$(MKDIR)"
LD_OPTS_VARS= \
-X 'github.com/crowdsecurity/go-cs-lib/pkg/version.Version=$(BUILD_VERSION)' \
-X 'github.com/crowdsecurity/go-cs-lib/pkg/version.BuildDate=$(BUILD_TIMESTAMP)' \
-X 'github.com/crowdsecurity/go-cs-lib/pkg/version.Tag=$(BUILD_TAG)' \
-X 'github.com/crowdsecurity/go-cs-lib/version.Version=$(BUILD_VERSION)' \
-X 'github.com/crowdsecurity/go-cs-lib/version.BuildDate=$(BUILD_TIMESTAMP)' \
-X 'github.com/crowdsecurity/go-cs-lib/version.Tag=$(BUILD_TAG)' \
-X '$(GO_MODULE_NAME)/pkg/cwversion.Codename=$(BUILD_CODENAME)' \
-X '$(GO_MODULE_NAME)/pkg/csconfig.defaultConfigDir=$(DEFAULT_CONFIGDIR)' \
-X '$(GO_MODULE_NAME)/pkg/csconfig.defaultDataDir=$(DEFAULT_DATADIR)'
@ -87,7 +92,6 @@ ifeq ($(PKG_CONFIG),)
endif
ifeq ($(RE2_CHECK),)
# we could detect the platform and suggest the command to install
RE2_FAIL := "libre2-dev is not installed, please install it or set BUILD_RE2_WASM=1 to use the WebAssembly version"
else
# += adds a space that we don't want
@ -96,6 +100,7 @@ LD_OPTS_VARS += -X '$(GO_MODULE_NAME)/pkg/cwversion.Libre2=C++'
endif
endif
# Build static to avoid the runtime dependency on libre2.so
ifeq ($(call bool,$(BUILD_STATIC)),1)
BUILD_TYPE = static
EXTLDFLAGS := -extldflags '-static'
@ -104,10 +109,19 @@ BUILD_TYPE = dynamic
EXTLDFLAGS :=
endif
export LD_OPTS=-ldflags "-s -w $(EXTLDFLAGS) $(LD_OPTS_VARS)" \
-trimpath -tags $(GO_TAGS)
# Build with debug symbols, and disable optimizations + inlining, to use Delve
ifeq ($(call bool,$(DEBUG)),1)
STRIP_SYMBOLS :=
DISABLE_OPTIMIZATION := -gcflags "-N -l"
else
STRIP_SYMBOLS := -s -w
DISABLE_OPTIMIZATION :=
endif
ifneq (,$(TEST_COVERAGE))
export LD_OPTS=-ldflags "$(STRIP_SYMBOLS) $(EXTLDFLAGS) $(LD_OPTS_VARS)" \
-trimpath -tags $(GO_TAGS) $(DISABLE_OPTIMIZATION)
ifeq ($(call bool,$(TEST_COVERAGE)),1)
LD_OPTS += -cover
endif
@ -130,19 +144,47 @@ ifneq (,$(RE2_CHECK))
else
$(info Fallback to WebAssembly regexp library. To use the C++ version, make sure you have installed libre2-dev and pkg-config.)
endif
ifeq ($(call bool,$(DEBUG)),1)
$(info Building with debug symbols and disabled optimizations)
endif
ifeq ($(call bool,$(TEST_COVERAGE)),1)
$(info Test coverage collection enabled)
endif
$(info )
.PHONY: all
all: clean test build
.PHONY: plugins
plugins:
@$(foreach plugin,$(PLUGINS), \
$(MAKE) -C $(PLUGINS_DIR)/$(plugin) build $(MAKE_FLAGS); \
$(MAKE) -C $(PLUGINS_DIR_PREFIX)$(plugin) build $(MAKE_FLAGS); \
)
# same as "$(MAKE) -f debian/rules clean" but without the dependency on debhelper
.PHONY: clean-debian
clean-debian:
@$(RM) -r debian/crowdsec
@$(RM) -r debian/crowdsec
@$(RM) -r debian/files
@$(RM) -r debian/.debhelper
@$(RM) -r debian/*.substvars
@$(RM) -r debian/*-stamp
.PHONY: clean-rpm
clean-rpm:
@$(RM) -r rpm/BUILD
@$(RM) -r rpm/BUILDROOT
@$(RM) -r rpm/RPMS
@$(RM) -r rpm/SOURCES/*.tar.gz
@$(RM) -r rpm/SRPMS
.PHONY: clean
clean: testclean
clean: clean-debian clean-rpm testclean
@$(MAKE) -C $(CROWDSEC_FOLDER) clean $(MAKE_FLAGS)
@$(MAKE) -C $(CSCLI_FOLDER) clean $(MAKE_FLAGS)
@$(RM) $(CROWDSEC_BIN) $(WIN_IGNORE_ERR)
@ -150,7 +192,7 @@ clean: testclean
@$(RM) *.log $(WIN_IGNORE_ERR)
@$(RM) crowdsec-release.tgz $(WIN_IGNORE_ERR)
@$(foreach plugin,$(PLUGINS), \
$(MAKE) -C $(PLUGINS_DIR)/$(plugin) clean $(MAKE_FLAGS); \
$(MAKE) -C $(PLUGINS_DIR_PREFIX)$(plugin) clean $(MAKE_FLAGS); \
)
.PHONY: cscli
@ -161,6 +203,12 @@ cscli: goversion
crowdsec: goversion
@$(MAKE) -C $(CROWDSEC_FOLDER) build $(MAKE_FLAGS)
.PHONY: notification-email
notification-email: goversion
@$(MAKE) -C cmd/notification-email build $(MAKE_FLAGS)
.PHONY: testclean
testclean: bats-clean
@$(RM) pkg/apiserver/ent $(WIN_IGNORE_ERR)
@ -184,8 +232,7 @@ test: testenv goversion
# run the tests with localstack and coverage
.PHONY: go-acc
go-acc: testenv goversion
go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS) | \
sed 's/ *coverage:.*of statements in.*//'
go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS)
# mock AWS services
.PHONY: localstack
@ -196,27 +243,17 @@ localstack:
localstack-stop:
docker-compose -f test/localstack/docker-compose.yml down
# list of plugins that contain go.mod
PLUGIN_VENDOR = $(foreach plugin,$(PLUGINS),$(shell if [ -f $(PLUGINS_DIR)/$(plugin)/go.mod ]; then echo $(PLUGINS_DIR)/$(plugin); fi))
# build vendor.tgz to be distributed with the release
.PHONY: vendor
vendor:
$(foreach plugin_dir,$(PLUGIN_VENDOR), \
cd $(plugin_dir) >/dev/null && \
$(GOCMD) mod vendor && \
cd - >/dev/null; \
)
$(GOCMD) mod vendor
tar -czf vendor.tgz vendor $(foreach plugin_dir,$(PLUGIN_VENDOR),$(plugin_dir)/vendor)
vendor: vendor-remove
$(GO) mod vendor
tar czf vendor.tgz vendor
tar --create --auto-compress --file=$(RELDIR)-vendor.tar.xz vendor
# remove vendor directories and vendor.tgz
.PHONY: vendor-remove
vendor-remove:
$(foreach plugin_dir,$(PLUGIN_VENDOR), \
$(RM) $(plugin_dir)/vendor; \
)
$(RM) vendor vendor.tgz
$(RM) vendor vendor.tgz *-vendor.tar.xz
.PHONY: package
package:
@ -227,9 +264,9 @@ package:
@$(CP) $(CSCLI_FOLDER)/$(CSCLI_BIN) $(RELDIR)/cmd/crowdsec-cli
@$(foreach plugin,$(PLUGINS), \
$(MKDIR) $(RELDIR)/$(PLUGINS_DIR)/$(plugin); \
$(CP) $(PLUGINS_DIR)/$(plugin)/notification-$(plugin)$(EXT) $(RELDIR)/$(PLUGINS_DIR)/$(plugin); \
$(CP) $(PLUGINS_DIR)/$(plugin)/$(plugin).yaml $(RELDIR)/$(PLUGINS_DIR)/$(plugin)/; \
$(MKDIR) $(RELDIR)/$(PLUGINS_DIR_PREFIX)$(plugin); \
$(CP) $(PLUGINS_DIR_PREFIX)$(plugin)/notification-$(plugin)$(EXT) $(RELDIR)/$(PLUGINS_DIR_PREFIX)$(plugin); \
$(CP) $(PLUGINS_DIR_PREFIX)$(plugin)/$(plugin).yaml $(RELDIR)/$(PLUGINS_DIR_PREFIX)$(plugin)/; \
)
@$(CPR) ./config $(RELDIR)

2
azure-pipelines.yml
View file

@ -27,7 +27,7 @@ stages:
- task: GoTool@0
displayName: "Install Go 1.20"
inputs:
version: '1.20.5'
version: '1.21.5'
- pwsh: |
choco install -y make

8
cmd/crowdsec-cli/Makefile
View file

@ -4,10 +4,8 @@ ifeq ($(OS), Windows_NT)
EXT = .exe
endif
# Go parameters
GOCMD = go
GOBUILD = $(GOCMD) build
GOTEST = $(GOCMD) test
GO = go
GOBUILD = $(GO) build
BINARY_NAME = cscli$(EXT)
PREFIX ?= "/"
@ -17,7 +15,7 @@ BIN_PREFIX = $(PREFIX)"/usr/local/bin/"
all: clean build
build: clean
$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
.PHONY: install
install: install-conf install-bin

117
cmd/crowdsec-cli/alerts.go
View file

@ -19,8 +19,9 @@ import (
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/models"
@ -124,6 +125,12 @@ func AlertsToTable(alerts *models.GetAlertsResponse, printMachine bool) error {
}
csvwriter.Flush()
} else if csConfig.Cscli.Output == "json" {
if *alerts == nil {
// avoid returning "null" in json
// could be cleaner if we used slice of alerts directly
fmt.Println("[]")
return nil
}
x, _ := json.MarshalIndent(alerts, "", " ")
fmt.Printf("%s", string(x))
} else if csConfig.Cscli.Output == "human" {
@ -200,12 +207,19 @@ func DisplayOneAlert(alert *models.Alert, withDetail bool) error {
return nil
}
func NewAlertsCmd() *cobra.Command {
var cmdAlerts = &cobra.Command{
type cliAlerts struct{}
func NewCLIAlerts() *cliAlerts {
return &cliAlerts{}
}
func (cli cliAlerts) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "alerts [action]",
Short: "Manage alerts",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
Aliases: []string{"alert"},
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
var err error
if err := csConfig.LoadAPIClient(); err != nil {
@ -230,15 +244,15 @@ func NewAlertsCmd() *cobra.Command {
},
}
cmdAlerts.AddCommand(NewAlertsListCmd())
cmdAlerts.AddCommand(NewAlertsInspectCmd())
cmdAlerts.AddCommand(NewAlertsFlushCmd())
cmdAlerts.AddCommand(NewAlertsDeleteCmd())
cmd.AddCommand(cli.NewListCmd())
cmd.AddCommand(cli.NewInspectCmd())
cmd.AddCommand(cli.NewFlushCmd())
cmd.AddCommand(cli.NewDeleteCmd())
return cmdAlerts
return cmd
}
func NewAlertsListCmd() *cobra.Command {
func (cli cliAlerts) NewListCmd() *cobra.Command {
var alertListFilter = apiclient.AlertsListOpts{
ScopeEquals: new(string),
ValueEquals: new(string),
@ -251,10 +265,11 @@ func NewAlertsListCmd() *cobra.Command {
IncludeCAPI: new(bool),
OriginEquals: new(string),
}
var limit = new(int)
limit := new(int)
contained := new(bool)
var printMachine bool
var cmdAlertsList = &cobra.Command{
cmd := &cobra.Command{
Use: "list [filters]",
Short: "List alerts",
Example: `cscli alerts list
@ -344,25 +359,25 @@ cscli alerts list --type ban`,
return nil
},
}
cmdAlertsList.Flags().SortFlags = false
cmdAlertsList.Flags().BoolVarP(alertListFilter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
cmdAlertsList.Flags().StringVar(alertListFilter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
cmdAlertsList.Flags().StringVar(alertListFilter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
cmdAlertsList.Flags().StringVarP(alertListFilter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
cmdAlertsList.Flags().StringVarP(alertListFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
cmdAlertsList.Flags().StringVarP(alertListFilter.RangeEquals, "range", "r", "", "restrict to alerts from this range (shorthand for --scope range --value <RANGE/X>)")
cmdAlertsList.Flags().StringVar(alertListFilter.TypeEquals, "type", "", "restrict to alerts with given decision type (ie. ban, captcha)")
cmdAlertsList.Flags().StringVar(alertListFilter.ScopeEquals, "scope", "", "restrict to alerts of this scope (ie. ip,range)")
cmdAlertsList.Flags().StringVarP(alertListFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmdAlertsList.Flags().StringVar(alertListFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmdAlertsList.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
cmdAlertsList.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that sent alerts")
cmdAlertsList.Flags().IntVarP(limit, "limit", "l", 50, "limit size of alerts list table (0 to view all alerts)")
cmd.Flags().SortFlags = false
cmd.Flags().BoolVarP(alertListFilter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
cmd.Flags().StringVar(alertListFilter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
cmd.Flags().StringVar(alertListFilter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
cmd.Flags().StringVarP(alertListFilter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
cmd.Flags().StringVarP(alertListFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
cmd.Flags().StringVarP(alertListFilter.RangeEquals, "range", "r", "", "restrict to alerts from this range (shorthand for --scope range --value <RANGE/X>)")
cmd.Flags().StringVar(alertListFilter.TypeEquals, "type", "", "restrict to alerts with given decision type (ie. ban, captcha)")
cmd.Flags().StringVar(alertListFilter.ScopeEquals, "scope", "", "restrict to alerts of this scope (ie. ip,range)")
cmd.Flags().StringVarP(alertListFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmd.Flags().StringVar(alertListFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmd.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
cmd.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that sent alerts")
cmd.Flags().IntVarP(limit, "limit", "l", 50, "limit size of alerts list table (0 to view all alerts)")
return cmdAlertsList
return cmd
}
func NewAlertsDeleteCmd() *cobra.Command {
func (cli cliAlerts) NewDeleteCmd() *cobra.Command {
var ActiveDecision *bool
var AlertDeleteAll bool
var delAlertByID string
@ -374,7 +389,7 @@ func NewAlertsDeleteCmd() *cobra.Command {
IPEquals: new(string),
RangeEquals: new(string),
}
var cmdAlertsDelete = &cobra.Command{
cmd := &cobra.Command{
Use: "delete [filters] [--all]",
Short: `Delete alerts
/!\ This command can be use only on the same machine than the local API.`,
@ -452,21 +467,21 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`,
return nil
},
}
cmdAlertsDelete.Flags().SortFlags = false
cmdAlertsDelete.Flags().StringVar(alertDeleteFilter.ScopeEquals, "scope", "", "the scope (ie. ip,range)")
cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmdAlertsDelete.Flags().StringVarP(alertDeleteFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmdAlertsDelete.Flags().StringVar(&delAlertByID, "id", "", "alert ID")
cmdAlertsDelete.Flags().BoolVarP(&AlertDeleteAll, "all", "a", false, "delete all alerts")
cmdAlertsDelete.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
return cmdAlertsDelete
cmd.Flags().SortFlags = false
cmd.Flags().StringVar(alertDeleteFilter.ScopeEquals, "scope", "", "the scope (ie. ip,range)")
cmd.Flags().StringVarP(alertDeleteFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmd.Flags().StringVarP(alertDeleteFilter.ScenarioEquals, "scenario", "s", "", "the scenario (ie. crowdsecurity/ssh-bf)")
cmd.Flags().StringVarP(alertDeleteFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmd.Flags().StringVarP(alertDeleteFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmd.Flags().StringVar(&delAlertByID, "id", "", "alert ID")
cmd.Flags().BoolVarP(&AlertDeleteAll, "all", "a", false, "delete all alerts")
cmd.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
return cmd
}
func NewAlertsInspectCmd() *cobra.Command {
func (cli cliAlerts) NewInspectCmd() *cobra.Command {
var details bool
var cmdAlertsInspect = &cobra.Command{
cmd := &cobra.Command{
Use: `inspect "alert_id"`,
Short: `Show info about an alert`,
Example: `cscli alerts inspect 123`,
@ -508,16 +523,16 @@ func NewAlertsInspectCmd() *cobra.Command {
return nil
},
}
cmdAlertsInspect.Flags().SortFlags = false
cmdAlertsInspect.Flags().BoolVarP(&details, "details", "d", false, "show alerts with events")
cmd.Flags().SortFlags = false
cmd.Flags().BoolVarP(&details, "details", "d", false, "show alerts with events")
return cmdAlertsInspect
return cmd
}
func NewAlertsFlushCmd() *cobra.Command {
func (cli cliAlerts) NewFlushCmd() *cobra.Command {
var maxItems int
var maxAge string
var cmdAlertsFlush = &cobra.Command{
cmd := &cobra.Command{
Use: `flush`,
Short: `Flush alerts
/!\ This command can be used only on the same machine than the local API`,
@ -525,8 +540,8 @@ func NewAlertsFlushCmd() *cobra.Command {
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
var err error
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
return fmt.Errorf("local API is disabled, please run this command on the local API machine")
if err := require.LAPI(csConfig); err != nil {
return err
}
dbClient, err = database.NewClient(csConfig.DbConfig)
if err != nil {
@ -543,9 +558,9 @@ func NewAlertsFlushCmd() *cobra.Command {
},
}
cmdAlertsFlush.Flags().SortFlags = false
cmdAlertsFlush.Flags().IntVar(&maxItems, "max-items", 5000, "Maximum number of alert items to keep in the database")
cmdAlertsFlush.Flags().StringVar(&maxAge, "max-age", "7d", "Maximum age of alert items to keep in the database")
cmd.Flags().SortFlags = false
cmd.Flags().IntVar(&maxItems, "max-items", 5000, "Maximum number of alert items to keep in the database")
cmd.Flags().StringVar(&maxAge, "max-age", "7d", "Maximum age of alert items to keep in the database")
return cmdAlertsFlush
return cmd
}

194
cmd/crowdsec-cli/bouncers.go
View file

@ -8,11 +8,13 @@ import (
"strings"
"time"
"github.com/AlecAivazis/survey/v2"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/exp/slices"
"slices"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/types"
@ -23,16 +25,18 @@ func getBouncers(out io.Writer, dbClient *database.Client) error {
if err != nil {
return fmt.Errorf("unable to list bouncers: %s", err)
}
if csConfig.Cscli.Output == "human" {
switch csConfig.Cscli.Output {
case "human":
getBouncersTable(out, bouncers)
} else if csConfig.Cscli.Output == "json" {
case "json":
enc := json.NewEncoder(out)
enc.SetIndent("", " ")
if err := enc.Encode(bouncers); err != nil {
return fmt.Errorf("failed to unmarshal: %w", err)
}
return nil
} else if csConfig.Cscli.Output == "raw" {
case "raw":
csvwriter := csv.NewWriter(out)
err := csvwriter.Write([]string{"name", "ip", "revoked", "last_pull", "type", "version", "auth_type"})
if err != nil {
@ -52,14 +56,52 @@ func getBouncers(out io.Writer, dbClient *database.Client) error {
}
csvwriter.Flush()
}
return nil
}
func NewBouncersListCmd() *cobra.Command {
cmdBouncersList := &cobra.Command{
type cliBouncers struct {}
func NewCLIBouncers() *cliBouncers {
return &cliBouncers{}
}
func (cli cliBouncers) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "bouncers [action]",
Short: "Manage bouncers [requires local API]",
Long: `To list/add/delete/prune bouncers.
Note: This command requires database direct access, so is intended to be run on Local API/master.
`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"bouncer"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
var err error
if err = require.LAPI(csConfig); err != nil {
return err
}
dbClient, err = database.NewClient(csConfig.DbConfig)
if err != nil {
return fmt.Errorf("unable to create new database client: %s", err)
}
return nil
},
}
cmd.AddCommand(cli.NewListCmd())
cmd.AddCommand(cli.NewAddCmd())
cmd.AddCommand(cli.NewDeleteCmd())
cmd.AddCommand(cli.NewPruneCmd())
return cmd
}
func (cli cliBouncers) NewListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "list",
Short: "List bouncers",
Long: `List bouncers`,
Short: "list all bouncers within the database",
Example: `cscli bouncers list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
@ -72,16 +114,13 @@ func NewBouncersListCmd() *cobra.Command {
},
}
return cmdBouncersList
return cmd
}
func runBouncersAdd(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
func (cli cliBouncers) add(cmd *cobra.Command, args []string) error {
keyLength := 32
keyLength, err := flags.GetInt("length")
if err != nil {
return err
}
flags := cmd.Flags()
key, err := flags.GetString("key")
if err != nil {
@ -106,13 +145,14 @@ func runBouncersAdd(cmd *cobra.Command, args []string) error {
return fmt.Errorf("unable to create bouncer: %s", err)
}
if csConfig.Cscli.Output == "human" {
fmt.Printf("Api key for '%s':\n\n", keyName)
switch csConfig.Cscli.Output {
case "human":
fmt.Printf("API key for '%s':\n\n", keyName)
fmt.Printf(" %s\n\n", apiKey)
fmt.Print("Please keep this key since you will not be able to retrieve it!\n")
} else if csConfig.Cscli.Output == "raw" {
case "raw":
fmt.Printf("%s", apiKey)
} else if csConfig.Cscli.Output == "json" {
case "json":
j, err := json.Marshal(apiKey)
if err != nil {
return fmt.Errorf("unable to marshal api key")
@ -123,28 +163,26 @@ func runBouncersAdd(cmd *cobra.Command, args []string) error {
return nil
}
func NewBouncersAddCmd() *cobra.Command {
cmdBouncersAdd := &cobra.Command{
Use: "add MyBouncerName [--length 16]",
Short: "add bouncer",
Long: `add bouncer`,
func (cli cliBouncers) NewAddCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "add MyBouncerName",
Short: "add a single bouncer to the database",
Example: `cscli bouncers add MyBouncerName
cscli bouncers add MyBouncerName -l 24
cscli bouncers add MyBouncerName -k <random-key>`,
cscli bouncers add MyBouncerName --key <random-key>`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: runBouncersAdd,
RunE: cli.add,
}
flags := cmdBouncersAdd.Flags()
flags.IntP("length", "l", 16, "length of the api key")
flags := cmd.Flags()
flags.StringP("length", "l", "", "length of the api key")
flags.MarkDeprecated("length", "use --key instead")
flags.StringP("key", "k", "", "api key for the bouncer")
return cmdBouncersAdd
return cmd
}
func runBouncersDelete(cmd *cobra.Command, args []string) error {
func (cli cliBouncers) delete(cmd *cobra.Command, args []string) error {
for _, bouncerID := range args {
err := dbClient.DeleteBouncer(bouncerID)
if err != nil {
@ -156,10 +194,10 @@ func runBouncersDelete(cmd *cobra.Command, args []string) error {
return nil
}
func NewBouncersDeleteCmd() *cobra.Command {
cmdBouncersDelete := &cobra.Command{
func (cli cliBouncers) NewDeleteCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "delete MyBouncerName",
Short: "delete bouncer",
Short: "delete bouncer(s) from the database",
Args: cobra.MinimumNArgs(1),
Aliases: []string{"remove"},
DisableAutoGenTag: true,
@ -182,38 +220,78 @@ func NewBouncersDeleteCmd() *cobra.Command {
}
return ret, cobra.ShellCompDirectiveNoFileComp
},
RunE: runBouncersDelete,
RunE: cli.delete,
}
return cmdBouncersDelete
return cmd
}
func NewBouncersCmd() *cobra.Command {
var cmdBouncers = &cobra.Command{
Use: "bouncers [action]",
Short: "Manage bouncers [requires local API]",
Long: `To list/add/delete bouncers.
Note: This command requires database direct access, so is intended to be run on Local API/master.
`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"bouncer"},
func (cli cliBouncers) NewPruneCmd() *cobra.Command {
var parsedDuration time.Duration
cmd := &cobra.Command{
Use: "prune",
Short: "prune multiple bouncers from the database",
Args: cobra.NoArgs,
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
Example: `cscli bouncers prune -d 60m
cscli bouncers prune -d 60m --force`,
PreRunE: func(cmd *cobra.Command, args []string) error {
dur, _ := cmd.Flags().GetString("duration")
var err error
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
return fmt.Errorf("local API is disabled, please run this command on the local API machine")
}
dbClient, err = database.NewClient(csConfig.DbConfig)
parsedDuration, err = time.ParseDuration(fmt.Sprintf("-%s", dur))
if err != nil {
return fmt.Errorf("unable to create new database client: %s", err)
return fmt.Errorf("unable to parse duration '%s': %s", dur, err)
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
force, _ := cmd.Flags().GetBool("force")
if parsedDuration >= 0-2*time.Minute {
var answer bool
prompt := &survey.Confirm{
Message: "The duration you provided is less than or equal 2 minutes this may remove active bouncers continue ?",
Default: false,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask about prune check: %s", err)
}
if !answer {
fmt.Println("user aborted prune no changes were made")
return nil
}
}
bouncers, err := dbClient.QueryBouncersLastPulltimeLT(time.Now().UTC().Add(parsedDuration))
if err != nil {
return fmt.Errorf("unable to query bouncers: %s", err)
}
if len(bouncers) == 0 {
fmt.Println("no bouncers to prune")
return nil
}
getBouncersTable(color.Output, bouncers)
if !force {
var answer bool
prompt := &survey.Confirm{
Message: "You are about to PERMANENTLY remove the above bouncers from the database these will NOT be recoverable, continue ?",
Default: false,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask about prune check: %s", err)
}
if !answer {
fmt.Println("user aborted prune no changes were made")
return nil
}
}
nbDeleted, err := dbClient.BulkDeleteBouncers(bouncers)
if err != nil {
return fmt.Errorf("unable to prune bouncers: %s", err)
}
fmt.Printf("successfully delete %d bouncers\n", nbDeleted)
return nil
},
}
cmdBouncers.AddCommand(NewBouncersListCmd())
cmdBouncers.AddCommand(NewBouncersAddCmd())
cmdBouncers.AddCommand(NewBouncersDeleteCmd())
return cmdBouncers
cmd.Flags().StringP("duration", "d", "60m", "duration of time since last pull")
cmd.Flags().Bool("force", false, "force prune without asking for confirmation")
return cmd
}

120
cmd/crowdsec-cli/capi.go
View file

@ -11,65 +11,71 @@ import (
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
const CAPIBaseURL string = "https://api.crowdsec.net/"
const CAPIURLPrefix = "v3"
const (
CAPIBaseURL = "https://api.crowdsec.net/"
CAPIURLPrefix = "v3"
)
func NewCapiCmd() *cobra.Command {
var cmdCapi = &cobra.Command{
type cliCapi struct{}
func NewCLICapi() *cliCapi {
return &cliCapi{}
}
func (cli cliCapi) NewCommand() *cobra.Command {
var cmd = &cobra.Command{
Use: "capi [action]",
Short: "Manage interaction with Central API (CAPI)",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIServer(); err != nil {
return fmt.Errorf("local API is disabled, please run this command on the local API machine: %w", err)
if err := require.LAPI(csConfig); err != nil {
return err
}
if csConfig.DisableAPI {
return nil
}
if csConfig.API.Server.OnlineClient == nil {
log.Fatalf("no configuration for Central API in '%s'", *csConfig.FilePath)
if err := require.CAPI(csConfig); err != nil {
return err
}
return nil
},
}
cmdCapi.AddCommand(NewCapiRegisterCmd())
cmdCapi.AddCommand(NewCapiStatusCmd())
cmd.AddCommand(cli.NewRegisterCmd())
cmd.AddCommand(cli.NewStatusCmd())
return cmdCapi
return cmd
}
func NewCapiRegisterCmd() *cobra.Command {
func (cli cliCapi) NewRegisterCmd() *cobra.Command {
var capiUserPrefix string
var outputFile string
var cmdCapiRegister = &cobra.Command{
var cmd = &cobra.Command{
Use: "register",
Short: "Register to Central API (CAPI)",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
var err error
capiUser, err := generateID(capiUserPrefix)
if err != nil {
log.Fatalf("unable to generate machine id: %s", err)
return fmt.Errorf("unable to generate machine id: %s", err)
}
password := strfmt.Password(generatePassword(passwordLength))
apiurl, err := url.Parse(types.CAPIBaseURL)
if err != nil {
log.Fatalf("unable to parse api url %s : %s", types.CAPIBaseURL, err)
return fmt.Errorf("unable to parse api url %s: %w", types.CAPIBaseURL, err)
}
_, err = apiclient.RegisterClient(&apiclient.Config{
MachineID: capiUser,
@ -80,7 +86,7 @@ func NewCapiRegisterCmd() *cobra.Command {
}, nil)
if err != nil {
log.Fatalf("api client register ('%s'): %s", types.CAPIBaseURL, err)
return fmt.Errorf("api client register ('%s'): %w", types.CAPIBaseURL, err)
}
log.Printf("Successfully registered to Central API (CAPI)")
@ -98,94 +104,94 @@ func NewCapiRegisterCmd() *cobra.Command {
Password: password.String(),
URL: types.CAPIBaseURL,
}
if fflag.PapiClient.IsEnabled() {
apiCfg.PapiURL = types.PAPIBaseURL
}
apiConfigDump, err := yaml.Marshal(apiCfg)
if err != nil {
log.Fatalf("unable to marshal api credentials: %s", err)
return fmt.Errorf("unable to marshal api credentials: %w", err)
}
if dumpFile != "" {
err = os.WriteFile(dumpFile, apiConfigDump, 0600)
err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
if err != nil {
log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
return fmt.Errorf("write api credentials in '%s' failed: %w", dumpFile, err)
}
log.Printf("Central API credentials dumped to '%s'", dumpFile)
log.Printf("Central API credentials written to '%s'", dumpFile)
} else {
fmt.Printf("%s\n", string(apiConfigDump))
}
log.Warning(ReloadMessage())
return nil
},
}
cmdCapiRegister.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
cmdCapiRegister.Flags().StringVar(&capiUserPrefix, "schmilblick", "", "set a schmilblick (use in tests only)")
if err := cmdCapiRegister.Flags().MarkHidden("schmilblick"); err != nil {
cmd.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")
cmd.Flags().StringVar(&capiUserPrefix, "schmilblick", "", "set a schmilblick (use in tests only)")
if err := cmd.Flags().MarkHidden("schmilblick"); err != nil {
log.Fatalf("failed to hide flag: %s", err)
}
return cmdCapiRegister
return cmd
}
func NewCapiStatusCmd() *cobra.Command {
var cmdCapiStatus = &cobra.Command{
func (cli cliCapi) NewStatusCmd() *cobra.Command {
var cmd = &cobra.Command{
Use: "status",
Short: "Check status with the Central API (CAPI)",
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
var err error
if csConfig.API.Server == nil {
log.Fatal("There is no configuration on 'api.server:'")
}
RunE: func(cmd *cobra.Command, args []string) error {
if csConfig.API.Server.OnlineClient == nil {
log.Fatalf("Please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
return fmt.Errorf("please provide credentials for the Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
}
if csConfig.API.Server.OnlineClient.Credentials == nil {
log.Fatalf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
return fmt.Errorf("no credentials for Central API (CAPI) in '%s'", csConfig.API.Server.OnlineClient.CredentialsFilePath)
}
password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
apiurl, err := url.Parse(csConfig.API.Server.OnlineClient.Credentials.URL)
if err != nil {
log.Fatalf("parsing api url ('%s'): %s", csConfig.API.Server.OnlineClient.Credentials.URL, err)
return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err)
}
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to load hub index : %s", err)
}
scenarios, err := cwhub.GetInstalledScenariosAsString()
hub, err := require.Hub(csConfig, nil)
if err != nil {
log.Fatalf("failed to get scenarios : %s", err)
return err
}
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("failed to get scenarios: %w", err)
}
if len(scenarios) == 0 {
log.Fatalf("no scenarios installed, abort")
return fmt.Errorf("no scenarios installed, abort")
}
Client, err = apiclient.NewDefaultClient(apiurl, CAPIURLPrefix, fmt.Sprintf("crowdsec/%s", version.String()), nil)
if err != nil {
log.Fatalf("init default client: %s", err)
return fmt.Errorf("init default client: %w", err)
}
t := models.WatcherAuthRequest{
MachineID: &csConfig.API.Server.OnlineClient.Credentials.Login,
Password: &password,
Scenarios: scenarios,
}
log.Infof("Loaded credentials from %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
log.Infof("Trying to authenticate with username %s on %s", csConfig.API.Server.OnlineClient.Credentials.Login, apiurl)
_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
if err != nil {
log.Fatalf("Failed to authenticate to Central API (CAPI) : %s", err)
return fmt.Errorf("failed to authenticate to Central API (CAPI): %w", err)
}
log.Infof("You can successfully interact with Central API (CAPI)")
return nil
},
}
return cmdCapiStatus
return cmd
}

183
cmd/crowdsec-cli/collections.go
View file

@ -1,183 +0,0 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCollectionsCmd() *cobra.Command {
var cmdCollections = &cobra.Command{
Use: "collections [action]",
Short: "Manage collections from hub",
Long: `Install/Remove/Upgrade/Inspect collections from the CrowdSec Hub.`,
/*TBD fix help*/
Args: cobra.MinimumNArgs(1),
Aliases: []string{"collection"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if csConfig.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
var ignoreError bool
var cmdCollectionsInstall = &cobra.Command{
Use: "install collection",
Short: "Install given collection(s)",
Long: `Fetch and install given collection(s) from hub`,
Example: `cscli collections install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.COLLECTIONS, args, toComplete)
},
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
for _, name := range args {
t := cwhub.GetItem(cwhub.COLLECTIONS, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.COLLECTIONS, name)
Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
if !ignoreError {
log.Fatalf("Error while installing '%s': %s", name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
}
}
},
}
cmdCollectionsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdCollectionsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdCollectionsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple collections")
cmdCollections.AddCommand(cmdCollectionsInstall)
var cmdCollectionsRemove = &cobra.Command{
Use: "remove collection",
Short: "Remove given collection(s)",
Long: `Remove given collection(s) from hub`,
Example: `cscli collections remove crowdsec/xxx crowdsec/xyz`,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
if all {
cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, forceAction)
return
}
if len(args) == 0 {
log.Fatal("Specify at least one collection to remove or '--all' flag.")
}
for _, name := range args {
if !forceAction {
item := cwhub.GetItem(cwhub.COLLECTIONS, name)
if item == nil {
log.Fatalf("unable to retrieve: %s\n", name)
}
if len(item.BelongsToCollections) > 0 {
log.Warningf("%s belongs to other collections :\n%s\n", name, item.BelongsToCollections)
log.Printf("Run 'sudo cscli collections remove %s --force' if you want to force remove this sub collection\n", name)
continue
}
}
cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, forceAction)
}
},
}
cmdCollectionsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdCollectionsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdCollectionsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the collections")
cmdCollections.AddCommand(cmdCollectionsRemove)
var cmdCollectionsUpgrade = &cobra.Command{
Use: "upgrade collection",
Short: "Upgrade given collection(s)",
Long: `Fetch and upgrade given collection(s) from hub`,
Example: `cscli collections upgrade crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
} else {
if len(args) == 0 {
log.Fatalf("no target collection to upgrade")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, forceAction)
}
}
},
}
cmdCollectionsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the collections")
cmdCollectionsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
cmdCollections.AddCommand(cmdCollectionsUpgrade)
var cmdCollectionsInspect = &cobra.Command{
Use: "inspect collection",
Short: "Inspect given collection",
Long: `Inspect given collection`,
Example: `cscli collections inspect crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.COLLECTIONS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
for _, name := range args {
InspectItem(name, cwhub.COLLECTIONS)
}
},
}
cmdCollectionsInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
cmdCollections.AddCommand(cmdCollectionsInspect)
var cmdCollectionsList = &cobra.Command{
Use: "list collection [-a]",
Short: "List all collections",
Long: `List all collections`,
Example: `cscli collections list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.COLLECTIONS}, args, false, true, all)
},
}
cmdCollectionsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
cmdCollections.AddCommand(cmdCollectionsList)
return cmdCollections
}

88
cmd/crowdsec-cli/config_backup.go
View file

@ -1,6 +1,7 @@
package main
import (
"encoding/json"
"fmt"
"os"
"path/filepath"
@ -9,9 +10,80 @@ import (
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
/* Backup crowdsec configurations to directory <dirPath> :
func backupHub(dirPath string) error {
var itemDirectory string
var upstreamParsers []string
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
for _, itemType := range cwhub.ItemTypes {
clog := log.WithFields(log.Fields{
"type": itemType,
})
itemMap := hub.GetItemMap(itemType)
if itemMap == nil {
clog.Infof("No %s to backup.", itemType)
continue
}
itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
if err = os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
}
upstreamParsers = []string{}
for k, v := range itemMap {
clog = clog.WithFields(log.Fields{
"file": v.Name,
})
if !v.State.Installed { //only backup installed ones
clog.Debugf("[%s] : not installed", k)
continue
}
//for the local/tainted ones, we back up the full file
if v.State.Tainted || v.State.IsLocal() || !v.State.UpToDate {
//we need to backup stages for parsers
if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS {
fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
if err = os.MkdirAll(fstagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
}
}
clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.State.Tainted, v.State.IsLocal(), v.State.UpToDate)
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
if err = CopyFile(v.State.LocalPath, tfile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.State.LocalPath, tfile, err)
}
clog.Infof("local/tainted saved %s to %s", v.State.LocalPath, tfile)
continue
}
clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.State.UpToDate)
clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.State.UpToDate)
upstreamParsers = append(upstreamParsers, v.Name)
}
//write the upstream items
upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
if err != nil {
return fmt.Errorf("failed marshaling upstream parsers : %s", err)
}
err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0o644)
if err != nil {
return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
}
clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
}
return nil
}
/*
Backup crowdsec configurations to directory <dirPath>:
- Main config (config.yaml)
- Profiles config (profiles.yaml)
@ -19,6 +91,7 @@ import (
- Backup of API credentials (local API and online API)
- List of scenarios, parsers, postoverflows and collections that are up-to-date
- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections
- Acquisition files (acquis.yaml, acquis.d/*.yaml)
*/
func backupConfigToDirectory(dirPath string) error {
var err error
@ -31,7 +104,7 @@ func backupConfigToDirectory(dirPath string) error {
/*if parent directory doesn't exist, bail out. create final dir with Mkdir*/
parentDir := filepath.Dir(dirPath)
if _, err := os.Stat(parentDir); err != nil {
if _, err = os.Stat(parentDir); err != nil {
return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err)
}
@ -120,7 +193,7 @@ func backupConfigToDirectory(dirPath string) error {
log.Infof("Saved profiles to %s", backupProfiles)
}
if err = BackupHub(dirPath); err != nil {
if err = backupHub(dirPath); err != nil {
return fmt.Errorf("failed to backup hub config: %s", err)
}
@ -128,15 +201,6 @@ func backupConfigToDirectory(dirPath string) error {
}
func runConfigBackup(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
return err
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
return fmt.Errorf("failed to get Hub index: %w", err)
}
if err := backupConfigToDirectory(args[0]); err != nil {
return fmt.Errorf("failed to backup config: %w", err)
}

11
cmd/crowdsec-cli/config_feature_flags.go
View file

@ -2,10 +2,12 @@ package main
import (
"fmt"
"path/filepath"
"github.com/fatih/color"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
)
@ -87,7 +89,14 @@ func runConfigFeatureFlags(cmd *cobra.Command, args []string) error {
fmt.Println("To enable a feature you can: ")
fmt.Println(" - set the environment variable CROWDSEC_FEATURE_<uppercase_feature_name> to true")
fmt.Printf(" - add the line '- <feature_name>' to the file %s/feature.yaml\n", csConfig.ConfigPaths.ConfigDir)
featurePath, err := filepath.Abs(csconfig.GetFeatureFilePath(ConfigFilePath))
if err != nil {
// we already read the file, shouldn't happen
return err
}
fmt.Printf(" - add the line '- <feature_name>' to the file %s\n", featurePath)
fmt.Println()
if len(enabled) == 0 && len(disabled) == 0 {

106
cmd/crowdsec-cli/config_restore.go
View file

@ -11,6 +11,7 @@ import (
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
@ -20,7 +21,94 @@ type OldAPICfg struct {
Password string `json:"password"`
}
/* Restore crowdsec configurations to directory <dirPath> :
func restoreHub(dirPath string) error {
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
for _, itype := range cwhub.ItemTypes {
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
if _, err = os.Stat(itemDirectory); err != nil {
log.Infof("no %s in backup", itype)
continue
}
/*restore the upstream items*/
upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
file, err := os.ReadFile(upstreamListFN)
if err != nil {
return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
}
var upstreamList []string
err = json.Unmarshal(file, &upstreamList)
if err != nil {
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
}
for _, toinstall := range upstreamList {
item := hub.GetItem(itype, toinstall)
if item == nil {
log.Errorf("Item %s/%s not found in hub", itype, toinstall)
continue
}
err := item.Install(false, false)
if err != nil {
log.Errorf("Error while installing %s : %s", toinstall, err)
}
}
/*restore the local and tainted items*/
files, err := os.ReadDir(itemDirectory)
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
}
for _, file := range files {
//this was the upstream data
if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
continue
}
if itype == cwhub.PARSERS || itype == cwhub.POSTOVERFLOWS {
//we expect a stage here
if !file.IsDir() {
continue
}
stage := file.Name()
stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
}
/*find items*/
ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
}
//finally copy item
for _, tfile := range ifiles {
log.Infof("Going to restore local/tainted [%s]", tfile.Name())
sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
} else {
log.Infof("Going to restore local/tainted [%s]", file.Name())
sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
}
}
return nil
}
/*
Restore crowdsec configurations to directory <dirPath>:
- Main config (config.yaml)
- Profiles config (profiles.yaml)
@ -28,6 +116,7 @@ type OldAPICfg struct {
- Backup of API credentials (local API and online API)
- List of scenarios, parsers, postoverflows and collections that are up-to-date
- Tainted/local/out-of-date scenarios, parsers, postoverflows and collections
- Acquisition files (acquis.yaml, acquis.d/*.yaml)
*/
func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
var err error
@ -94,7 +183,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
if csConfig.API.Server.OnlineClient != nil && csConfig.API.Server.OnlineClient.CredentialsFilePath != "" {
apiConfigDumpFile = csConfig.API.Server.OnlineClient.CredentialsFilePath
}
err = os.WriteFile(apiConfigDumpFile, apiConfigDump, 0o644)
err = os.WriteFile(apiConfigDumpFile, apiConfigDump, 0o600)
if err != nil {
return fmt.Errorf("write api credentials in '%s' failed: %s", apiConfigDumpFile, err)
}
@ -111,7 +200,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
/*if there is a acquisition dir, restore its content*/
if csConfig.Crowdsec.AcquisitionDirPath != "" {
if err = os.Mkdir(csConfig.Crowdsec.AcquisitionDirPath, 0o700); err != nil {
if err = os.MkdirAll(csConfig.Crowdsec.AcquisitionDirPath, 0o700); err != nil {
return fmt.Errorf("error while creating %s : %s", csConfig.Crowdsec.AcquisitionDirPath, err)
}
}
@ -166,7 +255,7 @@ func restoreConfigFromDirectory(dirPath string, oldBackup bool) error {
}
}
if err = RestoreHub(dirPath); err != nil {
if err = restoreHub(dirPath); err != nil {
return fmt.Errorf("failed to restore hub config : %s", err)
}
@ -181,15 +270,6 @@ func runConfigRestore(cmd *cobra.Command, args []string) error {
return err
}
if err := csConfig.LoadHub(); err != nil {
return err
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
return fmt.Errorf("failed to get Hub index: %w", err)
}
if err := restoreConfigFromDirectory(args[0], oldBackup); err != nil {
return fmt.Errorf("failed to restore config from %s: %w", args[0], err)
}

31
cmd/crowdsec-cli/config_show.go
View file

@ -7,6 +7,7 @@ import (
"text/template"
"github.com/antonmedv/expr"
"github.com/sanity-io/litter"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
@ -35,13 +36,13 @@ func showConfigKey(key string) error {
switch csConfig.Cscli.Output {
case "human", "raw":
// Don't use litter for strings, it adds quotes
// that we didn't have before
switch output.(type) {
case string:
fmt.Printf("%s\n", output)
case int:
fmt.Printf("%d\n", output)
fmt.Println(output)
default:
fmt.Printf("%v\n", output)
litter.Dump(output)
}
case "json":
data, err := json.MarshalIndent(output, "", " ")
@ -57,7 +58,6 @@ func showConfigKey(key string) error {
var configShowTemplate = `Global:
{{- if .ConfigPaths }}
- Configuration Folder : {{.ConfigPaths.ConfigDir}}
- Configuration Folder : {{.ConfigPaths.ConfigDir}}
- Data Folder : {{.ConfigPaths.DataDir}}
- Hub Folder : {{.ConfigPaths.HubDir}}
@ -71,7 +71,7 @@ var configShowTemplate = `Global:
{{- end }}
{{- if .Crowdsec }}
Crowdsec:
Crowdsec{{if and .Crowdsec.Enable (not (ValueBool .Crowdsec.Enable))}} (disabled){{end}}:
- Acquisition File : {{.Crowdsec.AcquisitionFilePath}}
- Parsers routines : {{.Crowdsec.ParserRoutinesCount}}
{{- if .Crowdsec.AcquisitionDirPath }}
@ -83,7 +83,6 @@ Crowdsec:
cscli:
- Output : {{.Cscli.Output}}
- Hub Branch : {{.Cscli.HubBranch}}
- Hub Folder : {{.Cscli.HubDir}}
{{- end }}
{{- if .API }}
@ -97,7 +96,7 @@ API Client:
{{- end }}
{{- if .API.Server }}
Local API Server:
Local API Server{{if and .API.Server.Enable (not (ValueBool .API.Server.Enable))}} (disabled){{end}}:
- Listen URL : {{.API.Server.ListenURI}}
- Profile File : {{.API.Server.ProfilesPath}}
@ -164,6 +163,12 @@ Central API:
- User : {{.DbConfig.User}}
- DB Name : {{.DbConfig.DbName}}
{{- end }}
{{- if .DbConfig.MaxOpenConns }}
- Max Open Conns : {{.DbConfig.MaxOpenConns}}
{{- end }}
{{- if ne .DbConfig.DecisionBulkSize 0 }}
- Decision Bulk Size : {{.DbConfig.DecisionBulkSize}}
{{- end }}
{{- if .DbConfig.Flush }}
{{- if .DbConfig.Flush.MaxAge }}
- Flush age : {{.DbConfig.Flush.MaxAge}}
@ -194,7 +199,15 @@ func runConfigShow(cmd *cobra.Command, args []string) error {
switch csConfig.Cscli.Output {
case "human":
tmp, err := template.New("config").Parse(configShowTemplate)
// The tests on .Enable look funny because the option has a true default which has
// not been set yet (we don't really load the LAPI) and go templates don't dereference
// pointers in boolean tests. Prefix notation is the cherry on top.
funcs := template.FuncMap{
// can't use generics here
"ValueBool": func(b *bool) bool { return b!=nil && *b },
}
tmp, err := template.New("config").Funcs(funcs).Parse(configShowTemplate)
if err != nil {
return err
}

169
cmd/crowdsec-cli/console.go
View file

@ -4,9 +4,7 @@ import (
"context"
"encoding/csv"
"encoding/json"
"errors"
"fmt"
"io/fs"
"net/url"
"os"
@ -16,14 +14,15 @@ import (
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/go-cs-lib/pkg/ptr"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
func NewConsoleCmd() *cobra.Command {
@ -33,24 +32,14 @@ func NewConsoleCmd() *cobra.Command {
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
var fdErr *fs.PathError
if errors.As(err, &fdErr) {
log.Fatalf("Unable to load Local API : %s", fdErr)
}
if err != nil {
log.Fatalf("Unable to load required Local API Configuration : %s", err)
}
log.Fatal("Local API is disabled, please run this command on the local API machine")
if err := require.LAPI(csConfig); err != nil {
return err
}
if csConfig.DisableAPI {
log.Fatal("Local API is disabled, please run this command on the local API machine")
if err := require.CAPI(csConfig); err != nil {
return err
}
if csConfig.API.Server.OnlineClient == nil {
log.Fatalf("No configuration for Central API (CAPI) in '%s'", *csConfig.FilePath)
}
if csConfig.API.Server.OnlineClient.Credentials == nil {
log.Fatal("You must configure Central API (CAPI) with `cscli capi register` before accessing console features.")
if err := require.CAPIRegistered(csConfig); err != nil {
return err
}
return nil
},
@ -74,25 +63,21 @@ After running this command your will need to validate the enrollment in the weba
`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
password := strfmt.Password(csConfig.API.Server.OnlineClient.Credentials.Password)
apiURL, err := url.Parse(csConfig.API.Server.OnlineClient.Credentials.URL)
if err != nil {
log.Fatalf("Could not parse CAPI URL : %s", err)
return fmt.Errorf("could not parse CAPI URL: %s", err)
}
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to load hub index : %s", err)
log.Info("Run 'sudo cscli hub update' to get the hub index")
}
scenarios, err := cwhub.GetInstalledScenariosAsString()
hub, err := require.Hub(csConfig, nil)
if err != nil {
log.Fatalf("failed to get scenarios : %s", err)
return err
}
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("failed to get installed scenarios: %s", err)
}
if len(scenarios) == 0 {
@ -109,20 +94,21 @@ After running this command your will need to validate the enrollment in the weba
})
resp, err := c.Auth.EnrollWatcher(context.Background(), args[0], name, tags, overwrite)
if err != nil {
log.Fatalf("Could not enroll instance: %s", err)
return fmt.Errorf("could not enroll instance: %s", err)
}
if resp.Response.StatusCode == 200 && !overwrite {
log.Warning("Instance already enrolled. You can use '--overwrite' to force enroll")
return
return nil
}
SetConsoleOpts(csconfig.CONSOLE_CONFIGS, true)
if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
log.Fatalf("failed writing console config : %s", err)
if err := SetConsoleOpts([]string{csconfig.SEND_MANUAL_SCENARIOS, csconfig.SEND_TAINTED_SCENARIOS}, true); err != nil {
return err
}
log.Infof("Enabled tainted&manual alerts sharing, see 'cscli console status'.")
log.Infof("Watcher successfully enrolled. Visit https://app.crowdsec.net to accept it.")
log.Infof("Please restart crowdsec after accepting the enrollment.")
log.Info("Enabled tainted&manual alerts sharing, see 'cscli console status'.")
log.Info("Watcher successfully enrolled. Visit https://app.crowdsec.net to accept it.")
log.Info("Please restart crowdsec after accepting the enrollment.")
return nil
},
}
cmdEnroll.Flags().StringVarP(&name, "name", "n", "", "Name to display in the console")
@ -140,21 +126,23 @@ After running this command your will need to validate the enrollment in the weba
Enable given information push to the central API. Allows to empower the console`,
ValidArgs: csconfig.CONSOLE_CONFIGS,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
if enableAll {
SetConsoleOpts(csconfig.CONSOLE_CONFIGS, true)
if err := SetConsoleOpts(csconfig.CONSOLE_CONFIGS, true); err != nil {
return err
}
log.Infof("All features have been enabled successfully")
} else {
if len(args) == 0 {
log.Fatalf("You must specify at least one feature to enable")
return fmt.Errorf("you must specify at least one feature to enable")
}
if err := SetConsoleOpts(args, true); err != nil {
return err
}
SetConsoleOpts(args, true)
log.Infof("%v have been enabled", args)
}
if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
log.Fatalf("failed writing console config : %s", err)
}
log.Infof(ReloadMessage())
return nil
},
}
cmdEnable.Flags().BoolVarP(&enableAll, "all", "a", false, "Enable all console options")
@ -167,49 +155,55 @@ Enable given information push to the central API. Allows to empower the console`
Long: `
Disable given information push to the central API.`,
ValidArgs: csconfig.CONSOLE_CONFIGS,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if disableAll {
SetConsoleOpts(csconfig.CONSOLE_CONFIGS, false)
} else {
SetConsoleOpts(args, false)
}
if err := csConfig.API.Server.DumpConsoleConfig(); err != nil {
log.Fatalf("failed writing console config : %s", err)
}
RunE: func(cmd *cobra.Command, args []string) error {
if disableAll {
if err := SetConsoleOpts(csconfig.CONSOLE_CONFIGS, false); err != nil {
return err
}
log.Infof("All features have been disabled")
} else {
if err := SetConsoleOpts(args, false); err != nil {
return err
}
log.Infof("%v have been disabled", args)
}
log.Infof(ReloadMessage())
return nil
},
}
cmdDisable.Flags().BoolVarP(&disableAll, "all", "a", false, "Disable all console options")
cmdConsole.AddCommand(cmdDisable)
cmdConsoleStatus := &cobra.Command{
Use: "status [option]",
Short: "Shows status of one or all console options",
Use: "status",
Short: "Shows status of the console options",
Example: `sudo cscli console status`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
switch csConfig.Cscli.Output {
case "human":
cmdConsoleStatusTable(color.Output, *csConfig)
case "json":
data, err := json.MarshalIndent(csConfig.API.Server.ConsoleConfig, "", " ")
if err != nil {
log.Fatalf("failed to marshal configuration: %s", err)
c := csConfig.API.Server.ConsoleConfig
out := map[string](*bool){
csconfig.SEND_MANUAL_SCENARIOS: c.ShareManualDecisions,
csconfig.SEND_CUSTOM_SCENARIOS: c.ShareCustomScenarios,
csconfig.SEND_TAINTED_SCENARIOS: c.ShareTaintedScenarios,
csconfig.SEND_CONTEXT: c.ShareContext,
csconfig.CONSOLE_MANAGEMENT: c.ConsoleManagement,
}
fmt.Printf("%s\n", string(data))
data, err := json.MarshalIndent(out, "", " ")
if err != nil {
return fmt.Errorf("failed to marshal configuration: %s", err)
}
fmt.Println(string(data))
case "raw":
csvwriter := csv.NewWriter(os.Stdout)
err := csvwriter.Write([]string{"option", "enabled"})
if err != nil {
log.Fatal(err)
return err
}
rows := [][]string{
@ -222,11 +216,12 @@ Disable given information push to the central API.`,
for _, row := range rows {
err = csvwriter.Write(row)
if err != nil {
log.Fatal(err)
return err
}
}
csvwriter.Flush()
}
return nil
},
}
cmdConsole.AddCommand(cmdConsoleStatus)
@ -234,13 +229,28 @@ Disable given information push to the central API.`,
return cmdConsole
}
func SetConsoleOpts(args []string, wanted bool) {
func dumpConsoleConfig(c *csconfig.LocalApiServerCfg) error {
out, err := yaml.Marshal(c.ConsoleConfig)
if err != nil {
return fmt.Errorf("while marshaling ConsoleConfig (for %s): %w", c.ConsoleConfigPath, err)
}
if c.ConsoleConfigPath == "" {
c.ConsoleConfigPath = csconfig.DefaultConsoleConfigFilePath
log.Debugf("Empty console_path, defaulting to %s", c.ConsoleConfigPath)
}
if err := os.WriteFile(c.ConsoleConfigPath, out, 0o600); err != nil {
return fmt.Errorf("while dumping console config to %s: %w", c.ConsoleConfigPath, err)
}
return nil
}
func SetConsoleOpts(args []string, wanted bool) error {
for _, arg := range args {
switch arg {
case csconfig.CONSOLE_MANAGEMENT:
if !fflag.PapiClient.IsEnabled() {
continue
}
/*for each flag check if it's already set before setting it*/
if csConfig.API.Server.ConsoleConfig.ConsoleManagement != nil {
if *csConfig.API.Server.ConsoleConfig.ConsoleManagement == wanted {
@ -265,12 +275,12 @@ func SetConsoleOpts(args []string, wanted bool) {
if changed {
fileContent, err := yaml.Marshal(csConfig.API.Server.OnlineClient.Credentials)
if err != nil {
log.Fatalf("Cannot marshal credentials: %s", err)
return fmt.Errorf("cannot marshal credentials: %s", err)
}
log.Infof("Updating credentials file: %s", csConfig.API.Server.OnlineClient.CredentialsFilePath)
err = os.WriteFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, fileContent, 0600)
err = os.WriteFile(csConfig.API.Server.OnlineClient.CredentialsFilePath, fileContent, 0o600)
if err != nil {
log.Fatalf("Cannot write credentials file: %s", err)
return fmt.Errorf("cannot write credentials file: %s", err)
}
}
}
@ -327,8 +337,13 @@ func SetConsoleOpts(args []string, wanted bool) {
csConfig.API.Server.ConsoleConfig.ShareContext = ptr.Of(wanted)
}
default:
log.Fatalf("unknown flag %s", arg)
return fmt.Errorf("unknown flag %s", arg)
}
}
if err := dumpConsoleConfig(csConfig.API.Server); err != nil {
return fmt.Errorf("failed writing console config: %s", err)
}
return nil
}

373
cmd/crowdsec-cli/dashboard.go
View file

@ -1,7 +1,8 @@
//go:build linux
package main
import (
"errors"
"fmt"
"math"
"os"
@ -10,6 +11,7 @@ import (
"path/filepath"
"strconv"
"strings"
"syscall"
"unicode"
"github.com/AlecAivazis/survey/v2"
@ -18,6 +20,8 @@ import (
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/metabase"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
var (
@ -27,6 +31,7 @@ var (
metabaseConfigPath string
metabaseConfigFolder = "metabase/"
metabaseConfigFile = "metabase.yaml"
metabaseImage = "metabase/metabase:v0.46.6.1"
/**/
metabaseListenAddress = "127.0.0.1"
metabaseListenPort = "3000"
@ -35,12 +40,18 @@ var (
forceYes bool
/*informations needed to setup a random password on user's behalf*/
// information needed to set up a random password on user's behalf
)
func NewDashboardCmd() *cobra.Command {
type cliDashboard struct{}
func NewCLIDashboard() *cliDashboard {
return &cliDashboard{}
}
func (cli cliDashboard) NewCommand() *cobra.Command {
/* ---- UPDATE COMMAND */
var cmdDashboard = &cobra.Command{
cmd := &cobra.Command{
Use: "dashboard [command]",
Short: "Manage your metabase dashboard container [requires local API]",
Long: `Install/Start/Stop/Remove a metabase container exposing dashboard and metrics.
@ -54,23 +65,23 @@ cscli dashboard start
cscli dashboard stop
cscli dashboard remove
`,
PersistentPreRun: func(cmd *cobra.Command, args []string) {
if err := metabase.TestAvailability(); err != nil {
log.Fatalf("%s", err)
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.LAPI(csConfig); err != nil {
return err
}
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
log.Fatal("Local API is disabled, please run this command on the local API machine")
if err := metabase.TestAvailability(); err != nil {
return err
}
metabaseConfigFolderPath := filepath.Join(csConfig.ConfigPaths.ConfigDir, metabaseConfigFolder)
metabaseConfigPath = filepath.Join(metabaseConfigFolderPath, metabaseConfigFile)
if err := os.MkdirAll(metabaseConfigFolderPath, os.ModePerm); err != nil {
log.Fatal(err)
return err
}
if err := csConfig.LoadDBConfig(); err != nil {
log.Errorf("This command requires direct database access (must be run on the local API machine)")
log.Fatal(err)
if err := require.DB(csConfig); err != nil {
return err
}
/*
@ -84,23 +95,23 @@ cscli dashboard remove
metabaseContainerID = oldContainerID
}
}
return nil
},
}
cmdDashboard.AddCommand(NewDashboardSetupCmd())
cmdDashboard.AddCommand(NewDashboardStartCmd())
cmdDashboard.AddCommand(NewDashboardStopCmd())
cmdDashboard.AddCommand(NewDashboardShowPasswordCmd())
cmdDashboard.AddCommand(NewDashboardRemoveCmd())
cmd.AddCommand(cli.NewSetupCmd())
cmd.AddCommand(cli.NewStartCmd())
cmd.AddCommand(cli.NewStopCmd())
cmd.AddCommand(cli.NewShowPasswordCmd())
cmd.AddCommand(cli.NewRemoveCmd())
return cmdDashboard
return cmd
}
func NewDashboardSetupCmd() *cobra.Command {
func (cli cliDashboard) NewSetupCmd() *cobra.Command {
var force bool
var cmdDashSetup = &cobra.Command{
cmd := &cobra.Command{
Use: "setup",
Short: "Setup a metabase container.",
Long: `Perform a metabase docker setup, download standard dashboards, create a fresh user and start the container`,
@ -111,7 +122,7 @@ cscli dashboard setup
cscli dashboard setup --listen 0.0.0.0
cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
`,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
if metabaseDbPath == "" {
metabaseDbPath = csConfig.ConfigPaths.DataDir
}
@ -123,70 +134,26 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
isValid = passwordIsValid(metabasePassword)
}
}
var answer bool
if valid, err := checkSystemMemory(); err == nil && !valid {
if !forceYes {
prompt := &survey.Confirm{
Message: "Metabase requires 1-2GB of RAM, your system is below this requirement continue ?",
Default: true,
}
if err := survey.AskOne(prompt, &answer); err != nil {
log.Warnf("unable to ask about RAM check: %s", err)
}
if !answer {
log.Fatal("Unable to continue due to RAM requirement")
}
} else {
log.Warnf("Metabase requires 1-2GB of RAM, your system is below this requirement")
}
if err := checkSystemMemory(&forceYes); err != nil {
return err
}
groupExist := false
dockerGroup, err := user.LookupGroup(crowdsecGroup)
if err == nil {
groupExist = true
warnIfNotLoopback(metabaseListenAddress)
if err := disclaimer(&forceYes); err != nil {
return err
}
if !forceYes && !groupExist {
prompt := &survey.Confirm{
Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup),
Default: true,
}
if err := survey.AskOne(prompt, &answer); err != nil {
log.Fatalf("unable to ask to force: %s", err)
}
}
if !answer && !forceYes && !groupExist {
log.Fatalf("unable to continue without creating '%s' group", crowdsecGroup)
}
if !groupExist {
groupAddCmd, err := exec.LookPath("groupadd")
if err != nil {
log.Fatalf("unable to find 'groupadd' command, can't continue")
}
groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}}
if err := groupAdd.Run(); err != nil {
log.Fatalf("unable to add group '%s': %s", dockerGroup, err)
}
dockerGroup, err = user.LookupGroup(crowdsecGroup)
if err != nil {
log.Fatalf("unable to lookup '%s' group: %+v", dockerGroup, err)
}
}
intID, err := strconv.Atoi(dockerGroup.Gid)
dockerGroup, err := checkGroups(&forceYes)
if err != nil {
log.Fatalf("unable to convert group ID to int: %s", err)
return err
}
if err := os.Chown(csConfig.DbConfig.DbPath, 0, intID); err != nil {
log.Fatalf("unable to chown sqlite db file '%s': %s", csConfig.DbConfig.DbPath, err)
if err = chownDatabase(dockerGroup.Gid); err != nil {
return err
}
mb, err := metabase.SetupMetabase(csConfig.API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDbPath, dockerGroup.Gid, metabaseContainerID)
mb, err := metabase.SetupMetabase(csConfig.API.Server.DbConfig, metabaseListenAddress, metabaseListenPort, metabaseUser, metabasePassword, metabaseDbPath, dockerGroup.Gid, metabaseContainerID, metabaseImage)
if err != nil {
log.Fatal(err)
return err
}
if err := mb.DumpConfig(metabaseConfigPath); err != nil {
log.Fatal(err)
return err
}
log.Infof("Metabase is ready")
@ -194,79 +161,87 @@ cscli dashboard setup -l 0.0.0.0 -p 443 --password <password>
fmt.Printf("\tURL : '%s'\n", mb.Config.ListenURL)
fmt.Printf("\tusername : '%s'\n", mb.Config.Username)
fmt.Printf("\tpassword : '%s'\n", mb.Config.Password)
return nil
},
}
cmdDashSetup.Flags().BoolVarP(&force, "force", "f", false, "Force setup : override existing files.")
cmdDashSetup.Flags().StringVarP(&metabaseDbPath, "dir", "d", "", "Shared directory with metabase container.")
cmdDashSetup.Flags().StringVarP(&metabaseListenAddress, "listen", "l", metabaseListenAddress, "Listen address of container")
cmdDashSetup.Flags().StringVarP(&metabaseListenPort, "port", "p", metabaseListenPort, "Listen port of container")
cmdDashSetup.Flags().BoolVarP(&forceYes, "yes", "y", false, "force yes")
//cmdDashSetup.Flags().StringVarP(&metabaseUser, "user", "u", "crowdsec@crowdsec.net", "metabase user")
cmdDashSetup.Flags().StringVar(&metabasePassword, "password", "", "metabase password")
cmd.Flags().BoolVarP(&force, "force", "f", false, "Force setup : override existing files")
cmd.Flags().StringVarP(&metabaseDbPath, "dir", "d", "", "Shared directory with metabase container")
cmd.Flags().StringVarP(&metabaseListenAddress, "listen", "l", metabaseListenAddress, "Listen address of container")
cmd.Flags().StringVar(&metabaseImage, "metabase-image", metabaseImage, "Metabase image to use")
cmd.Flags().StringVarP(&metabaseListenPort, "port", "p", metabaseListenPort, "Listen port of container")
cmd.Flags().BoolVarP(&forceYes, "yes", "y", false, "force yes")
//cmd.Flags().StringVarP(&metabaseUser, "user", "u", "crowdsec@crowdsec.net", "metabase user")
cmd.Flags().StringVar(&metabasePassword, "password", "", "metabase password")
return cmdDashSetup
return cmd
}
func NewDashboardStartCmd() *cobra.Command {
var cmdDashStart = &cobra.Command{
func (cli cliDashboard) NewStartCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "start",
Short: "Start the metabase container.",
Long: `Stats the metabase container using docker.`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
mb, err := metabase.NewMetabase(metabaseConfigPath, metabaseContainerID)
if err != nil {
log.Fatal(err)
return err
}
warnIfNotLoopback(mb.Config.ListenAddr)
if err := disclaimer(&forceYes); err != nil {
return err
}
if err := mb.Container.Start(); err != nil {
log.Fatalf("Failed to start metabase container : %s", err)
return fmt.Errorf("failed to start metabase container : %s", err)
}
log.Infof("Started metabase")
log.Infof("url : http://%s:%s", metabaseListenAddress, metabaseListenPort)
log.Infof("url : http://%s:%s", mb.Config.ListenAddr, mb.Config.ListenPort)
return nil
},
}
return cmdDashStart
cmd.Flags().BoolVarP(&forceYes, "yes", "y", false, "force yes")
return cmd
}
func NewDashboardStopCmd() *cobra.Command {
var cmdDashStop = &cobra.Command{
func (cli cliDashboard) NewStopCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "stop",
Short: "Stops the metabase container.",
Long: `Stops the metabase container using docker.`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
if err := metabase.StopContainer(metabaseContainerID); err != nil {
log.Fatalf("unable to stop container '%s': %s", metabaseContainerID, err)
return fmt.Errorf("unable to stop container '%s': %s", metabaseContainerID, err)
}
return nil
},
}
return cmdDashStop
return cmd
}
func NewDashboardShowPasswordCmd() *cobra.Command {
var cmdDashShowPassword = &cobra.Command{Use: "show-password",
func (cli cliDashboard) NewShowPasswordCmd() *cobra.Command {
cmd := &cobra.Command{Use: "show-password",
Short: "displays password of metabase.",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
m := metabase.Metabase{}
if err := m.LoadConfig(metabaseConfigPath); err != nil {
log.Fatal(err)
return err
}
log.Printf("'%s'", m.Config.Password)
return nil
},
}
return cmdDashShowPassword
return cmd
}
func NewDashboardRemoveCmd() *cobra.Command {
func (cli cliDashboard) NewRemoveCmd() *cobra.Command {
var force bool
var cmdDashRemove = &cobra.Command{
cmd := &cobra.Command{
Use: "remove",
Short: "removes the metabase container.",
Long: `removes the metabase container using docker.`,
@ -276,66 +251,74 @@ func NewDashboardRemoveCmd() *cobra.Command {
cscli dashboard remove
cscli dashboard remove --force
`,
Run: func(cmd *cobra.Command, args []string) {
answer := true
RunE: func(cmd *cobra.Command, args []string) error {
if !forceYes {
var answer bool
prompt := &survey.Confirm{
Message: "Do you really want to remove crowdsec dashboard? (all your changes will be lost)",
Default: true,
}
if err := survey.AskOne(prompt, &answer); err != nil {
log.Fatalf("unable to ask to force: %s", err)
return fmt.Errorf("unable to ask to force: %s", err)
}
if !answer {
return fmt.Errorf("user stated no to continue")
}
}
if answer {
if metabase.IsContainerExist(metabaseContainerID) {
log.Debugf("Stopping container %s", metabaseContainerID)
if err := metabase.StopContainer(metabaseContainerID); err != nil {
log.Warningf("unable to stop container '%s': %s", metabaseContainerID, err)
if metabase.IsContainerExist(metabaseContainerID) {
log.Debugf("Stopping container %s", metabaseContainerID)
if err := metabase.StopContainer(metabaseContainerID); err != nil {
log.Warningf("unable to stop container '%s': %s", metabaseContainerID, err)
}
dockerGroup, err := user.LookupGroup(crowdsecGroup)
if err == nil { // if group exist, remove it
groupDelCmd, err := exec.LookPath("groupdel")
if err != nil {
return fmt.Errorf("unable to find 'groupdel' command, can't continue")
}
dockerGroup, err := user.LookupGroup(crowdsecGroup)
if err == nil { // if group exist, remove it
groupDelCmd, err := exec.LookPath("groupdel")
if err != nil {
log.Fatalf("unable to find 'groupdel' command, can't continue")
}
groupDel := &exec.Cmd{Path: groupDelCmd, Args: []string{groupDelCmd, crowdsecGroup}}
if err := groupDel.Run(); err != nil {
log.Errorf("unable to delete group '%s': %s", dockerGroup, err)
}
groupDel := &exec.Cmd{Path: groupDelCmd, Args: []string{groupDelCmd, crowdsecGroup}}
if err := groupDel.Run(); err != nil {
log.Warnf("unable to delete group '%s': %s", dockerGroup, err)
}
log.Debugf("Removing container %s", metabaseContainerID)
if err := metabase.RemoveContainer(metabaseContainerID); err != nil {
log.Warningf("unable to remove container '%s': %s", metabaseContainerID, err)
}
log.Infof("container %s stopped & removed", metabaseContainerID)
}
log.Debugf("Removing metabase db %s", csConfig.ConfigPaths.DataDir)
if err := metabase.RemoveDatabase(csConfig.ConfigPaths.DataDir); err != nil {
log.Warningf("failed to remove metabase internal db : %s", err)
log.Debugf("Removing container %s", metabaseContainerID)
if err := metabase.RemoveContainer(metabaseContainerID); err != nil {
log.Warnf("unable to remove container '%s': %s", metabaseContainerID, err)
}
if force {
if err := metabase.RemoveImageContainer(); err != nil {
if !strings.Contains(err.Error(), "No such image") {
log.Fatalf("removing docker image: %s", err)
}
log.Infof("container %s stopped & removed", metabaseContainerID)
}
log.Debugf("Removing metabase db %s", csConfig.ConfigPaths.DataDir)
if err := metabase.RemoveDatabase(csConfig.ConfigPaths.DataDir); err != nil {
log.Warnf("failed to remove metabase internal db : %s", err)
}
if force {
m := metabase.Metabase{}
if err := m.LoadConfig(metabaseConfigPath); err != nil {
return err
}
if err := metabase.RemoveImageContainer(m.Config.Image); err != nil {
if !strings.Contains(err.Error(), "No such image") {
return fmt.Errorf("removing docker image: %s", err)
}
}
}
return nil
},
}
cmdDashRemove.Flags().BoolVarP(&force, "force", "f", false, "Remove also the metabase image")
cmdDashRemove.Flags().BoolVarP(&forceYes, "yes", "y", false, "force yes")
cmd.Flags().BoolVarP(&force, "force", "f", false, "Remove also the metabase image")
cmd.Flags().BoolVarP(&forceYes, "yes", "y", false, "force yes")
return cmdDashRemove
return cmd
}
func passwordIsValid(password string) bool {
hasDigit := false
for _, j := range password {
if unicode.IsDigit(j) {
hasDigit = true
break
}
}
@ -344,16 +327,108 @@ func passwordIsValid(password string) bool {
return false
}
return true
}
func checkSystemMemory() (bool, error) {
func checkSystemMemory(forceYes *bool) error {
totMem := memory.TotalMemory()
if totMem == 0 {
return true, errors.New("Unable to get system total memory")
if totMem >= uint64(math.Pow(2, 30)) {
return nil
}
if uint64(math.Pow(2, 30)) >= totMem {
return false, nil
if !*forceYes {
var answer bool
prompt := &survey.Confirm{
Message: "Metabase requires 1-2GB of RAM, your system is below this requirement continue ?",
Default: true,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask about RAM check: %s", err)
}
if !answer {
return fmt.Errorf("user stated no to continue")
}
return nil
}
return true, nil
log.Warn("Metabase requires 1-2GB of RAM, your system is below this requirement")
return nil
}
func warnIfNotLoopback(addr string) {
if addr == "127.0.0.1" || addr == "::1" {
return
}
log.Warnf("You are potentially exposing your metabase port to the internet (addr: %s), please consider using a reverse proxy", addr)
}
func disclaimer(forceYes *bool) error {
if !*forceYes {
var answer bool
prompt := &survey.Confirm{
Message: "CrowdSec takes no responsibility for the security of your metabase instance. Do you accept these responsibilities ?",
Default: true,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask to question: %s", err)
}
if !answer {
return fmt.Errorf("user stated no to responsibilities")
}
return nil
}
log.Warn("CrowdSec takes no responsibility for the security of your metabase instance. You used force yes, so you accept this disclaimer")
return nil
}
func checkGroups(forceYes *bool) (*user.Group, error) {
dockerGroup, err := user.LookupGroup(crowdsecGroup)
if err == nil {
return dockerGroup, nil
}
if !*forceYes {
var answer bool
prompt := &survey.Confirm{
Message: fmt.Sprintf("For metabase docker to be able to access SQLite file we need to add a new group called '%s' to the system, is it ok for you ?", crowdsecGroup),
Default: true,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return dockerGroup, fmt.Errorf("unable to ask to question: %s", err)
}
if !answer {
return dockerGroup, fmt.Errorf("unable to continue without creating '%s' group", crowdsecGroup)
}
}
groupAddCmd, err := exec.LookPath("groupadd")
if err != nil {
return dockerGroup, fmt.Errorf("unable to find 'groupadd' command, can't continue")
}
groupAdd := &exec.Cmd{Path: groupAddCmd, Args: []string{groupAddCmd, crowdsecGroup}}
if err := groupAdd.Run(); err != nil {
return dockerGroup, fmt.Errorf("unable to add group '%s': %s", dockerGroup, err)
}
return user.LookupGroup(crowdsecGroup)
}
func chownDatabase(gid string) error {
intID, err := strconv.Atoi(gid)
if err != nil {
return fmt.Errorf("unable to convert group ID to int: %s", err)
}
if stat, err := os.Stat(csConfig.DbConfig.DbPath); !os.IsNotExist(err) {
info := stat.Sys()
if err := os.Chown(csConfig.DbConfig.DbPath, int(info.(*syscall.Stat_t).Uid), intID); err != nil {
return fmt.Errorf("unable to chown sqlite db file '%s': %s", csConfig.DbConfig.DbPath, err)
}
}
if csConfig.DbConfig.Type == "sqlite" && csConfig.DbConfig.UseWal != nil && *csConfig.DbConfig.UseWal {
for _, ext := range []string{"-wal", "-shm"} {
file := csConfig.DbConfig.DbPath + ext
if stat, err := os.Stat(file); !os.IsNotExist(err) {
info := stat.Sys()
if err := os.Chown(file, int(info.(*syscall.Stat_t).Uid), intID); err != nil {
return fmt.Errorf("unable to chown sqlite db file '%s': %s", file, err)
}
}
}
}
return nil
}

28
cmd/crowdsec-cli/dashboard_unsupported.go Normal file
View file

@ -0,0 +1,28 @@
//go:build !linux
package main
import (
"runtime"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
)
type cliDashboard struct{}
func NewCLIDashboard() *cliDashboard {
return &cliDashboard{}
}
func (cli cliDashboard) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "dashboard",
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
log.Infof("Dashboard command is disabled on %s", runtime.GOOS)
},
}
return cmd
}

113
cmd/crowdsec-cli/decisions.go
View file

@ -16,7 +16,7 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/models"
@ -81,6 +81,12 @@ func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error
}
csvwriter.Flush()
} else if csConfig.Cscli.Output == "json" {
if *alerts == nil {
// avoid returning "null" in `json"
// could be cleaner if we used slice of alerts directly
fmt.Println("[]")
return nil
}
x, _ := json.MarshalIndent(alerts, "", " ")
fmt.Printf("%s", string(x))
} else if csConfig.Cscli.Output == "human" {
@ -96,8 +102,15 @@ func DecisionsToTable(alerts *models.GetAlertsResponse, printMachine bool) error
return nil
}
func NewDecisionsCmd() *cobra.Command {
var cmdDecisions = &cobra.Command{
type cliDecisions struct {}
func NewCLIDecisions() *cliDecisions {
return &cliDecisions{}
}
func (cli cliDecisions) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "decisions [action]",
Short: "Manage decisions",
Long: `Add/List/Delete/Import decisions from LAPI`,
@ -129,15 +142,15 @@ func NewDecisionsCmd() *cobra.Command {
},
}
cmdDecisions.AddCommand(NewDecisionsListCmd())
cmdDecisions.AddCommand(NewDecisionsAddCmd())
cmdDecisions.AddCommand(NewDecisionsDeleteCmd())
cmdDecisions.AddCommand(NewDecisionsImportCmd())
cmd.AddCommand(cli.NewListCmd())
cmd.AddCommand(cli.NewAddCmd())
cmd.AddCommand(cli.NewDeleteCmd())
cmd.AddCommand(cli.NewImportCmd())
return cmdDecisions
return cmd
}
func NewDecisionsListCmd() *cobra.Command {
func (cli cliDecisions) NewListCmd() *cobra.Command {
var filter = apiclient.AlertsListOpts{
ValueEquals: new(string),
ScopeEquals: new(string),
@ -155,7 +168,7 @@ func NewDecisionsListCmd() *cobra.Command {
contained := new(bool)
var printMachine bool
var cmdDecisionsList = &cobra.Command{
cmd := &cobra.Command{
Use: "list [options]",
Short: "List decisions from LAPI",
Example: `cscli decisions list -i 1.2.3.4
@ -245,26 +258,26 @@ cscli decisions list -t ban
return nil
},
}
cmdDecisionsList.Flags().SortFlags = false
cmdDecisionsList.Flags().BoolVarP(filter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
cmdDecisionsList.Flags().StringVar(filter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
cmdDecisionsList.Flags().StringVar(filter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
cmdDecisionsList.Flags().StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)")
cmdDecisionsList.Flags().StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)")
cmdDecisionsList.Flags().StringVar(filter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmdDecisionsList.Flags().StringVarP(filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)")
cmdDecisionsList.Flags().StringVarP(filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)")
cmdDecisionsList.Flags().StringVarP(filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
cmdDecisionsList.Flags().StringVarP(filter.RangeEquals, "range", "r", "", "restrict to alerts from this source range (shorthand for --scope range --value <RANGE>)")
cmdDecisionsList.Flags().IntVarP(filter.Limit, "limit", "l", 100, "number of alerts to get (use 0 to remove the limit)")
cmdDecisionsList.Flags().BoolVar(NoSimu, "no-simu", false, "exclude decisions in simulation mode")
cmdDecisionsList.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that triggered decisions")
cmdDecisionsList.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
cmd.Flags().SortFlags = false
cmd.Flags().BoolVarP(filter.IncludeCAPI, "all", "a", false, "Include decisions from Central API")
cmd.Flags().StringVar(filter.Since, "since", "", "restrict to alerts newer than since (ie. 4h, 30d)")
cmd.Flags().StringVar(filter.Until, "until", "", "restrict to alerts older than until (ie. 4h, 30d)")
cmd.Flags().StringVarP(filter.TypeEquals, "type", "t", "", "restrict to this decision type (ie. ban,captcha)")
cmd.Flags().StringVar(filter.ScopeEquals, "scope", "", "restrict to this scope (ie. ip,range,session)")
cmd.Flags().StringVar(filter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmd.Flags().StringVarP(filter.ValueEquals, "value", "v", "", "restrict to this value (ie. 1.2.3.4,userName)")
cmd.Flags().StringVarP(filter.ScenarioEquals, "scenario", "s", "", "restrict to this scenario (ie. crowdsecurity/ssh-bf)")
cmd.Flags().StringVarP(filter.IPEquals, "ip", "i", "", "restrict to alerts from this source ip (shorthand for --scope ip --value <IP>)")
cmd.Flags().StringVarP(filter.RangeEquals, "range", "r", "", "restrict to alerts from this source range (shorthand for --scope range --value <RANGE>)")
cmd.Flags().IntVarP(filter.Limit, "limit", "l", 100, "number of alerts to get (use 0 to remove the limit)")
cmd.Flags().BoolVar(NoSimu, "no-simu", false, "exclude decisions in simulation mode")
cmd.Flags().BoolVarP(&printMachine, "machine", "m", false, "print machines that triggered decisions")
cmd.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
return cmdDecisionsList
return cmd
}
func NewDecisionsAddCmd() *cobra.Command {
func (cli cliDecisions) NewAddCmd() *cobra.Command {
var (
addIP string
addRange string
@ -275,7 +288,7 @@ func NewDecisionsAddCmd() *cobra.Command {
addType string
)
var cmdDecisionsAdd = &cobra.Command{
cmd := &cobra.Command{
Use: "add [options]",
Short: "Add decision to LAPI",
Example: `cscli decisions add --ip 1.2.3.4
@ -363,19 +376,19 @@ cscli decisions add --scope username --value foobar
},
}
cmdDecisionsAdd.Flags().SortFlags = false
cmdDecisionsAdd.Flags().StringVarP(&addIP, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmdDecisionsAdd.Flags().StringVarP(&addRange, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmdDecisionsAdd.Flags().StringVarP(&addDuration, "duration", "d", "4h", "Decision duration (ie. 1h,4h,30m)")
cmdDecisionsAdd.Flags().StringVarP(&addValue, "value", "v", "", "The value (ie. --scope username --value foobar)")
cmdDecisionsAdd.Flags().StringVar(&addScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)")
cmdDecisionsAdd.Flags().StringVarP(&addReason, "reason", "R", "", "Decision reason (ie. scenario-name)")
cmdDecisionsAdd.Flags().StringVarP(&addType, "type", "t", "ban", "Decision type (ie. ban,captcha,throttle)")
cmd.Flags().SortFlags = false
cmd.Flags().StringVarP(&addIP, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmd.Flags().StringVarP(&addRange, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmd.Flags().StringVarP(&addDuration, "duration", "d", "4h", "Decision duration (ie. 1h,4h,30m)")
cmd.Flags().StringVarP(&addValue, "value", "v", "", "The value (ie. --scope username --value foobar)")
cmd.Flags().StringVar(&addScope, "scope", types.Ip, "Decision scope (ie. ip,range,username)")
cmd.Flags().StringVarP(&addReason, "reason", "R", "", "Decision reason (ie. scenario-name)")
cmd.Flags().StringVarP(&addType, "type", "t", "ban", "Decision type (ie. ban,captcha,throttle)")
return cmdDecisionsAdd
return cmd
}
func NewDecisionsDeleteCmd() *cobra.Command {
func (cli cliDecisions) NewDeleteCmd() *cobra.Command {
var delFilter = apiclient.DecisionsDeleteOpts{
ScopeEquals: new(string),
ValueEquals: new(string),
@ -389,7 +402,7 @@ func NewDecisionsDeleteCmd() *cobra.Command {
var delDecisionAll bool
contained := new(bool)
var cmdDecisionsDelete = &cobra.Command{
cmd := &cobra.Command{
Use: "delete [options]",
Short: "Delete decisions",
DisableAutoGenTag: true,
@ -466,17 +479,17 @@ cscli decisions delete --type captcha
},
}
cmdDecisionsDelete.Flags().SortFlags = false
cmdDecisionsDelete.Flags().StringVarP(delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmdDecisionsDelete.Flags().StringVarP(delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmdDecisionsDelete.Flags().StringVarP(delFilter.TypeEquals, "type", "t", "", "the decision type (ie. ban,captcha)")
cmdDecisionsDelete.Flags().StringVarP(delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmdDecisionsDelete.Flags().StringVarP(delFilter.ScenarioEquals, "scenario", "s", "", "the scenario name (ie. crowdsecurity/ssh-bf)")
cmdDecisionsDelete.Flags().StringVar(delFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmd.Flags().SortFlags = false
cmd.Flags().StringVarP(delFilter.IPEquals, "ip", "i", "", "Source ip (shorthand for --scope ip --value <IP>)")
cmd.Flags().StringVarP(delFilter.RangeEquals, "range", "r", "", "Range source ip (shorthand for --scope range --value <RANGE>)")
cmd.Flags().StringVarP(delFilter.TypeEquals, "type", "t", "", "the decision type (ie. ban,captcha)")
cmd.Flags().StringVarP(delFilter.ValueEquals, "value", "v", "", "the value to match for in the specified scope")
cmd.Flags().StringVarP(delFilter.ScenarioEquals, "scenario", "s", "", "the scenario name (ie. crowdsecurity/ssh-bf)")
cmd.Flags().StringVar(delFilter.OriginEquals, "origin", "", fmt.Sprintf("the value to match for the specified origin (%s ...)", strings.Join(types.GetOrigins(), ",")))
cmdDecisionsDelete.Flags().StringVar(&delDecisionId, "id", "", "decision id")
cmdDecisionsDelete.Flags().BoolVar(&delDecisionAll, "all", false, "delete all decisions")
cmdDecisionsDelete.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
cmd.Flags().StringVar(&delDecisionId, "id", "", "decision id")
cmd.Flags().BoolVar(&delDecisionAll, "all", false, "delete all decisions")
cmd.Flags().BoolVar(contained, "contained", false, "query decisions contained by range")
return cmdDecisionsDelete
return cmd
}

37
cmd/crowdsec-cli/decisions_import.go
View file

@ -15,8 +15,8 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/pkg/ptr"
"github.com/crowdsecurity/go-cs-lib/pkg/slicetools"
"github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/crowdsecurity/go-cs-lib/slicetools"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/types"
@ -63,7 +63,7 @@ func parseDecisionList(content []byte, format string) ([]decisionRaw, error) {
}
func runDecisionsImport(cmd *cobra.Command, args []string) error {
func (cli cliDecisions) runImport(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
input, err := flags.GetString("input")
@ -188,7 +188,9 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error {
}
}
alerts := models.AddAlertsRequest{}
if len(decisions) > 1000 {
log.Infof("You are about to add %d decisions, this may take a while", len(decisions))
}
for _, chunk := range slicetools.Chunks(decisions, batchSize) {
log.Debugf("Processing chunk of %d decisions", len(chunk))
@ -212,16 +214,11 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error {
ScenarioVersion: ptr.Of(""),
Decisions: chunk,
}
alerts = append(alerts, &importAlert)
}
if len(decisions) > 1000 {
log.Infof("You are about to add %d decisions, this may take a while", len(decisions))
}
_, _, err = Client.Alerts.Add(context.Background(), alerts)
if err != nil {
return err
_, _, err = Client.Alerts.Add(context.Background(), models.AddAlertsRequest{&importAlert})
if err != nil {
return err
}
}
log.Infof("Imported %d decisions", len(decisions))
@ -229,13 +226,13 @@ func runDecisionsImport(cmd *cobra.Command, args []string) error {
}
func NewDecisionsImportCmd() *cobra.Command {
var cmdDecisionsImport = &cobra.Command{
func (cli cliDecisions) NewImportCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "import [options]",
Short: "Import decisions from a file or pipe",
Long: "expected format:\n" +
"csv : any of duration,reason,scope,type,value, with a header line\n" +
`json : {"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"}`,
"json :" + "`{" + `"duration" : "24h", "reason" : "my_scenario", "scope" : "ip", "type" : "ban", "value" : "x.y.z.z"` + "}`",
DisableAutoGenTag: true,
Example: `decisions.csv:
duration,scope,value
@ -253,10 +250,10 @@ Raw values, standard input:
$ echo "1.2.3.4" | cscli decisions import -i - --format values
`,
RunE: runDecisionsImport,
RunE: cli.runImport,
}
flags := cmdDecisionsImport.Flags()
flags := cmd.Flags()
flags.SortFlags = false
flags.StringP("input", "i", "", "Input file")
flags.StringP("duration", "d", "4h", "Decision duration: 1h,4h,30m")
@ -266,7 +263,7 @@ $ echo "1.2.3.4" | cscli decisions import -i - --format values
flags.Int("batch", 0, "Split import in batches of N decisions")
flags.String("format", "", "Input format: 'json', 'csv' or 'values' (each line is a value, no headers)")
cmdDecisionsImport.MarkFlagRequired("input")
cmd.MarkFlagRequired("input")
return cmdDecisionsImport
return cmd
}

187
cmd/crowdsec-cli/explain.go
View file

@ -12,10 +12,107 @@ import (
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/hubtest"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
func runExplain(cmd *cobra.Command, args []string) error {
func GetLineCountForFile(filepath string) (int, error) {
f, err := os.Open(filepath)
if err != nil {
return 0, err
}
defer f.Close()
lc := 0
fs := bufio.NewReader(f)
for {
input, err := fs.ReadBytes('\n')
if len(input) > 1 {
lc++
}
if err != nil && err == io.EOF {
break
}
}
return lc, nil
}
type cliExplain struct {}
func NewCLIExplain() *cliExplain {
return &cliExplain{}
}
func (cli cliExplain) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "explain",
Short: "Explain log pipeline",
Long: `
Explain log pipeline
`,
Example: `
cscli explain --file ./myfile.log --type nginx
cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog
cscli explain --dsn "file://myfile.log" --type nginx
tail -n 5 myfile.log | cscli explain --type nginx -f -
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: cli.run,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
logFile, err := flags.GetString("file")
if err != nil {
return err
}
dsn, err := flags.GetString("dsn")
if err != nil {
return err
}
logLine, err := flags.GetString("log")
if err != nil {
return err
}
logType, err := flags.GetString("type")
if err != nil {
return err
}
if logLine == "" && logFile == "" && dsn == "" {
printHelp(cmd)
fmt.Println()
return fmt.Errorf("please provide --log, --file or --dsn flag")
}
if logType == "" {
printHelp(cmd)
fmt.Println()
return fmt.Errorf("please provide --type flag")
}
fileInfo, _ := os.Stdin.Stat()
if logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) {
return fmt.Errorf("the option -f - is intended to work with pipes")
}
return nil
},
}
flags := cmd.Flags()
flags.StringP("file", "f", "", "Log file to test")
flags.StringP("dsn", "d", "", "DSN to test")
flags.StringP("log", "l", "", "Log line to test")
flags.StringP("type", "t", "", "Type of the acquisition to test")
flags.String("labels", "", "Additional labels to add to the acquisition format (key:value,key2:value2)")
flags.BoolP("verbose", "v", false, "Display individual changes")
flags.Bool("failures", false, "Only show failed lines")
flags.Bool("only-successful-parsers", false, "Only show successful parsers")
flags.String("crowdsec", "crowdsec", "Path to crowdsec")
return cmd
}
func (cli cliExplain) run(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
logFile, err := flags.GetString("file")
@ -61,17 +158,9 @@ func runExplain(cmd *cobra.Command, args []string) error {
return err
}
fileInfo, _ := os.Stdin.Stat()
if logType == "" || (logLine == "" && logFile == "" && dsn == "") {
printHelp(cmd)
fmt.Println()
fmt.Printf("Please provide --type flag\n")
os.Exit(1)
}
if logFile == "-" && ((fileInfo.Mode() & os.ModeCharDevice) == os.ModeCharDevice) {
return fmt.Errorf("the option -f - is intended to work with pipes")
labels, err := flags.GetString("labels")
if err != nil {
return err
}
var f *os.File
@ -81,6 +170,13 @@ func runExplain(cmd *cobra.Command, args []string) error {
if err != nil {
return fmt.Errorf("couldn't create a temporary directory to store cscli explain result: %s", err)
}
defer func() {
if _, err := os.Stat(dir); !os.IsNotExist(err) {
if err := os.RemoveAll(dir); err != nil {
log.Errorf("unable to delete temporary directory '%s': %s", dir, err)
}
}
}()
tmpFile := ""
// we create a temporary log file if a log line/stdin has been provided
if logLine != "" || logFile == "-" {
@ -103,13 +199,15 @@ func runExplain(cmd *cobra.Command, args []string) error {
if err != nil && err == io.EOF {
break
}
_, err = f.Write(input)
if err != nil {
if len(input) > 1 {
_, err = f.Write(input)
}
if err != nil || len(input) <= 1 {
errCount++
}
}
if errCount > 0 {
log.Warnf("Failed to write %d lines to tmp file", errCount)
log.Warnf("Failed to write %d lines to %s", errCount, tmpFile)
}
}
f.Close()
@ -123,9 +221,16 @@ func runExplain(cmd *cobra.Command, args []string) error {
return fmt.Errorf("unable to get absolute path of '%s', exiting", logFile)
}
dsn = fmt.Sprintf("file://%s", absolutePath)
lineCount := types.GetLineCountForFile(absolutePath)
lineCount, err := GetLineCountForFile(absolutePath)
if err != nil {
return err
}
log.Debugf("file %s has %d lines", absolutePath, lineCount)
if lineCount == 0 {
return fmt.Errorf("the log file is empty: %s", absolutePath)
}
if lineCount > 100 {
log.Warnf("log file contains %d lines. This may take lot of resources.", lineCount)
log.Warnf("%s contains %d lines. This may take a lot of resources.", absolutePath, lineCount)
}
}
@ -134,6 +239,10 @@ func runExplain(cmd *cobra.Command, args []string) error {
}
cmdArgs := []string{"-c", ConfigFilePath, "-type", logType, "-dsn", dsn, "-dump-data", dir, "-no-api"}
if labels != "" {
log.Debugf("adding labels %s", labels)
cmdArgs = append(cmdArgs, "-label", labels)
}
crowdsecCmd := exec.Command(crowdsec, cmdArgs...)
output, err := crowdsecCmd.CombinedOutput()
if err != nil {
@ -141,12 +250,6 @@ func runExplain(cmd *cobra.Command, args []string) error {
return fmt.Errorf("fail to run crowdsec for test: %v", err)
}
// rm the temporary log file if only a log line/stdin was provided
if tmpFile != "" {
if err := os.Remove(tmpFile); err != nil {
return fmt.Errorf("unable to remove tmp log file '%s': %+v", tmpFile, err)
}
}
parserDumpFile := filepath.Join(dir, hubtest.ParserResultFileName)
bucketStateDumpFile := filepath.Join(dir, hubtest.BucketPourResultFileName)
@ -162,41 +265,5 @@ func runExplain(cmd *cobra.Command, args []string) error {
hubtest.DumpTree(*parserDump, *bucketStateDump, opts)
if err := os.RemoveAll(dir); err != nil {
return fmt.Errorf("unable to delete temporary directory '%s': %s", dir, err)
}
return nil
}
func NewExplainCmd() *cobra.Command {
cmdExplain := &cobra.Command{
Use: "explain",
Short: "Explain log pipeline",
Long: `
Explain log pipeline
`,
Example: `
cscli explain --file ./myfile.log --type nginx
cscli explain --log "Sep 19 18:33:22 scw-d95986 sshd[24347]: pam_unix(sshd:auth): authentication failure; logname= uid=0 euid=0 tty=ssh ruser= rhost=1.2.3.4" --type syslog
cscli explain --dsn "file://myfile.log" --type nginx
tail -n 5 myfile.log | cscli explain --type nginx -f -
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: runExplain,
}
flags := cmdExplain.Flags()
flags.StringP("file", "f", "", "Log file to test")
flags.StringP("dsn", "d", "", "DSN to test")
flags.StringP("log", "l", "", "Log line to test")
flags.StringP("type", "t", "", "Type of the acquisition to test")
flags.BoolP("verbose", "v", false, "Display individual changes")
flags.Bool("failures", false, "Only show failed lines")
flags.Bool("only-successful-parsers", false, "Only show successful parsers")
flags.String("crowdsec", "crowdsec", "Path to crowdsec")
return cmdExplain
}

290
cmd/crowdsec-cli/hub.go
View file

@ -1,164 +1,220 @@
package main
import (
"errors"
"encoding/json"
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewHubCmd() *cobra.Command {
var cmdHub = &cobra.Command{
type cliHub struct{}
func NewCLIHub() *cliHub {
return &cliHub{}
}
func (cli cliHub) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "hub [action]",
Short: "Manage Hub",
Long: `
Hub management
Short: "Manage hub index",
Long: `Hub management
List/update parsers/scenarios/postoverflows/collections from [Crowdsec Hub](https://hub.crowdsec.net).
The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.
`,
Example: `
cscli hub list # List all installed configurations
cscli hub update # Download list of available configurations from the hub
`,
The Hub is managed by cscli, to get the latest hub files from [Crowdsec Hub](https://hub.crowdsec.net), you need to update.`,
Example: `cscli hub list
cscli hub update
cscli hub upgrade`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
return nil
},
}
cmdHub.PersistentFlags().StringVarP(&cwhub.HubBranch, "branch", "b", "", "Use given branch from hub")
cmdHub.AddCommand(NewHubListCmd())
cmdHub.AddCommand(NewHubUpdateCmd())
cmdHub.AddCommand(NewHubUpgradeCmd())
cmd.AddCommand(cli.NewListCmd())
cmd.AddCommand(cli.NewUpdateCmd())
cmd.AddCommand(cli.NewUpgradeCmd())
cmd.AddCommand(cli.NewTypesCmd())
return cmdHub
return cmd
}
func NewHubListCmd() *cobra.Command {
var cmdHubList = &cobra.Command{
func (cli cliHub) list(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
all, err := flags.GetBool("all")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
for _, v := range hub.Warnings {
log.Info(v)
}
for _, line := range hub.ItemStats() {
log.Info(line)
}
items := make(map[string][]*cwhub.Item)
for _, itemType := range cwhub.ItemTypes {
items[itemType], err = selectItems(hub, itemType, nil, !all)
if err != nil {
return err
}
}
err = listItems(color.Output, cwhub.ItemTypes, items, true)
if err != nil {
return err
}
return nil
}
func (cli cliHub) NewListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "list [-a]",
Short: "List installed configs",
Short: "List all installed configurations",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
}
//use LocalSync to get warnings about tainted / outdated items
_, warn := cwhub.LocalSync(csConfig.Hub)
for _, v := range warn {
log.Info(v)
}
cwhub.DisplaySummary()
ListItems(color.Output, []string{
cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW,
}, args, true, false, all)
},
RunE: cli.list,
}
cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmdHubList
flags := cmd.Flags()
flags.BoolP("all", "a", false, "List disabled items as well")
return cmd
}
func NewHubUpdateCmd() *cobra.Command {
var cmdHubUpdate = &cobra.Command{
func (cli cliHub) update(cmd *cobra.Command, args []string) error {
local := csConfig.Hub
remote := require.RemoteHub(csConfig)
// don't use require.Hub because if there is no index file, it would fail
hub, err := cwhub.NewHub(local, remote, true)
if err != nil {
return fmt.Errorf("failed to update hub: %w", err)
}
for _, v := range hub.Warnings {
log.Info(v)
}
return nil
}
func (cli cliHub) NewUpdateCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "update",
Short: "Fetch available configs from hub",
Short: "Download the latest index (catalog of available configurations)",
Long: `
Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs.
Fetches the .index.json file from the hub, containing the list of available configs.
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
return nil
},
Run: func(cmd *cobra.Command, args []string) {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
if errors.Is(err, cwhub.ErrIndexNotFound) {
log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch)
cwhub.HubBranch = "master"
if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to get Hub index after retry : %v", err)
}
} else {
log.Fatalf("Failed to get Hub index : %v", err)
}
}
//use LocalSync to get warnings about tainted / outdated items
_, warn := cwhub.LocalSync(csConfig.Hub)
for _, v := range warn {
log.Info(v)
}
},
RunE: cli.update,
}
return cmdHubUpdate
return cmd
}
func NewHubUpgradeCmd() *cobra.Command {
var cmdHubUpgrade = &cobra.Command{
func (cli cliHub) upgrade(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
force, err := flags.GetBool("force")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
for _, itemType := range cwhub.ItemTypes {
items, err := hub.GetInstalledItems(itemType)
if err != nil {
return err
}
updated := 0
log.Infof("Upgrading %s", itemType)
for _, item := range items {
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
updated++
}
}
log.Infof("Upgraded %d %s", updated, itemType)
}
return nil
}
func (cli cliHub) NewUpgradeCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "upgrade",
Short: "Upgrade all configs installed from hub",
Short: "Upgrade all configurations to their latest version",
Long: `
Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if you want the latest versions available.
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
return nil
},
Run: func(cmd *cobra.Command, args []string) {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
}
log.Infof("Upgrading collections")
cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", forceAction)
log.Infof("Upgrading parsers")
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
log.Infof("Upgrading scenarios")
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
log.Infof("Upgrading postoverflows")
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
},
RunE: cli.upgrade,
}
cmdHubUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
return cmdHubUpgrade
flags := cmd.Flags()
flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
return cmd
}
func (cli cliHub) types(cmd *cobra.Command, args []string) error {
switch csConfig.Cscli.Output {
case "human":
s, err := yaml.Marshal(cwhub.ItemTypes)
if err != nil {
return err
}
fmt.Print(string(s))
case "json":
jsonStr, err := json.Marshal(cwhub.ItemTypes)
if err != nil {
return err
}
fmt.Println(string(jsonStr))
case "raw":
for _, itemType := range cwhub.ItemTypes {
fmt.Println(itemType)
}
}
return nil
}
func (cli cliHub) NewTypesCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "types",
Short: "List supported item types",
Long: `
List the types of supported hub items.
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: cli.types,
}
return cmd
}

105
cmd/crowdsec-cli/hubappsec.go Normal file
View file

@ -0,0 +1,105 @@
package main
import (
"fmt"
"os"
"golang.org/x/text/cases"
"golang.org/x/text/language"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/pkg/appsec"
"github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIAppsecConfig() *cliItem {
return &cliItem{
name: cwhub.APPSEC_CONFIGS,
singular: "appsec-config",
oneOrMore: "appsec-config(s)",
help: cliHelp{
example: `cscli appsec-configs list -a
cscli appsec-configs install crowdsecurity/vpatch
cscli appsec-configs inspect crowdsecurity/vpatch
cscli appsec-configs upgrade crowdsecurity/vpatch
cscli appsec-configs remove crowdsecurity/vpatch
`,
},
installHelp: cliHelp{
example: `cscli appsec-configs install crowdsecurity/vpatch`,
},
removeHelp: cliHelp{
example: `cscli appsec-configs remove crowdsecurity/vpatch`,
},
upgradeHelp: cliHelp{
example: `cscli appsec-configs upgrade crowdsecurity/vpatch`,
},
inspectHelp: cliHelp{
example: `cscli appsec-configs inspect crowdsecurity/vpatch`,
},
listHelp: cliHelp{
example: `cscli appsec-configs list
cscli appsec-configs list -a
cscli appsec-configs list crowdsecurity/vpatch`,
},
}
}
func NewCLIAppsecRule() *cliItem {
inspectDetail := func(item *cwhub.Item) error {
appsecRule := appsec.AppsecCollectionConfig{}
yamlContent, err := os.ReadFile(item.State.LocalPath)
if err != nil {
return fmt.Errorf("unable to read file %s : %s", item.State.LocalPath, err)
}
if err := yaml.Unmarshal(yamlContent, &appsecRule); err != nil {
return fmt.Errorf("unable to unmarshal yaml file %s : %s", item.State.LocalPath, err)
}
for _, ruleType := range appsec_rule.SupportedTypes() {
fmt.Printf("\n%s format:\n", cases.Title(language.Und, cases.NoLower).String(ruleType))
for _, rule := range appsecRule.Rules {
convertedRule, _, err := rule.Convert(ruleType, appsecRule.Name)
if err != nil {
return fmt.Errorf("unable to convert rule %s : %s", rule.Name, err)
}
fmt.Println(convertedRule)
}
}
return nil
}
return &cliItem{
name: "appsec-rules",
singular: "appsec-rule",
oneOrMore: "appsec-rule(s)",
help: cliHelp{
example: `cscli appsec-rules list -a
cscli appsec-rules install crowdsecurity/crs
cscli appsec-rules inspect crowdsecurity/crs
cscli appsec-rules upgrade crowdsecurity/crs
cscli appsec-rules remove crowdsecurity/crs
`,
},
installHelp: cliHelp{
example: `cscli appsec-rules install crowdsecurity/crs`,
},
removeHelp: cliHelp{
example: `cscli appsec-rules remove crowdsecurity/crs`,
},
upgradeHelp: cliHelp{
example: `cscli appsec-rules upgrade crowdsecurity/crs`,
},
inspectHelp: cliHelp{
example: `cscli appsec-rules inspect crowdsecurity/crs`,
},
inspectDetail: inspectDetail,
listHelp: cliHelp{
example: `cscli appsec-rules list
cscli appsec-rules list -a
cscli appsec-rules list crowdsecurity/crs`,
},
}
}

40
cmd/crowdsec-cli/hubcollection.go Normal file
View file

@ -0,0 +1,40 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLICollection() *cliItem {
return &cliItem{
name: cwhub.COLLECTIONS,
singular: "collection",
oneOrMore: "collection(s)",
help: cliHelp{
example: `cscli collections list -a
cscli collections install crowdsecurity/http-cve crowdsecurity/iptables
cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables
cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables
cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
`,
},
installHelp: cliHelp{
example: `cscli collections install crowdsecurity/http-cve crowdsecurity/iptables`,
},
removeHelp: cliHelp{
example: `cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables`,
},
upgradeHelp: cliHelp{
example: `cscli collections upgrade crowdsecurity/http-cve crowdsecurity/iptables`,
},
inspectHelp: cliHelp{
example: `cscli collections inspect crowdsecurity/http-cve crowdsecurity/iptables`,
},
listHelp: cliHelp{
example: `cscli collections list
cscli collections list -a
cscli collections list crowdsecurity/http-cve crowdsecurity/iptables
List only enabled collections unless "-a" or names are specified.`,
},
}
}

40
cmd/crowdsec-cli/hubcontext.go Normal file
View file

@ -0,0 +1,40 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIContext() *cliItem {
return &cliItem{
name: cwhub.CONTEXTS,
singular: "context",
oneOrMore: "context(s)",
help: cliHelp{
example: `cscli contexts list -a
cscli contexts install crowdsecurity/yyy crowdsecurity/zzz
cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz
cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz
cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz
`,
},
installHelp: cliHelp{
example: `cscli contexts install crowdsecurity/yyy crowdsecurity/zzz`,
},
removeHelp: cliHelp{
example: `cscli contexts remove crowdsecurity/yyy crowdsecurity/zzz`,
},
upgradeHelp: cliHelp{
example: `cscli contexts upgrade crowdsecurity/yyy crowdsecurity/zzz`,
},
inspectHelp: cliHelp{
example: `cscli contexts inspect crowdsecurity/yyy crowdsecurity/zzz`,
},
listHelp: cliHelp{
example: `cscli contexts list
cscli contexts list -a
cscli contexts list crowdsecurity/yyy crowdsecurity/zzz
List only enabled contexts unless "-a" or names are specified.`,
},
}
}

40
cmd/crowdsec-cli/hubparser.go Normal file
View file

@ -0,0 +1,40 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIParser() *cliItem {
return &cliItem{
name: cwhub.PARSERS,
singular: "parser",
oneOrMore: "parser(s)",
help: cliHelp{
example: `cscli parsers list -a
cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers inspect crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs
cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
`,
},
installHelp: cliHelp{
example: `cscli parsers install crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
removeHelp: cliHelp{
example: `cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
upgradeHelp: cliHelp{
example: `cscli parsers upgrade crowdsecurity/caddy-logs crowdsecurity/sshd-logs`,
},
inspectHelp: cliHelp{
example: `cscli parsers inspect crowdsecurity/httpd-logs crowdsecurity/sshd-logs`,
},
listHelp: cliHelp{
example: `cscli parsers list
cscli parsers list -a
cscli parsers list crowdsecurity/caddy-logs crowdsecurity/sshd-logs
List only enabled parsers unless "-a" or names are specified.`,
},
}
}

40
cmd/crowdsec-cli/hubpostoverflow.go Normal file
View file

@ -0,0 +1,40 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIPostOverflow() *cliItem {
return &cliItem{
name: cwhub.POSTOVERFLOWS,
singular: "postoverflow",
oneOrMore: "postoverflow(s)",
help: cliHelp{
example: `cscli postoverflows list -a
cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns
cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
`,
},
installHelp: cliHelp{
example: `cscli postoverflows install crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
removeHelp: cliHelp{
example: `cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
upgradeHelp: cliHelp{
example: `cscli postoverflows upgrade crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
inspectHelp: cliHelp{
example: `cscli postoverflows inspect crowdsecurity/cdn-whitelist crowdsecurity/rdns`,
},
listHelp: cliHelp{
example: `cscli postoverflows list
cscli postoverflows list -a
cscli postoverflows list crowdsecurity/cdn-whitelist crowdsecurity/rdns
List only enabled postoverflows unless "-a" or names are specified.`,
},
}
}

40
cmd/crowdsec-cli/hubscenario.go Normal file
View file

@ -0,0 +1,40 @@
package main
import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewCLIScenario() *cliItem {
return &cliItem{
name: cwhub.SCENARIOS,
singular: "scenario",
oneOrMore: "scenario(s)",
help: cliHelp{
example: `cscli scenarios list -a
cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing
cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
`,
},
installHelp: cliHelp{
example: `cscli scenarios install crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
removeHelp: cliHelp{
example: `cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
upgradeHelp: cliHelp{
example: `cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
inspectHelp: cliHelp{
example: `cscli scenarios inspect crowdsecurity/ssh-bf crowdsecurity/http-probing`,
},
listHelp: cliHelp{
example: `cscli scenarios list
cscli scenarios list -a
cscli scenarios list crowdsecurity/ssh-bf crowdsecurity/http-probing
List only enabled scenarios unless "-a" or names are specified.`,
},
}
}

384
cmd/crowdsec-cli/hubtest.go
View file

@ -18,16 +18,23 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/hubtest"
)
var (
HubTest hubtest.HubTest
)
var HubTest hubtest.HubTest
var HubAppsecTests hubtest.HubTest
var hubPtr *hubtest.HubTest
var isAppsecTest bool
func NewHubTestCmd() *cobra.Command {
type cliHubTest struct{}
func NewCLIHubTest() *cliHubTest {
return &cliHubTest{}
}
func (cli cliHubTest) NewCommand() *cobra.Command {
var hubPath string
var crowdsecPath string
var cscliPath string
var cmdHubTest = &cobra.Command{
cmd := &cobra.Command{
Use: "hubtest",
Short: "Run functional tests on hub configurations",
Long: "Run functional tests on hub configurations (parsers, scenarios, collections...)",
@ -35,32 +42,42 @@ func NewHubTestCmd() *cobra.Command {
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
var err error
HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath)
HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, false)
if err != nil {
return fmt.Errorf("unable to load hubtest: %+v", err)
}
HubAppsecTests, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, true)
if err != nil {
return fmt.Errorf("unable to load appsec specific hubtest: %+v", err)
}
/*commands will use the hubPtr, will point to the default hubTest object, or the one dedicated to appsec tests*/
hubPtr = &HubTest
if isAppsecTest {
hubPtr = &HubAppsecTests
}
return nil
},
}
cmdHubTest.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
cmdHubTest.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
cmdHubTest.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
cmdHubTest.AddCommand(NewHubTestCreateCmd())
cmdHubTest.AddCommand(NewHubTestRunCmd())
cmdHubTest.AddCommand(NewHubTestCleanCmd())
cmdHubTest.AddCommand(NewHubTestInfoCmd())
cmdHubTest.AddCommand(NewHubTestListCmd())
cmdHubTest.AddCommand(NewHubTestCoverageCmd())
cmdHubTest.AddCommand(NewHubTestEvalCmd())
cmdHubTest.AddCommand(NewHubTestExplainCmd())
cmd.PersistentFlags().StringVar(&hubPath, "hub", ".", "Path to hub folder")
cmd.PersistentFlags().StringVar(&crowdsecPath, "crowdsec", "crowdsec", "Path to crowdsec")
cmd.PersistentFlags().StringVar(&cscliPath, "cscli", "cscli", "Path to cscli")
cmd.PersistentFlags().BoolVar(&isAppsecTest, "appsec", false, "Command relates to appsec tests")
return cmdHubTest
cmd.AddCommand(cli.NewCreateCmd())
cmd.AddCommand(cli.NewRunCmd())
cmd.AddCommand(cli.NewCleanCmd())
cmd.AddCommand(cli.NewInfoCmd())
cmd.AddCommand(cli.NewListCmd())
cmd.AddCommand(cli.NewCoverageCmd())
cmd.AddCommand(cli.NewEvalCmd())
cmd.AddCommand(cli.NewExplainCmd())
return cmd
}
func NewHubTestCreateCmd() *cobra.Command {
func (cli cliHubTest) NewCreateCmd() *cobra.Command {
parsers := []string{}
postoverflows := []string{}
scenarios := []string{}
@ -68,7 +85,7 @@ func NewHubTestCreateCmd() *cobra.Command {
var labels map[string]string
var logType string
var cmdHubTestCreate = &cobra.Command{
cmd := &cobra.Command{
Use: "create",
Short: "create [test_name]",
Example: `cscli hubtest create my-awesome-test --type syslog
@ -78,7 +95,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
testName := args[0]
testPath := filepath.Join(HubTest.HubTestPath, testName)
testPath := filepath.Join(hubPtr.HubTestPath, testName)
if _, err := os.Stat(testPath); os.IsExist(err) {
return fmt.Errorf("test '%s' already exists in '%s', exiting", testName, testPath)
}
@ -91,54 +108,77 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
return fmt.Errorf("unable to create folder '%s': %+v", testPath, err)
}
// create empty log file
logFileName := fmt.Sprintf("%s.log", testName)
logFilePath := filepath.Join(testPath, logFileName)
logFile, err := os.Create(logFilePath)
if err != nil {
return err
}
logFile.Close()
// create empty parser assertion file
parserAssertFilePath := filepath.Join(testPath, hubtest.ParserAssertFileName)
parserAssertFile, err := os.Create(parserAssertFilePath)
if err != nil {
return err
}
parserAssertFile.Close()
// create empty scenario assertion file
scenarioAssertFilePath := filepath.Join(testPath, hubtest.ScenarioAssertFileName)
scenarioAssertFile, err := os.Create(scenarioAssertFilePath)
if err != nil {
return err
}
scenarioAssertFile.Close()
parsers = append(parsers, "crowdsecurity/syslog-logs")
parsers = append(parsers, "crowdsecurity/dateparse-enrich")
if len(scenarios) == 0 {
scenarios = append(scenarios, "")
}
if len(postoverflows) == 0 {
postoverflows = append(postoverflows, "")
}
configFileData := &hubtest.HubTestItemConfig{
Parsers: parsers,
Scenarios: scenarios,
PostOVerflows: postoverflows,
LogFile: logFileName,
LogType: logType,
IgnoreParsers: ignoreParsers,
Labels: labels,
}
configFilePath := filepath.Join(testPath, "config.yaml")
fd, err := os.OpenFile(configFilePath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
configFileData := &hubtest.HubTestItemConfig{}
if logType == "appsec" {
//create empty nuclei template file
nucleiFileName := fmt.Sprintf("%s.yaml", testName)
nucleiFilePath := filepath.Join(testPath, nucleiFileName)
nucleiFile, err := os.Create(nucleiFilePath)
if err != nil {
return err
}
nucleiFile.Close()
configFileData.AppsecRules = []string{"your_rule_here.yaml"}
configFileData.NucleiTemplate = nucleiFileName
fmt.Println()
fmt.Printf(" Test name : %s\n", testName)
fmt.Printf(" Test path : %s\n", testPath)
fmt.Printf(" Nuclei Template : %s\n", nucleiFileName)
} else {
// create empty log file
logFileName := fmt.Sprintf("%s.log", testName)
logFilePath := filepath.Join(testPath, logFileName)
logFile, err := os.Create(logFilePath)
if err != nil {
return err
}
logFile.Close()
// create empty parser assertion file
parserAssertFilePath := filepath.Join(testPath, hubtest.ParserAssertFileName)
parserAssertFile, err := os.Create(parserAssertFilePath)
if err != nil {
return err
}
parserAssertFile.Close()
// create empty scenario assertion file
scenarioAssertFilePath := filepath.Join(testPath, hubtest.ScenarioAssertFileName)
scenarioAssertFile, err := os.Create(scenarioAssertFilePath)
if err != nil {
return err
}
scenarioAssertFile.Close()
parsers = append(parsers, "crowdsecurity/syslog-logs")
parsers = append(parsers, "crowdsecurity/dateparse-enrich")
if len(scenarios) == 0 {
scenarios = append(scenarios, "")
}
if len(postoverflows) == 0 {
postoverflows = append(postoverflows, "")
}
configFileData.Parsers = parsers
configFileData.Scenarios = scenarios
configFileData.PostOverflows = postoverflows
configFileData.LogFile = logFileName
configFileData.LogType = logType
configFileData.IgnoreParsers = ignoreParsers
configFileData.Labels = labels
fmt.Println()
fmt.Printf(" Test name : %s\n", testName)
fmt.Printf(" Test path : %s\n", testPath)
fmt.Printf(" Log file : %s (please fill it with logs)\n", logFilePath)
fmt.Printf(" Parser assertion file : %s (please fill it with assertion)\n", parserAssertFilePath)
fmt.Printf(" Scenario assertion file : %s (please fill it with assertion)\n", scenarioAssertFilePath)
fmt.Printf(" Configuration File : %s (please fill it with parsers, scenarios...)\n", configFilePath)
}
fd, err := os.Create(configFilePath)
if err != nil {
return fmt.Errorf("open: %s", err)
}
@ -153,56 +193,50 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
if err := fd.Close(); err != nil {
return fmt.Errorf("close: %s", err)
}
fmt.Println()
fmt.Printf(" Test name : %s\n", testName)
fmt.Printf(" Test path : %s\n", testPath)
fmt.Printf(" Log file : %s (please fill it with logs)\n", logFilePath)
fmt.Printf(" Parser assertion file : %s (please fill it with assertion)\n", parserAssertFilePath)
fmt.Printf(" Scenario assertion file : %s (please fill it with assertion)\n", scenarioAssertFilePath)
fmt.Printf(" Configuration File : %s (please fill it with parsers, scenarios...)\n", configFilePath)
return nil
},
}
cmdHubTestCreate.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test")
cmdHubTestCreate.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test")
cmdHubTestCreate.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test")
cmdHubTestCreate.Flags().StringSliceVarP(&scenarios, "scenarios", "s", scenarios, "Scenarios to add to test")
cmdHubTestCreate.PersistentFlags().BoolVar(&ignoreParsers, "ignore-parsers", false, "Don't run test on parsers")
return cmdHubTestCreate
cmd.PersistentFlags().StringVarP(&logType, "type", "t", "", "Log type of the test")
cmd.Flags().StringSliceVarP(&parsers, "parsers", "p", parsers, "Parsers to add to test")
cmd.Flags().StringSliceVar(&postoverflows, "postoverflows", postoverflows, "Postoverflows to add to test")
cmd.Flags().StringSliceVarP(&scenarios, "scenarios", "s", scenarios, "Scenarios to add to test")
cmd.PersistentFlags().BoolVar(&ignoreParsers, "ignore-parsers", false, "Don't run test on parsers")
return cmd
}
func NewHubTestRunCmd() *cobra.Command {
func (cli cliHubTest) NewRunCmd() *cobra.Command {
var noClean bool
var runAll bool
var forceClean bool
var cmdHubTestRun = &cobra.Command{
var cmd = &cobra.Command{
Use: "run",
Short: "run [test_name]",
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if !runAll && len(args) == 0 {
printHelp(cmd)
return fmt.Errorf("Please provide test to run or --all flag")
return fmt.Errorf("please provide test to run or --all flag")
}
if runAll {
if err := HubTest.LoadAllTests(); err != nil {
if err := hubPtr.LoadAllTests(); err != nil {
return fmt.Errorf("unable to load all tests: %+v", err)
}
} else {
for _, testName := range args {
_, err := HubTest.LoadTestItem(testName)
_, err := hubPtr.LoadTestItem(testName)
if err != nil {
return fmt.Errorf("unable to load test '%s': %s", testName, err)
}
}
}
for _, test := range HubTest.Tests {
// set timezone to avoid DST issues
os.Setenv("TZ", "UTC")
for _, test := range hubPtr.Tests {
if csConfig.Cscli.Output == "human" {
log.Infof("Running test '%s'", test.Name)
}
@ -217,8 +251,8 @@ func NewHubTestRunCmd() *cobra.Command {
PersistentPostRunE: func(cmd *cobra.Command, args []string) error {
success := true
testResult := make(map[string]bool)
for _, test := range HubTest.Tests {
if test.AutoGen {
for _, test := range hubPtr.Tests {
if test.AutoGen && !isAppsecTest {
if test.ParserAssert.AutoGenAssert {
log.Warningf("Assert file '%s' is empty, generating assertion:", test.ParserAssert.File)
fmt.Println()
@ -293,9 +327,11 @@ func NewHubTestRunCmd() *cobra.Command {
}
}
}
if csConfig.Cscli.Output == "human" {
switch csConfig.Cscli.Output {
case "human":
hubTestResultTable(color.Output, testResult)
} else if csConfig.Cscli.Output == "json" {
case "json":
jsonResult := make(map[string][]string, 0)
jsonResult["success"] = make([]string, 0)
jsonResult["fail"] = make([]string, 0)
@ -311,6 +347,8 @@ func NewHubTestRunCmd() *cobra.Command {
return fmt.Errorf("unable to json test result: %s", err)
}
fmt.Println(string(jsonStr))
default:
return fmt.Errorf("only human/json output modes are supported")
}
if !success {
@ -320,23 +358,23 @@ func NewHubTestRunCmd() *cobra.Command {
return nil
},
}
cmdHubTestRun.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
cmdHubTestRun.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
cmdHubTestRun.Flags().BoolVar(&runAll, "all", false, "Run all tests")
return cmdHubTestRun
cmd.Flags().BoolVar(&noClean, "no-clean", false, "Don't clean runtime environment if test succeed")
cmd.Flags().BoolVar(&forceClean, "clean", false, "Clean runtime environment if test fail")
cmd.Flags().BoolVar(&runAll, "all", false, "Run all tests")
return cmd
}
func NewHubTestCleanCmd() *cobra.Command {
var cmdHubTestClean = &cobra.Command{
func (cli cliHubTest) NewCleanCmd() *cobra.Command {
var cmd = &cobra.Command{
Use: "clean",
Short: "clean [test_name]",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
for _, testName := range args {
test, err := HubTest.LoadTestItem(testName)
test, err := hubPtr.LoadTestItem(testName)
if err != nil {
return fmt.Errorf("unable to load test '%s': %s", testName, err)
}
@ -349,28 +387,32 @@ func NewHubTestCleanCmd() *cobra.Command {
},
}
return cmdHubTestClean
return cmd
}
func NewHubTestInfoCmd() *cobra.Command {
var cmdHubTestInfo = &cobra.Command{
func (cli cliHubTest) NewInfoCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "info",
Short: "info [test_name]",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
for _, testName := range args {
test, err := HubTest.LoadTestItem(testName)
test, err := hubPtr.LoadTestItem(testName)
if err != nil {
return fmt.Errorf("unable to load test '%s': %s", testName, err)
}
fmt.Println()
fmt.Printf(" Test name : %s\n", test.Name)
fmt.Printf(" Test path : %s\n", test.Path)
fmt.Printf(" Log file : %s\n", filepath.Join(test.Path, test.Config.LogFile))
fmt.Printf(" Parser assertion file : %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName))
fmt.Printf(" Scenario assertion file : %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName))
if isAppsecTest {
fmt.Printf(" Nuclei Template : %s\n", test.Config.NucleiTemplate)
fmt.Printf(" Appsec Rules : %s\n", strings.Join(test.Config.AppsecRules, ", "))
} else {
fmt.Printf(" Log file : %s\n", filepath.Join(test.Path, test.Config.LogFile))
fmt.Printf(" Parser assertion file : %s\n", filepath.Join(test.Path, hubtest.ParserAssertFileName))
fmt.Printf(" Scenario assertion file : %s\n", filepath.Join(test.Path, hubtest.ScenarioAssertFileName))
}
fmt.Printf(" Configuration File : %s\n", filepath.Join(test.Path, "config.yaml"))
}
@ -378,25 +420,24 @@ func NewHubTestInfoCmd() *cobra.Command {
},
}
return cmdHubTestInfo
return cmd
}
func NewHubTestListCmd() *cobra.Command {
var cmdHubTestList = &cobra.Command{
func (cli cliHubTest) NewListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "list",
Short: "list",
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
if err := HubTest.LoadAllTests(); err != nil {
if err := hubPtr.LoadAllTests(); err != nil {
return fmt.Errorf("unable to load all tests: %s", err)
}
switch csConfig.Cscli.Output {
case "human":
hubTestListTable(color.Output, HubTest.Tests)
hubTestListTable(color.Output, hubPtr.Tests)
case "json":
j, err := json.MarshalIndent(HubTest.Tests, " ", " ")
j, err := json.MarshalIndent(hubPtr.Tests, " ", " ")
if err != nil {
return err
}
@ -409,31 +450,34 @@ func NewHubTestListCmd() *cobra.Command {
},
}
return cmdHubTestList
return cmd
}
func NewHubTestCoverageCmd() *cobra.Command {
func (cli cliHubTest) NewCoverageCmd() *cobra.Command {
var showParserCov bool
var showScenarioCov bool
var showOnlyPercent bool
var showAppsecCov bool
var cmdHubTestCoverage = &cobra.Command{
cmd := &cobra.Command{
Use: "coverage",
Short: "coverage",
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
//for this one we explicitly don't do for appsec
if err := HubTest.LoadAllTests(); err != nil {
return fmt.Errorf("unable to load all tests: %+v", err)
}
var err error
scenarioCoverage := []hubtest.ScenarioCoverage{}
parserCoverage := []hubtest.ParserCoverage{}
scenarioCoverage := []hubtest.Coverage{}
parserCoverage := []hubtest.Coverage{}
appsecRuleCoverage := []hubtest.Coverage{}
scenarioCoveragePercent := 0
parserCoveragePercent := 0
appsecRuleCoveragePercent := 0
// if both are false (flag by default), show both
showAll := !showScenarioCov && !showParserCov
showAll := !showScenarioCov && !showParserCov && !showAppsecCov
if showParserCov || showAll {
parserCoverage, err = HubTest.GetParsersCoverage()
@ -443,7 +487,7 @@ func NewHubTestCoverageCmd() *cobra.Command {
parserTested := 0
for _, test := range parserCoverage {
if test.TestsCount > 0 {
parserTested += 1
parserTested++
}
}
parserCoveragePercent = int(math.Round((float64(parserTested) / float64(len(parserCoverage)) * 100)))
@ -454,27 +498,47 @@ func NewHubTestCoverageCmd() *cobra.Command {
if err != nil {
return fmt.Errorf("while getting scenario coverage: %s", err)
}
scenarioTested := 0
for _, test := range scenarioCoverage {
if test.TestsCount > 0 {
scenarioTested += 1
scenarioTested++
}
}
scenarioCoveragePercent = int(math.Round((float64(scenarioTested) / float64(len(scenarioCoverage)) * 100)))
}
if showAppsecCov || showAll {
appsecRuleCoverage, err = HubTest.GetAppsecCoverage()
if err != nil {
return fmt.Errorf("while getting scenario coverage: %s", err)
}
appsecRuleTested := 0
for _, test := range appsecRuleCoverage {
if test.TestsCount > 0 {
appsecRuleTested++
}
}
appsecRuleCoveragePercent = int(math.Round((float64(appsecRuleTested) / float64(len(appsecRuleCoverage)) * 100)))
}
if showOnlyPercent {
if showAll {
fmt.Printf("parsers=%d%%\nscenarios=%d%%", parserCoveragePercent, scenarioCoveragePercent)
fmt.Printf("parsers=%d%%\nscenarios=%d%%\nappsec_rules=%d%%", parserCoveragePercent, scenarioCoveragePercent, appsecRuleCoveragePercent)
} else if showParserCov {
fmt.Printf("parsers=%d%%", parserCoveragePercent)
} else if showScenarioCov {
fmt.Printf("scenarios=%d%%", scenarioCoveragePercent)
} else if showAppsecCov {
fmt.Printf("appsec_rules=%d%%", appsecRuleCoveragePercent)
}
os.Exit(0)
}
if csConfig.Cscli.Output == "human" {
switch csConfig.Cscli.Output {
case "human":
if showParserCov || showAll {
hubTestParserCoverageTable(color.Output, parserCoverage)
}
@ -482,6 +546,11 @@ func NewHubTestCoverageCmd() *cobra.Command {
if showScenarioCov || showAll {
hubTestScenarioCoverageTable(color.Output, scenarioCoverage)
}
if showAppsecCov || showAll {
hubTestAppsecRuleCoverageTable(color.Output, appsecRuleCoverage)
}
fmt.Println()
if showParserCov || showAll {
fmt.Printf("PARSERS : %d%% of coverage\n", parserCoveragePercent)
@ -489,7 +558,10 @@ func NewHubTestCoverageCmd() *cobra.Command {
if showScenarioCov || showAll {
fmt.Printf("SCENARIOS : %d%% of coverage\n", scenarioCoveragePercent)
}
} else if csConfig.Cscli.Output == "json" {
if showAppsecCov || showAll {
fmt.Printf("APPSEC RULES : %d%% of coverage\n", appsecRuleCoveragePercent)
}
case "json":
dump, err := json.MarshalIndent(parserCoverage, "", " ")
if err != nil {
return err
@ -500,56 +572,66 @@ func NewHubTestCoverageCmd() *cobra.Command {
return err
}
fmt.Printf("%s", dump)
} else {
dump, err = json.MarshalIndent(appsecRuleCoverage, "", " ")
if err != nil {
return err
}
fmt.Printf("%s", dump)
default:
return fmt.Errorf("only human/json output modes are supported")
}
return nil
},
}
cmdHubTestCoverage.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
cmdHubTestCoverage.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
cmdHubTestCoverage.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
return cmdHubTestCoverage
cmd.PersistentFlags().BoolVar(&showOnlyPercent, "percent", false, "Show only percentages of coverage")
cmd.PersistentFlags().BoolVar(&showParserCov, "parsers", false, "Show only parsers coverage")
cmd.PersistentFlags().BoolVar(&showScenarioCov, "scenarios", false, "Show only scenarios coverage")
cmd.PersistentFlags().BoolVar(&showAppsecCov, "appsec", false, "Show only appsec coverage")
return cmd
}
func NewHubTestEvalCmd() *cobra.Command {
func (cli cliHubTest) NewEvalCmd() *cobra.Command {
var evalExpression string
var cmdHubTestEval = &cobra.Command{
cmd := &cobra.Command{
Use: "eval",
Short: "eval [test_name]",
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
for _, testName := range args {
test, err := HubTest.LoadTestItem(testName)
test, err := hubPtr.LoadTestItem(testName)
if err != nil {
return fmt.Errorf("can't load test: %+v", err)
}
err = test.ParserAssert.LoadTest(test.ParserResultFile)
if err != nil {
return fmt.Errorf("can't load test results from '%s': %+v", test.ParserResultFile, err)
}
output, err := test.ParserAssert.EvalExpression(evalExpression)
if err != nil {
return err
}
fmt.Print(output)
}
return nil
},
}
cmdHubTestEval.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval")
return cmdHubTestEval
cmd.PersistentFlags().StringVarP(&evalExpression, "expr", "e", "", "Expression to eval")
return cmd
}
func NewHubTestExplainCmd() *cobra.Command {
var cmdHubTestExplain = &cobra.Command{
func (cli cliHubTest) NewExplainCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "explain",
Short: "explain [test_name]",
Args: cobra.ExactArgs(1),
@ -562,24 +644,22 @@ func NewHubTestExplainCmd() *cobra.Command {
}
err = test.ParserAssert.LoadTest(test.ParserResultFile)
if err != nil {
err := test.Run()
if err != nil {
if err = test.Run(); err != nil {
return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
}
err = test.ParserAssert.LoadTest(test.ParserResultFile)
if err != nil {
if err = test.ParserAssert.LoadTest(test.ParserResultFile); err != nil {
return fmt.Errorf("unable to load parser result after run: %s", err)
}
}
err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
if err != nil {
err := test.Run()
if err != nil {
if err = test.Run(); err != nil {
return fmt.Errorf("running test '%s' failed: %+v", test.Name, err)
}
err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile)
if err != nil {
if err = test.ScenarioAssert.LoadTest(test.ScenarioResultFile, test.BucketPourResultFile); err != nil {
return fmt.Errorf("unable to load scenario result after run: %s", err)
}
}
@ -591,5 +671,5 @@ func NewHubTestExplainCmd() *cobra.Command {
},
}
return cmdHubTestExplain
return cmd
}

30
cmd/crowdsec-cli/hubtest_table.go
View file

@ -41,39 +41,61 @@ func hubTestListTable(out io.Writer, tests []*hubtest.HubTestItem) {
t.Render()
}
func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.ParserCoverage) {
func hubTestParserCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
t := newLightTable(out)
t.SetHeaders("Parser", "Status", "Number of tests")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
parserTested := 0
for _, test := range coverage {
status := emoji.RedCircle.String()
if test.TestsCount > 0 {
status = emoji.GreenCircle.String()
parserTested++
}
t.AddRow(test.Parser, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
}
t.Render()
}
func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.ScenarioCoverage) {
func hubTestAppsecRuleCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
t := newLightTable(out)
t.SetHeaders("Appsec Rule", "Status", "Number of tests")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
parserTested := 0
for _, test := range coverage {
status := emoji.RedCircle.String()
if test.TestsCount > 0 {
status = emoji.GreenCircle.String()
parserTested++
}
t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
}
t.Render()
}
func hubTestScenarioCoverageTable(out io.Writer, coverage []hubtest.Coverage) {
t := newLightTable(out)
t.SetHeaders("Scenario", "Status", "Number of tests")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft)
parserTested := 0
for _, test := range coverage {
status := emoji.RedCircle.String()
if test.TestsCount > 0 {
status = emoji.GreenCircle.String()
parserTested++
}
t.AddRow(test.Scenario, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
t.AddRow(test.Name, status, fmt.Sprintf("%d times (across %d tests)", test.TestsCount, len(test.PresentIn)))
}
t.Render()

238
cmd/crowdsec-cli/item_metrics.go Normal file
View file

@ -0,0 +1,238 @@
package main
import (
"fmt"
"math"
"net/http"
"strconv"
"strings"
"time"
"github.com/fatih/color"
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/prom2json"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func ShowMetrics(hubItem *cwhub.Item) error {
switch hubItem.Type {
case cwhub.PARSERS:
metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
parserMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.SCENARIOS:
metrics := GetScenarioMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
scenarioMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.COLLECTIONS:
for _, sub := range hubItem.SubItems() {
if err := ShowMetrics(sub); err != nil {
return err
}
}
case cwhub.APPSEC_RULES:
log.Error("FIXME: not implemented yet")
default:
// no metrics for this item type
}
return nil
}
// GetParserMetric is a complete rip from prom2json
func GetParserMetric(url string, itemName string) map[string]map[string]int {
stats := make(map[string]map[string]int)
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
source, ok := metric.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v", metric.Labels)
} else {
if srctype, ok := metric.Labels["type"]; ok {
source = srctype + ":" + source
}
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_reader_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
stats[source]["parsed"] = 0
stats[source]["reads"] = 0
stats[source]["unparsed"] = 0
stats[source]["hits"] = 0
}
stats[source]["reads"] += ival
case "cs_parser_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_parser_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
case "cs_node_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["hits"] += ival
case "cs_node_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_node_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
default:
continue
}
}
}
return stats
}
func GetScenarioMetric(url string, itemName string) map[string]int {
stats := make(map[string]int)
stats["instantiation"] = 0
stats["curr_count"] = 0
stats["overflow"] = 0
stats["pour"] = 0
stats["underflow"] = 0
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_bucket_created_total":
stats["instantiation"] += ival
case "cs_buckets":
stats["curr_count"] += ival
case "cs_bucket_overflowed_total":
stats["overflow"] += ival
case "cs_bucket_poured_total":
stats["pour"] += ival
case "cs_bucket_underflowed_total":
stats["underflow"] += ival
default:
continue
}
}
}
return stats
}
func GetPrometheusMetric(url string) []*prom2json.Family {
mfChan := make(chan *dto.MetricFamily, 1024)
// Start with the DefaultTransport for sane defaults.
transport := http.DefaultTransport.(*http.Transport).Clone()
// Conservatively disable HTTP keep-alives as this program will only
// ever need a single HTTP request.
transport.DisableKeepAlives = true
// Timeout early if the server doesn't even return the headers.
transport.ResponseHeaderTimeout = time.Minute
go func() {
defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
err := prom2json.FetchMetricFamilies(url, mfChan, transport)
if err != nil {
log.Fatalf("failed to fetch prometheus metrics : %v", err)
}
}()
result := []*prom2json.Family{}
for mf := range mfChan {
result = append(result, prom2json.NewFamily(mf))
}
log.Debugf("Finished reading prometheus output, %d entries", len(result))
return result
}
type unit struct {
value int64
symbol string
}
var ranges = []unit{
{value: 1e18, symbol: "E"},
{value: 1e15, symbol: "P"},
{value: 1e12, symbol: "T"},
{value: 1e9, symbol: "G"},
{value: 1e6, symbol: "M"},
{value: 1e3, symbol: "k"},
{value: 1, symbol: ""},
}
func formatNumber(num int) string {
goodUnit := unit{}
for _, u := range ranges {
if int64(num) >= u.value {
goodUnit = u
break
}
}
if goodUnit.value == 1 {
return fmt.Sprintf("%d%s", num, goodUnit.symbol)
}
res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
}

85
cmd/crowdsec-cli/item_suggest.go Normal file
View file

@ -0,0 +1,85 @@
package main
import (
"fmt"
"strings"
"github.com/agext/levenshtein"
"github.com/spf13/cobra"
"slices"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
// suggestNearestMessage returns a message with the most similar item name, if one is found
func suggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string {
const maxDistance = 7
score := 100
nearest := ""
for _, item := range hub.GetItemMap(itemType) {
d := levenshtein.Distance(itemName, item.Name, nil)
if d < score {
score = d
nearest = item.Name
}
}
msg := fmt.Sprintf("can't find '%s' in %s", itemName, itemType)
if score < maxDistance {
msg += fmt.Sprintf(", did you mean '%s'?", nearest)
}
return msg
}
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
hub, err := require.Hub(csConfig, nil)
if err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
for _, item := range hub.GetItemMap(itemType) {
if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
comp = append(comp, item.Name)
}
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
hub, err := require.Hub(csConfig, nil)
if err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
items, err := hub.GetInstalledItemNames(itemType)
if err != nil {
cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
if toComplete != "" {
for _, item := range items {
if strings.Contains(item, toComplete) {
comp = append(comp, item)
}
}
} else {
comp = items
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}

454
cmd/crowdsec-cli/itemcli.go Normal file
View file

@ -0,0 +1,454 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/coalesce"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
type cliHelp struct {
// Example is required, the others have a default value
// generated from the item type
use string
short string
long string
example string
}
type cliItem struct {
name string // plural, as used in the hub index
singular string
oneOrMore string // parenthetical pluralizaion: "parser(s)"
help cliHelp
installHelp cliHelp
removeHelp cliHelp
upgradeHelp cliHelp
inspectHelp cliHelp
inspectDetail func(item *cwhub.Item) error
listHelp cliHelp
}
func (it cliItem) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: coalesce.String(it.help.use, fmt.Sprintf("%s <action> [item]...", it.name)),
Short: coalesce.String(it.help.short, fmt.Sprintf("Manage hub %s", it.name)),
Long: it.help.long,
Example: it.help.example,
Args: cobra.MinimumNArgs(1),
Aliases: []string{it.singular},
DisableAutoGenTag: true,
}
cmd.AddCommand(it.NewInstallCmd())
cmd.AddCommand(it.NewRemoveCmd())
cmd.AddCommand(it.NewUpgradeCmd())
cmd.AddCommand(it.NewInspectCmd())
cmd.AddCommand(it.NewListCmd())
return cmd
}
func (it cliItem) Install(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
downloadOnly, err := flags.GetBool("download-only")
if err != nil {
return err
}
force, err := flags.GetBool("force")
if err != nil {
return err
}
ignoreError, err := flags.GetBool("ignore")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
for _, name := range args {
item := hub.GetItem(it.name, name)
if item == nil {
msg := suggestNearestMessage(hub, it.name, name)
if !ignoreError {
return fmt.Errorf(msg)
}
log.Errorf(msg)
continue
}
if err := item.Install(force, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", item.Name, err)
}
log.Errorf("Error while installing '%s': %s", item.Name, err)
}
}
log.Infof(ReloadMessage())
return nil
}
func (it cliItem) NewInstallCmd() *cobra.Command {
cmd := &cobra.Command{
Use: coalesce.String(it.installHelp.use, "install [item]..."),
Short: coalesce.String(it.installHelp.short, fmt.Sprintf("Install given %s", it.oneOrMore)),
Long: coalesce.String(it.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", it.name)),
Example: it.installHelp.example,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(it.name, args, toComplete)
},
RunE: it.Install,
}
flags := cmd.Flags()
flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
flags.Bool("ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", it.name))
return cmd
}
// return the names of the installed parents of an item, used to check if we can remove it
func istalledParentNames(item *cwhub.Item) []string {
ret := make([]string, 0)
for _, parent := range item.Ancestors() {
if parent.State.Installed {
ret = append(ret, parent.Name)
}
}
return ret
}
func (it cliItem) Remove(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
purge, err := flags.GetBool("purge")
if err != nil {
return err
}
force, err := flags.GetBool("force")
if err != nil {
return err
}
all, err := flags.GetBool("all")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
if all {
getter := hub.GetInstalledItems
if purge {
getter = hub.GetAllItems
}
items, err := getter(it.name)
if err != nil {
return err
}
removed := 0
for _, item := range items {
didRemove, err := item.Remove(purge, force)
if err != nil {
return err
}
if didRemove {
log.Infof("Removed %s", item.Name)
removed++
}
}
log.Infof("Removed %d %s", removed, it.name)
if removed > 0 {
log.Infof(ReloadMessage())
}
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular)
}
removed := 0
for _, itemName := range args {
item := hub.GetItem(it.name, itemName)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
}
parents := istalledParentNames(item)
if !force && len(parents) > 0 {
log.Warningf("%s belongs to collections: %s", item.Name, parents)
log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular)
continue
}
didRemove, err := item.Remove(purge, force)
if err != nil {
return err
}
if didRemove {
log.Infof("Removed %s", item.Name)
removed++
}
}
log.Infof("Removed %d %s", removed, it.name)
if removed > 0 {
log.Infof(ReloadMessage())
}
return nil
}
func (it cliItem) NewRemoveCmd() *cobra.Command {
cmd := &cobra.Command{
Use: coalesce.String(it.removeHelp.use, "remove [item]..."),
Short: coalesce.String(it.removeHelp.short, fmt.Sprintf("Remove given %s", it.oneOrMore)),
Long: coalesce.String(it.removeHelp.long, fmt.Sprintf("Remove one or more %s", it.name)),
Example: it.removeHelp.example,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(it.name, args, toComplete)
},
RunE: it.Remove,
}
flags := cmd.Flags()
flags.Bool("purge", false, "Delete source file too")
flags.Bool("force", false, "Force remove: remove tainted and outdated files")
flags.Bool("all", false, fmt.Sprintf("Remove all the %s", it.name))
return cmd
}
func (it cliItem) Upgrade(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
force, err := flags.GetBool("force")
if err != nil {
return err
}
all, err := flags.GetBool("all")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
if all {
items, err := hub.GetInstalledItems(it.name)
if err != nil {
return err
}
updated := 0
for _, item := range items {
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
updated++
}
}
log.Infof("Updated %d %s", updated, it.name)
if updated > 0 {
log.Infof(ReloadMessage())
}
return nil
}
if len(args) == 0 {
return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular)
}
updated := 0
for _, itemName := range args {
item := hub.GetItem(it.name, itemName)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
}
didUpdate, err := item.Upgrade(force)
if err != nil {
return err
}
if didUpdate {
log.Infof("Updated %s", item.Name)
updated++
}
}
if updated > 0 {
log.Infof(ReloadMessage())
}
return nil
}
func (it cliItem) NewUpgradeCmd() *cobra.Command {
cmd := &cobra.Command{
Use: coalesce.String(it.upgradeHelp.use, "upgrade [item]..."),
Short: coalesce.String(it.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", it.oneOrMore)),
Long: coalesce.String(it.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", it.name)),
Example: it.upgradeHelp.example,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(it.name, args, toComplete)
},
RunE: it.Upgrade,
}
flags := cmd.Flags()
flags.BoolP("all", "a", false, fmt.Sprintf("Upgrade all the %s", it.name))
flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
return cmd
}
func (it cliItem) Inspect(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
url, err := flags.GetString("url")
if err != nil {
return err
}
if url != "" {
csConfig.Cscli.PrometheusUrl = url
}
noMetrics, err := flags.GetBool("no-metrics")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
for _, name := range args {
item := hub.GetItem(it.name, name)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", name, it.name)
}
if err = InspectItem(item, !noMetrics); err != nil {
return err
}
if it.inspectDetail != nil {
if err = it.inspectDetail(item); err != nil {
return err
}
}
}
return nil
}
func (it cliItem) NewInspectCmd() *cobra.Command {
cmd := &cobra.Command{
Use: coalesce.String(it.inspectHelp.use, "inspect [item]..."),
Short: coalesce.String(it.inspectHelp.short, fmt.Sprintf("Inspect given %s", it.oneOrMore)),
Long: coalesce.String(it.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", it.name)),
Example: it.inspectHelp.example,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(it.name, args, toComplete)
},
RunE: it.Inspect,
}
flags := cmd.Flags()
flags.StringP("url", "u", "", "Prometheus url")
flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
return cmd
}
func (it cliItem) List(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
all, err := flags.GetBool("all")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
items := make(map[string][]*cwhub.Item)
items[it.name], err = selectItems(hub, it.name, args, !all)
if err != nil {
return err
}
if err = listItems(color.Output, []string{it.name}, items, false); err != nil {
return err
}
return nil
}
func (it cliItem) NewListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: coalesce.String(it.listHelp.use, "list [item... | -a]"),
Short: coalesce.String(it.listHelp.short, fmt.Sprintf("List %s", it.oneOrMore)),
Long: coalesce.String(it.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", it.name)),
Example: it.listHelp.example,
DisableAutoGenTag: true,
RunE: it.List,
}
flags := cmd.Flags()
flags.BoolP("all", "a", false, "List disabled items as well")
return cmd
}

165
cmd/crowdsec-cli/items.go Normal file
View file

@ -0,0 +1,165 @@
package main
import (
"encoding/csv"
"encoding/json"
"fmt"
"io"
"os"
"strings"
"gopkg.in/yaml.v3"
"slices"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
// selectItems returns a slice of items of a given type, selected by name and sorted by case-insensitive name
func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]*cwhub.Item, error) {
itemNames := hub.GetItemNames(itemType)
notExist := []string{}
if len(args) > 0 {
for _, arg := range args {
if !slices.Contains(itemNames, arg) {
notExist = append(notExist, arg)
}
}
}
if len(notExist) > 0 {
return nil, fmt.Errorf("item(s) '%s' not found in %s", strings.Join(notExist, ", "), itemType)
}
if len(args) > 0 {
itemNames = args
installedOnly = false
}
items := make([]*cwhub.Item, 0, len(itemNames))
for _, itemName := range itemNames {
item := hub.GetItem(itemType, itemName)
if installedOnly && !item.State.Installed {
continue
}
items = append(items, item)
}
cwhub.SortItemSlice(items)
return items, nil
}
func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item, omitIfEmpty bool) error {
switch csConfig.Cscli.Output {
case "human":
nothingToDisplay := true
for _, itemType := range itemTypes {
if omitIfEmpty && len(items[itemType]) == 0 {
continue
}
listHubItemTable(out, "\n"+strings.ToUpper(itemType), items[itemType])
nothingToDisplay = false
}
if nothingToDisplay {
fmt.Println("No items to display")
}
case "json":
type itemHubStatus struct {
Name string `json:"name"`
LocalVersion string `json:"local_version"`
LocalPath string `json:"local_path"`
Description string `json:"description"`
UTF8Status string `json:"utf8_status"`
Status string `json:"status"`
}
hubStatus := make(map[string][]itemHubStatus)
for _, itemType := range itemTypes {
// empty slice in case there are no items of this type
hubStatus[itemType] = make([]itemHubStatus, len(items[itemType]))
for i, item := range items[itemType] {
status := item.State.Text()
status_emo := item.State.Emoji()
hubStatus[itemType][i] = itemHubStatus{
Name: item.Name,
LocalVersion: item.State.LocalVersion,
LocalPath: item.State.LocalPath,
Description: item.Description,
Status: status,
UTF8Status: fmt.Sprintf("%v %s", status_emo, status),
}
}
}
x, err := json.MarshalIndent(hubStatus, "", " ")
if err != nil {
return fmt.Errorf("failed to unmarshal: %w", err)
}
out.Write(x)
case "raw":
csvwriter := csv.NewWriter(out)
header := []string{"name", "status", "version", "description"}
if len(itemTypes) > 1 {
header = append(header, "type")
}
if err := csvwriter.Write(header); err != nil {
return fmt.Errorf("failed to write header: %s", err)
}
for _, itemType := range itemTypes {
for _, item := range items[itemType] {
row := []string{
item.Name,
item.State.Text(),
item.State.LocalVersion,
item.Description,
}
if len(itemTypes) > 1 {
row = append(row, itemType)
}
if err := csvwriter.Write(row); err != nil {
return fmt.Errorf("failed to write raw output: %s", err)
}
}
}
csvwriter.Flush()
default:
return fmt.Errorf("unknown output format '%s'", csConfig.Cscli.Output)
}
return nil
}
func InspectItem(item *cwhub.Item, showMetrics bool) error {
switch csConfig.Cscli.Output {
case "human", "raw":
enc := yaml.NewEncoder(os.Stdout)
enc.SetIndent(2)
if err := enc.Encode(item); err != nil {
return fmt.Errorf("unable to encode item: %s", err)
}
case "json":
b, err := json.MarshalIndent(*item, "", " ")
if err != nil {
return fmt.Errorf("unable to marshal item: %s", err)
}
fmt.Print(string(b))
}
if csConfig.Cscli.Output == "human" && showMetrics {
fmt.Printf("\nCurrent metrics: \n")
if err := ShowMetrics(item); err != nil {
return err
}
}
return nil
}

256
cmd/crowdsec-cli/lapi.go
View file

@ -2,6 +2,7 @@ package main
import (
"context"
"errors"
"fmt"
"net/url"
"os"
@ -11,11 +12,12 @@ import (
"github.com/go-openapi/strfmt"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/exp/slices"
"gopkg.in/yaml.v2"
"slices"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
@ -25,28 +27,24 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/parser"
)
var LAPIURLPrefix string = "v1"
const LAPIURLPrefix = "v1"
func runLapiStatus(cmd *cobra.Command, args []string) error {
var err error
password := strfmt.Password(csConfig.API.Client.Credentials.Password)
apiurl, err := url.Parse(csConfig.API.Client.Credentials.URL)
login := csConfig.API.Client.Credentials.Login
if err != nil {
log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
}
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
return fmt.Errorf("parsing api url: %w", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to load hub index : %s", err)
}
scenarios, err := cwhub.GetInstalledScenariosAsString()
hub, err := require.Hub(csConfig, nil)
if err != nil {
log.Fatalf("failed to get scenarios : %s", err)
return err
}
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("failed to get scenarios: %w", err)
}
Client, err = apiclient.NewDefaultClient(apiurl,
@ -54,28 +52,27 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
fmt.Sprintf("crowdsec/%s", version.String()),
nil)
if err != nil {
log.Fatalf("init default client: %s", err)
return fmt.Errorf("init default client: %w", err)
}
t := models.WatcherAuthRequest{
MachineID: &login,
Password: &password,
Scenarios: scenarios,
}
log.Infof("Loaded credentials from %s", csConfig.API.Client.CredentialsFilePath)
log.Infof("Trying to authenticate with username %s on %s", login, apiurl)
_, _, err = Client.Auth.AuthenticateWatcher(context.Background(), t)
if err != nil {
log.Fatalf("Failed to authenticate to Local API (LAPI) : %s", err)
} else {
log.Infof("You can successfully interact with Local API (LAPI)")
return fmt.Errorf("failed to authenticate to Local API (LAPI): %w", err)
}
log.Infof("You can successfully interact with Local API (LAPI)")
return nil
}
func runLapiRegister(cmd *cobra.Command, args []string) error {
var err error
flags := cmd.Flags()
apiURL, err := flags.GetString("url")
@ -96,13 +93,13 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
if lapiUser == "" {
lapiUser, err = generateID("")
if err != nil {
log.Fatalf("unable to generate machine id: %s", err)
return fmt.Errorf("unable to generate machine id: %w", err)
}
}
password := strfmt.Password(generatePassword(passwordLength))
apiurl, err := prepareApiURl(csConfig.API.Client, apiURL)
apiurl, err := prepareApiURL(csConfig.API.Client, apiURL)
if err != nil {
log.Fatalf("parsing api url: %s", err)
return fmt.Errorf("parsing api url: %w", err)
}
_, err = apiclient.RegisterClient(&apiclient.Config{
MachineID: lapiUser,
@ -113,7 +110,7 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
}, nil)
if err != nil {
log.Fatalf("api client register: %s", err)
return fmt.Errorf("api client register: %w", err)
}
log.Printf("Successfully registered to Local API (LAPI)")
@ -133,14 +130,14 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
}
apiConfigDump, err := yaml.Marshal(apiCfg)
if err != nil {
log.Fatalf("unable to marshal api credentials: %s", err)
return fmt.Errorf("unable to marshal api credentials: %w", err)
}
if dumpFile != "" {
err = os.WriteFile(dumpFile, apiConfigDump, 0644)
err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
if err != nil {
log.Fatalf("write api credentials in '%s' failed: %s", dumpFile, err)
return fmt.Errorf("write api credentials to '%s' failed: %w", dumpFile, err)
}
log.Printf("Local API credentials dumped to '%s'", dumpFile)
log.Printf("Local API credentials written to '%s'", dumpFile)
} else {
fmt.Printf("%s\n", string(apiConfigDump))
}
@ -149,13 +146,12 @@ func runLapiRegister(cmd *cobra.Command, args []string) error {
return nil
}
func prepareApiURl(clientConfig *csconfig.LocalApiClientCfg, apiURL string) (*url.URL, error) {
func prepareApiURL(clientConfig *csconfig.LocalApiClientCfg, apiURL string) (*url.URL, error) {
if apiURL == "" {
if clientConfig != nil && clientConfig.Credentials != nil && clientConfig.Credentials.URL != "" {
apiURL = clientConfig.Credentials.URL
} else {
return nil, fmt.Errorf("No Local API URL. Please provide it in your configuration or with the -u parameter")
if clientConfig == nil || clientConfig.Credentials == nil || clientConfig.Credentials.URL == "" {
return nil, fmt.Errorf("no Local API URL. Please provide it in your configuration or with the -u parameter")
}
apiURL = clientConfig.Credentials.URL
}
if !strings.HasSuffix(apiURL, "/") {
@ -184,7 +180,7 @@ func NewLapiRegisterCmd() *cobra.Command {
cmdLapiRegister := &cobra.Command{
Use: "register",
Short: "Register a machine to Local API (LAPI)",
Long: `Register you machine to the Local API (LAPI).
Long: `Register your machine to the Local API (LAPI).
Keep in mind the machine needs to be validated by an administrator on LAPI side to be effective.`,
Args: cobra.MinimumNArgs(0),
DisableAutoGenTag: true,
@ -200,7 +196,7 @@ Keep in mind the machine needs to be validated by an administrator on LAPI side
}
func NewLapiCmd() *cobra.Command {
var cmdLapi = &cobra.Command{
cmdLapi := &cobra.Command{
Use: "lapi [action]",
Short: "Manage interaction with Local API (LAPI)",
Args: cobra.MinimumNArgs(1),
@ -220,6 +216,30 @@ func NewLapiCmd() *cobra.Command {
return cmdLapi
}
func AddContext(key string, values []string) error {
if err := alertcontext.ValidateContextExpr(key, values); err != nil {
return fmt.Errorf("invalid context configuration :%s", err)
}
if _, ok := csConfig.Crowdsec.ContextToSend[key]; !ok {
csConfig.Crowdsec.ContextToSend[key] = make([]string, 0)
log.Infof("key '%s' added", key)
}
data := csConfig.Crowdsec.ContextToSend[key]
for _, val := range values {
if !slices.Contains(data, val) {
log.Infof("value '%s' added to key '%s'", val, key)
data = append(data, val)
}
csConfig.Crowdsec.ContextToSend[key] = data
}
if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
return err
}
return nil
}
func NewLapiContextCmd() *cobra.Command {
cmdContext := &cobra.Command{
Use: "context [command]",
@ -229,11 +249,11 @@ func NewLapiContextCmd() *cobra.Command {
if err := csConfig.LoadCrowdsec(); err != nil {
fileNotFoundMessage := fmt.Sprintf("failed to open context file: open %s: no such file or directory", csConfig.Crowdsec.ConsoleContextPath)
if err.Error() != fileNotFoundMessage {
log.Fatalf("Unable to load CrowdSec Agent: %s", err)
return fmt.Errorf("unable to start CrowdSec agent: %w", err)
}
}
if csConfig.DisableAgent {
log.Fatalf("Agent is disabled and lapi context can only be used on the agent")
return errors.New("agent is disabled and lapi context can only be used on the agent")
}
return nil
@ -250,32 +270,40 @@ func NewLapiContextCmd() *cobra.Command {
Short: "Add context to send with alerts. You must specify the output key with the expr value you want",
Example: `cscli lapi context add --key source_ip --value evt.Meta.source_ip
cscli lapi context add --key file_source --value evt.Line.Src
cscli lapi context add --value evt.Meta.source_ip --value evt.Meta.target_user
`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if err := alertcontext.ValidateContextExpr(keyToAdd, valuesToAdd); err != nil {
log.Fatalf("invalid context configuration :%s", err)
RunE: func(cmd *cobra.Command, args []string) error {
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
if _, ok := csConfig.Crowdsec.ContextToSend[keyToAdd]; !ok {
csConfig.Crowdsec.ContextToSend[keyToAdd] = make([]string, 0)
log.Infof("key '%s' added", keyToAdd)
if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil {
return fmt.Errorf("while loading context: %w", err)
}
data := csConfig.Crowdsec.ContextToSend[keyToAdd]
for _, val := range valuesToAdd {
if !slices.Contains(data, val) {
log.Infof("value '%s' added to key '%s'", val, keyToAdd)
data = append(data, val)
if keyToAdd != "" {
if err := AddContext(keyToAdd, valuesToAdd); err != nil {
return err
}
csConfig.Crowdsec.ContextToSend[keyToAdd] = data
return nil
}
if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
log.Fatalf(err.Error())
for _, v := range valuesToAdd {
keySlice := strings.Split(v, ".")
key := keySlice[len(keySlice)-1]
value := []string{v}
if err := AddContext(key, value); err != nil {
return err
}
}
return nil
},
}
cmdContextAdd.Flags().StringVarP(&keyToAdd, "key", "k", "", "The key of the different values to send")
cmdContextAdd.Flags().StringSliceVar(&valuesToAdd, "value", []string{}, "The expr fields to associate with the key")
cmdContextAdd.MarkFlagRequired("key")
cmdContextAdd.MarkFlagRequired("value")
cmdContext.AddCommand(cmdContextAdd)
@ -283,19 +311,29 @@ cscli lapi context add --key file_source --value evt.Line.Src
Use: "status",
Short: "List context to send with alerts",
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
RunE: func(cmd *cobra.Command, args []string) error {
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
if err = alertcontext.LoadConsoleContext(csConfig, hub); err != nil {
return fmt.Errorf("while loading context: %w", err)
}
if len(csConfig.Crowdsec.ContextToSend) == 0 {
fmt.Println("No context found on this agent. You can use 'cscli lapi context add' to add context to your alerts.")
return
return nil
}
dump, err := yaml.Marshal(csConfig.Crowdsec.ContextToSend)
if err != nil {
log.Fatalf("unable to show context status: %s", err)
return fmt.Errorf("unable to show context status: %w", err)
}
fmt.Println(string(dump))
fmt.Print(string(dump))
return nil
},
}
cmdContext.AddCommand(cmdContextStatus)
@ -308,30 +346,27 @@ cscli lapi context add --key file_source --value evt.Line.Src
cscli lapi context detect crowdsecurity/sshd-logs
`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
var err error
RunE: func(cmd *cobra.Command, args []string) error {
if !detectAll && len(args) == 0 {
log.Infof("Please provide parsers to detect or --all flag.")
printHelp(cmd)
}
// to avoid all the log.Info from the loaders functions
log.SetLevel(log.ErrorLevel)
log.SetLevel(log.WarnLevel)
err = exprhelpers.Init(nil)
if err := exprhelpers.Init(nil); err != nil {
return fmt.Errorf("failed to init expr helpers: %w", err)
}
hub, err := require.Hub(csConfig, nil)
if err != nil {
log.Fatalf("Failed to init expr helpers : %s", err)
return err
}
// Populate cwhub package tools
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to load hub index : %s", err)
}
csParsers := parser.NewParsers()
csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil {
log.Fatalf("unable to load parsers: %s", err)
return fmt.Errorf("unable to load parsers: %w", err)
}
fieldByParsers := make(map[string][]string)
@ -351,7 +386,6 @@ cscli lapi context detect crowdsecurity/sshd-logs
fieldByParsers[node.Name] = append(fieldByParsers[node.Name], field)
}
}
}
fmt.Printf("Acquisition :\n\n")
@ -384,59 +418,17 @@ cscli lapi context detect crowdsecurity/sshd-logs
log.Errorf("parser '%s' not found, can't detect fields", parserNotFound)
}
}
return nil
},
}
cmdContextDetect.Flags().BoolVarP(&detectAll, "all", "a", false, "Detect evt field for all installed parser")
cmdContext.AddCommand(cmdContextDetect)
var keysToDelete []string
var valuesToDelete []string
cmdContextDelete := &cobra.Command{
Use: "delete",
Short: "Delete context to send with alerts",
Example: `cscli lapi context delete --key source_ip
cscli lapi context delete --value evt.Line.Src
`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if len(keysToDelete) == 0 && len(valuesToDelete) == 0 {
log.Fatalf("please provide at least a key or a value to delete")
}
for _, key := range keysToDelete {
if _, ok := csConfig.Crowdsec.ContextToSend[key]; ok {
delete(csConfig.Crowdsec.ContextToSend, key)
log.Infof("key '%s' has been removed", key)
} else {
log.Warningf("key '%s' doesn't exist", key)
}
}
for _, value := range valuesToDelete {
valueFound := false
for key, context := range csConfig.Crowdsec.ContextToSend {
if slices.Contains(context, value) {
valueFound = true
csConfig.Crowdsec.ContextToSend[key] = removeFromSlice(value, context)
log.Infof("value '%s' has been removed from key '%s'", value, key)
}
if len(csConfig.Crowdsec.ContextToSend[key]) == 0 {
delete(csConfig.Crowdsec.ContextToSend, key)
}
}
if !valueFound {
log.Warningf("value '%s' not found", value)
}
}
if err := csConfig.Crowdsec.DumpContextConfigFile(); err != nil {
log.Fatalf(err.Error())
}
},
Deprecated: "please manually edit the context file.",
}
cmdContextDelete.Flags().StringSliceVarP(&keysToDelete, "key", "k", []string{}, "The keys to delete")
cmdContextDelete.Flags().StringSliceVar(&valuesToDelete, "value", []string{}, "The expr fields to delete")
cmdContext.AddCommand(cmdContextDelete)
return cmdContext
@ -444,6 +436,7 @@ cscli lapi context delete --value evt.Line.Src
func detectStaticField(GrokStatics []parser.ExtraField) []string {
ret := make([]string, 0)
for _, static := range GrokStatics {
if static.Parsed != "" {
fieldName := fmt.Sprintf("evt.Parsed.%s", static.Parsed)
@ -472,7 +465,8 @@ func detectStaticField(GrokStatics []parser.ExtraField) []string {
}
func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
var ret = make([]string, 0)
ret := make([]string, 0)
if node.Grok.RunTimeRegexp != nil {
for _, capturedField := range node.Grok.RunTimeRegexp.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
@ -484,13 +478,13 @@ func detectNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
if node.Grok.RegexpName != "" {
grokCompiled, err := parserCTX.Grok.Get(node.Grok.RegexpName)
if err != nil {
log.Warningf("Can't get subgrok: %s", err)
}
for _, capturedField := range grokCompiled.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
if !slices.Contains(ret, fieldName) {
ret = append(ret, fieldName)
// ignore error (parser does not exist?)
if err == nil {
for _, capturedField := range grokCompiled.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
if !slices.Contains(ret, fieldName) {
ret = append(ret, fieldName)
}
}
}
}
@ -530,13 +524,13 @@ func detectSubNode(node parser.Node, parserCTX parser.UnixParserCtx) []string {
}
if subnode.Grok.RegexpName != "" {
grokCompiled, err := parserCTX.Grok.Get(subnode.Grok.RegexpName)
if err != nil {
log.Warningf("Can't get subgrok: %s", err)
}
for _, capturedField := range grokCompiled.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
if !slices.Contains(ret, fieldName) {
ret = append(ret, fieldName)
if err == nil {
// ignore error (parser does not exist?)
for _, capturedField := range grokCompiled.Names() {
fieldName := fmt.Sprintf("evt.Parsed.%s", capturedField)
if !slices.Contains(ret, fieldName) {
ret = append(ret, fieldName)
}
}
}
}

268
cmd/crowdsec-cli/machines.go
View file

@ -8,6 +8,7 @@ import (
"io"
"math/big"
"os"
"slices"
"strings"
"time"
@ -17,7 +18,6 @@ import (
"github.com/google/uuid"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/exp/slices"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/machineid"
@ -26,11 +26,11 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/database/ent"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
var (
passwordLength = 64
)
const passwordLength = 64
func generatePassword(length int) string {
upper := "ABCDEFGHIJKLMNOPQRSTUVWXY"
@ -41,6 +41,7 @@ func generatePassword(length int) string {
charsetLength := len(charset)
buf := make([]byte, length)
for i := 0; i < length; i++ {
rInt, err := saferand.Int(saferand.Reader, big.NewInt(int64(charsetLength)))
if err != nil {
@ -141,23 +142,52 @@ func getAgents(out io.Writer, dbClient *database.Client) error {
return nil
}
func NewMachinesListCmd() *cobra.Command {
cmdMachinesList := &cobra.Command{
Use: "list",
Short: "List machines",
Long: `List `,
Example: `cscli machines list`,
Args: cobra.MaximumNArgs(1),
type cliMachines struct {}
func NewCLIMachines() *cliMachines {
return &cliMachines{}
}
func (cli cliMachines) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "machines [action]",
Short: "Manage local API machines [requires local API]",
Long: `To list/add/delete/validate/prune machines.
Note: This command requires database direct access, so is intended to be run on the local API machine.
`,
Example: `cscli machines [action]`,
DisableAutoGenTag: true,
PreRunE: func(cmd *cobra.Command, args []string) error {
Aliases: []string{"machine"},
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
var err error
if err := require.LAPI(csConfig); err != nil {
return err
}
dbClient, err = database.NewClient(csConfig.DbConfig)
if err != nil {
return fmt.Errorf("unable to create new database client: %s", err)
}
return nil
},
}
cmd.AddCommand(cli.NewListCmd())
cmd.AddCommand(cli.NewAddCmd())
cmd.AddCommand(cli.NewDeleteCmd())
cmd.AddCommand(cli.NewValidateCmd())
cmd.AddCommand(cli.NewPruneCmd())
return cmd
}
func (cli cliMachines) NewListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "list",
Short: "list all machines in the database",
Long: `list all machines in the database with their status and last heartbeat`,
Example: `cscli machines list`,
Args: cobra.NoArgs,
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
err := getAgents(color.Output, dbClient)
if err != nil {
@ -168,13 +198,13 @@ func NewMachinesListCmd() *cobra.Command {
},
}
return cmdMachinesList
return cmd
}
func NewMachinesAddCmd() *cobra.Command {
cmdMachinesAdd := &cobra.Command{
func (cli cliMachines) NewAddCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "add",
Short: "add machine to the database.",
Short: "add a single machine to the database",
DisableAutoGenTag: true,
Long: `Register a new machine in the database. cscli should be on the same machine as LAPI.`,
Example: `
@ -182,19 +212,10 @@ cscli machines add --auto
cscli machines add MyTestMachine --auto
cscli machines add MyTestMachine --password MyPassword
`,
PreRunE: func(cmd *cobra.Command, args []string) error {
var err error
dbClient, err = database.NewClient(csConfig.DbConfig)
if err != nil {
return fmt.Errorf("unable to create new database client: %s", err)
}
return nil
},
RunE: runMachinesAdd,
RunE: cli.add,
}
flags := cmdMachinesAdd.Flags()
flags := cmd.Flags()
flags.StringP("password", "p", "", "machine password to login to the API")
flags.StringP("file", "f", "", "output file destination (defaults to "+csconfig.DefaultConfigPath("local_api_credentials.yaml")+")")
flags.StringP("url", "u", "", "URL of the local API")
@ -202,13 +223,10 @@ cscli machines add MyTestMachine --password MyPassword
flags.BoolP("auto", "a", false, "automatically generate password (and username if not provided)")
flags.Bool("force", false, "will force add the machine if it already exist")
return cmdMachinesAdd
return cmd
}
func runMachinesAdd(cmd *cobra.Command, args []string) error {
var dumpFile string
var err error
func (cli cliMachines) add(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
machinePassword, err := flags.GetString("password")
@ -216,7 +234,7 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
return err
}
outputFile, err := flags.GetString("file")
dumpFile, err := flags.GetString("file")
if err != nil {
return err
}
@ -236,7 +254,7 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
return err
}
forceAdd, err := flags.GetBool("force")
force, err := flags.GetBool("force")
if err != nil {
return err
}
@ -258,17 +276,28 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
}
/*check if file already exists*/
if outputFile != "" {
dumpFile = outputFile
} else if csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" {
dumpFile = csConfig.API.Client.CredentialsFilePath
if dumpFile == "" && csConfig.API.Client != nil && csConfig.API.Client.CredentialsFilePath != "" {
credFile := csConfig.API.Client.CredentialsFilePath
// use the default only if the file does not exist
_, err := os.Stat(credFile)
switch {
case os.IsNotExist(err) || force:
dumpFile = csConfig.API.Client.CredentialsFilePath
case err != nil:
return fmt.Errorf("unable to stat '%s': %s", credFile, err)
default:
return fmt.Errorf(`credentials file '%s' already exists: please remove it, use "--force" or specify a different file with "-f" ("-f -" for standard output)`, credFile)
}
}
if dumpFile == "" {
return fmt.Errorf(`please specify a file to dump credentials to, with -f ("-f -" for standard output)`)
}
// create a password if it's not specified by user
if machinePassword == "" && !interactive {
if !autoAdd {
printHelp(cmd)
return nil
return fmt.Errorf("please specify a password with --password or use --auto")
}
machinePassword = generatePassword(passwordLength)
} else if machinePassword == "" && interactive {
@ -278,7 +307,7 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
survey.AskOne(qs, &machinePassword)
}
password := strfmt.Password(machinePassword)
_, err = dbClient.CreateMachine(&machineID, &password, "", true, forceAdd, types.PasswordAuthType)
_, err = dbClient.CreateMachine(&machineID, &password, "", true, force, types.PasswordAuthType)
if err != nil {
return fmt.Errorf("unable to create machine: %s", err)
}
@ -303,11 +332,11 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
return fmt.Errorf("unable to marshal api credentials: %s", err)
}
if dumpFile != "" && dumpFile != "-" {
err = os.WriteFile(dumpFile, apiConfigDump, 0644)
err = os.WriteFile(dumpFile, apiConfigDump, 0o600)
if err != nil {
return fmt.Errorf("write api credentials in '%s' failed: %s", dumpFile, err)
}
log.Printf("API credentials dumped to '%s'", dumpFile)
log.Printf("API credentials written to '%s'", dumpFile)
} else {
fmt.Printf("%s\n", string(apiConfigDump))
}
@ -315,29 +344,15 @@ func runMachinesAdd(cmd *cobra.Command, args []string) error {
return nil
}
func NewMachinesDeleteCmd() *cobra.Command {
cmdMachinesDelete := &cobra.Command{
func (cli cliMachines) NewDeleteCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "delete [machine_name]...",
Short: "delete machines",
Short: "delete machine(s) by name",
Example: `cscli machines delete "machine1" "machine2"`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"remove"},
DisableAutoGenTag: true,
PreRunE: func(cmd *cobra.Command, args []string) error {
var err error
dbClient, err = database.NewClient(csConfig.DbConfig)
if err != nil {
return fmt.Errorf("unable to create new database client: %s", err)
}
return nil
},
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
var err error
dbClient, err = getDBClient()
if err != nil {
cobra.CompError("unable to create new database client: " + err.Error())
return nil, cobra.ShellCompDirectiveNoFileComp
}
machines, err := dbClient.ListMachines()
if err != nil {
cobra.CompError("unable to list machines " + err.Error())
@ -350,13 +365,13 @@ func NewMachinesDeleteCmd() *cobra.Command {
}
return ret, cobra.ShellCompDirectiveNoFileComp
},
RunE: runMachinesDelete,
RunE: cli.delete,
}
return cmdMachinesDelete
return cmd
}
func runMachinesDelete(cmd *cobra.Command, args []string) error {
func (cli cliMachines) delete(cmd *cobra.Command, args []string) error {
for _, machineID := range args {
err := dbClient.DeleteWatcher(machineID)
if err != nil {
@ -369,23 +384,94 @@ func runMachinesDelete(cmd *cobra.Command, args []string) error {
return nil
}
func NewMachinesValidateCmd() *cobra.Command {
cmdMachinesValidate := &cobra.Command{
func (cli cliMachines) NewPruneCmd() *cobra.Command {
var parsedDuration time.Duration
cmd := &cobra.Command{
Use: "prune",
Short: "prune multiple machines from the database",
Long: `prune multiple machines that are not validated or have not connected to the local API in a given duration.`,
Example: `cscli machines prune
cscli machines prune --duration 1h
cscli machines prune --not-validated-only --force`,
Args: cobra.NoArgs,
DisableAutoGenTag: true,
PreRunE: func(cmd *cobra.Command, args []string) error {
dur, _ := cmd.Flags().GetString("duration")
var err error
parsedDuration, err = time.ParseDuration(fmt.Sprintf("-%s", dur))
if err != nil {
return fmt.Errorf("unable to parse duration '%s': %s", dur, err)
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
notValidOnly, _ := cmd.Flags().GetBool("not-validated-only")
force, _ := cmd.Flags().GetBool("force")
if parsedDuration >= 0-60*time.Second && !notValidOnly {
var answer bool
prompt := &survey.Confirm{
Message: "The duration you provided is less than or equal 60 seconds this can break installations do you want to continue ?",
Default: false,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask about prune check: %s", err)
}
if !answer {
fmt.Println("user aborted prune no changes were made")
return nil
}
}
machines := make([]*ent.Machine, 0)
if pending, err := dbClient.QueryPendingMachine(); err == nil {
machines = append(machines, pending...)
}
if !notValidOnly {
if pending, err := dbClient.QueryLastValidatedHeartbeatLT(time.Now().UTC().Add(parsedDuration)); err == nil {
machines = append(machines, pending...)
}
}
if len(machines) == 0 {
fmt.Println("no machines to prune")
return nil
}
getAgentsTable(color.Output, machines)
if !force {
var answer bool
prompt := &survey.Confirm{
Message: "You are about to PERMANENTLY remove the above machines from the database these will NOT be recoverable, continue ?",
Default: false,
}
if err := survey.AskOne(prompt, &answer); err != nil {
return fmt.Errorf("unable to ask about prune check: %s", err)
}
if !answer {
fmt.Println("user aborted prune no changes were made")
return nil
}
}
nbDeleted, err := dbClient.BulkDeleteWatchers(machines)
if err != nil {
return fmt.Errorf("unable to prune machines: %s", err)
}
fmt.Printf("successfully delete %d machines\n", nbDeleted)
return nil
},
}
cmd.Flags().StringP("duration", "d", "10m", "duration of time since validated machine last heartbeat")
cmd.Flags().Bool("not-validated-only", false, "only prune machines that are not validated")
cmd.Flags().Bool("force", false, "force prune without asking for confirmation")
return cmd
}
func (cli cliMachines) NewValidateCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "validate",
Short: "validate a machine to access the local API",
Long: `validate a machine to access the local API.`,
Example: `cscli machines validate "machine_name"`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
PreRunE: func(cmd *cobra.Command, args []string) error {
var err error
dbClient, err = database.NewClient(csConfig.DbConfig)
if err != nil {
return fmt.Errorf("unable to create new database client: %s", err)
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
machineID := args[0]
if err := dbClient.ValidateMachine(machineID); err != nil {
@ -397,35 +483,5 @@ func NewMachinesValidateCmd() *cobra.Command {
},
}
return cmdMachinesValidate
}
func NewMachinesCmd() *cobra.Command {
var cmdMachines = &cobra.Command{
Use: "machines [action]",
Short: "Manage local API machines [requires local API]",
Long: `To list/add/delete/validate machines.
Note: This command requires database direct access, so is intended to be run on the local API machine.
`,
Example: `cscli machines [action]`,
DisableAutoGenTag: true,
Aliases: []string{"machine"},
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
if err != nil {
log.Errorf("local api : %s", err)
}
return fmt.Errorf("local API is disabled, please run this command on the local API machine")
}
return nil
},
}
cmdMachines.AddCommand(NewMachinesListCmd())
cmdMachines.AddCommand(NewMachinesAddCmd())
cmdMachines.AddCommand(NewMachinesDeleteCmd())
cmdMachines.AddCommand(NewMachinesValidateCmd())
return cmdMachines
return cmd
}

94
cmd/crowdsec-cli/main.go
View file

@ -3,19 +3,18 @@ package main
import (
"fmt"
"os"
"path"
"path/filepath"
"strings"
"slices"
"github.com/fatih/color"
cc "github.com/ivanpirog/coloredcobra"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/cobra/doc"
"golang.org/x/exp/slices"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
@ -30,15 +29,11 @@ var dbClient *database.Client
var OutputFormat string
var OutputColor string
var downloadOnly bool
var forceAction bool
var purge bool
var all bool
var prometheusURL string
var mergedConfig string
// flagBranch overrides the value in csConfig.Cscli.HubBranch
var flagBranch = ""
func initConfig() {
var err error
if trace_lvl {
@ -54,14 +49,11 @@ func initConfig() {
}
if !slices.Contains(NoNeedConfig, os.Args[1]) {
log.Debugf("Using %s as configuration file", ConfigFilePath)
csConfig, mergedConfig, err = csconfig.NewConfig(ConfigFilePath, false, false, true)
if err != nil {
log.Fatal(err)
}
log.Debugf("Using %s as configuration file", ConfigFilePath)
if err := csConfig.LoadCSCLI(); err != nil {
log.Fatal(err)
}
} else {
csConfig = csconfig.NewDefaultConfig()
}
@ -72,13 +64,10 @@ func initConfig() {
log.Debugf("Enabled feature flags: %s", fflist)
}
if csConfig.Cscli == nil {
log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath)
if flagBranch != "" {
csConfig.Cscli.HubBranch = flagBranch
}
if cwhub.HubBranch == "" && csConfig.Cscli.HubBranch != "" {
cwhub.HubBranch = csConfig.Cscli.HubBranch
}
if OutputFormat != "" {
csConfig.Cscli.Output = OutputFormat
if OutputFormat != "json" && OutputFormat != "raw" && OutputFormat != "human" {
@ -104,9 +93,10 @@ func initConfig() {
}
var validArgs = []string{
"scenarios", "parsers", "collections", "capi", "lapi", "postoverflows", "machines",
"metrics", "bouncers", "alerts", "decisions", "simulation", "hub", "dashboard",
"config", "completion", "version", "console", "notifications", "support",
"alerts", "appsec-configs", "appsec-rules", "bouncers", "capi", "collections",
"completion", "config", "console", "contexts", "dashboard", "decisions", "explain",
"hub", "hubtest", "lapi", "machines", "metrics", "notifications", "parsers",
"postoverflows", "scenarios", "simulation", "support", "version",
}
func prepender(filename string) string {
@ -116,7 +106,7 @@ title: %s
---
`
name := filepath.Base(filename)
base := strings.TrimSuffix(name, path.Ext(name))
base := strings.TrimSuffix(name, filepath.Ext(name))
return fmt.Sprintf(header, base, strings.ReplaceAll(base, "_", " "))
}
@ -135,7 +125,7 @@ var (
func main() {
// set the formatter asap and worry about level later
logFormatter := &log.TextFormatter{TimestampFormat: "02-01-2006 15:04:05", FullTimestamp: true}
logFormatter := &log.TextFormatter{TimestampFormat: "2006-01-02 15:04:05", FullTimestamp: true}
log.SetFormatter(logFormatter)
if err := fflag.RegisterAllFeatures(); err != nil {
@ -189,7 +179,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
/*usage*/
var cmdVersion = &cobra.Command{
Use: "version",
Short: "Display version and exit.",
Short: "Display version",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
@ -199,15 +189,15 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
rootCmd.AddCommand(cmdVersion)
rootCmd.PersistentFlags().StringVarP(&ConfigFilePath, "config", "c", csconfig.DefaultConfigPath("config.yaml"), "path to crowdsec config file")
rootCmd.PersistentFlags().StringVarP(&OutputFormat, "output", "o", "", "Output format: human, json, raw.")
rootCmd.PersistentFlags().StringVarP(&OutputColor, "color", "", "auto", "Output color: yes, no, auto.")
rootCmd.PersistentFlags().BoolVar(&dbg_lvl, "debug", false, "Set logging to debug.")
rootCmd.PersistentFlags().BoolVar(&nfo_lvl, "info", false, "Set logging to info.")
rootCmd.PersistentFlags().BoolVar(&wrn_lvl, "warning", false, "Set logging to warning.")
rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error.")
rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace.")
rootCmd.PersistentFlags().StringVarP(&OutputFormat, "output", "o", "", "Output format: human, json, raw")
rootCmd.PersistentFlags().StringVarP(&OutputColor, "color", "", "auto", "Output color: yes, no, auto")
rootCmd.PersistentFlags().BoolVar(&dbg_lvl, "debug", false, "Set logging to debug")
rootCmd.PersistentFlags().BoolVar(&nfo_lvl, "info", false, "Set logging to info")
rootCmd.PersistentFlags().BoolVar(&wrn_lvl, "warning", false, "Set logging to warning")
rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error")
rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace")
rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "", "Override hub branch on github")
rootCmd.PersistentFlags().StringVar(&flagBranch, "branch", "", "Override hub branch on github")
if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil {
log.Fatalf("failed to hide flag: %s", err)
}
@ -236,33 +226,37 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
rootCmd.PersistentFlags().SortFlags = false
rootCmd.AddCommand(NewConfigCmd())
rootCmd.AddCommand(NewHubCmd())
rootCmd.AddCommand(NewCLIHub().NewCommand())
rootCmd.AddCommand(NewMetricsCmd())
rootCmd.AddCommand(NewDashboardCmd())
rootCmd.AddCommand(NewDecisionsCmd())
rootCmd.AddCommand(NewAlertsCmd())
rootCmd.AddCommand(NewSimulationCmds())
rootCmd.AddCommand(NewBouncersCmd())
rootCmd.AddCommand(NewMachinesCmd())
rootCmd.AddCommand(NewParsersCmd())
rootCmd.AddCommand(NewScenariosCmd())
rootCmd.AddCommand(NewCollectionsCmd())
rootCmd.AddCommand(NewPostOverflowsCmd())
rootCmd.AddCommand(NewCapiCmd())
rootCmd.AddCommand(NewCLIDashboard().NewCommand())
rootCmd.AddCommand(NewCLIDecisions().NewCommand())
rootCmd.AddCommand(NewCLIAlerts().NewCommand())
rootCmd.AddCommand(NewCLISimulation().NewCommand())
rootCmd.AddCommand(NewCLIBouncers().NewCommand())
rootCmd.AddCommand(NewCLIMachines().NewCommand())
rootCmd.AddCommand(NewCLICapi().NewCommand())
rootCmd.AddCommand(NewLapiCmd())
rootCmd.AddCommand(NewCompletionCmd())
rootCmd.AddCommand(NewConsoleCmd())
rootCmd.AddCommand(NewExplainCmd())
rootCmd.AddCommand(NewHubTestCmd())
rootCmd.AddCommand(NewNotificationsCmd())
rootCmd.AddCommand(NewSupportCmd())
rootCmd.AddCommand(NewCLIExplain().NewCommand())
rootCmd.AddCommand(NewCLIHubTest().NewCommand())
rootCmd.AddCommand(NewCLINotifications().NewCommand())
rootCmd.AddCommand(NewCLISupport().NewCommand())
rootCmd.AddCommand(NewCLIPapi().NewCommand())
rootCmd.AddCommand(NewCLICollection().NewCommand())
rootCmd.AddCommand(NewCLIParser().NewCommand())
rootCmd.AddCommand(NewCLIScenario().NewCommand())
rootCmd.AddCommand(NewCLIPostOverflow().NewCommand())
rootCmd.AddCommand(NewCLIContext().NewCommand())
rootCmd.AddCommand(NewCLIAppsecConfig().NewCommand())
rootCmd.AddCommand(NewCLIAppsecRule().NewCommand())
if fflag.CscliSetup.IsEnabled() {
rootCmd.AddCommand(NewSetupCmd())
}
if fflag.PapiClient.IsEnabled() {
rootCmd.AddCommand(NewPapiCmd())
rootCmd.AddCommand(NewCLIPapi().NewCommand())
}
if err := rootCmd.Execute(); err != nil {

118
cmd/crowdsec-cli/metrics.go
View file

@ -14,9 +14,9 @@ import (
"github.com/prometheus/prom2json"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v2"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/go-cs-lib/trace"
)
// FormatPrometheusMetrics is a complete rip from prom2json
@ -63,6 +63,8 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
lapi_machine_stats := map[string]map[string]map[string]int{}
lapi_bouncer_stats := map[string]map[string]map[string]int{}
decisions_stats := map[string]map[string]map[string]int{}
appsec_engine_stats := map[string]map[string]int{}
appsec_rule_stats := map[string]map[string]map[string]int{}
alerts_stats := map[string]int{}
stash_stats := map[string]struct {
Type string
@ -226,10 +228,30 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
Type string
Count int
}{Type: mtype, Count: ival}
case "cs_appsec_reqs_total":
if _, ok := appsec_engine_stats[metric.Labels["appsec_engine"]]; !ok {
appsec_engine_stats[metric.Labels["appsec_engine"]] = make(map[string]int, 0)
}
appsec_engine_stats[metric.Labels["appsec_engine"]]["processed"] = ival
case "cs_appsec_block_total":
if _, ok := appsec_engine_stats[metric.Labels["appsec_engine"]]; !ok {
appsec_engine_stats[metric.Labels["appsec_engine"]] = make(map[string]int, 0)
}
appsec_engine_stats[metric.Labels["appsec_engine"]]["blocked"] = ival
case "cs_appsec_rule_hits":
appsecEngine := metric.Labels["appsec_engine"]
ruleID := metric.Labels["rule_name"]
if _, ok := appsec_rule_stats[appsecEngine]; !ok {
appsec_rule_stats[appsecEngine] = make(map[string]map[string]int, 0)
}
if _, ok := appsec_rule_stats[appsecEngine][ruleID]; !ok {
appsec_rule_stats[appsecEngine][ruleID] = make(map[string]int, 0)
}
appsec_rule_stats[appsecEngine][ruleID]["triggered"] = ival
default:
log.Debugf("unknown: %+v", fam.Name)
continue
}
}
}
@ -244,35 +266,61 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error
decisionStatsTable(out, decisions_stats)
alertStatsTable(out, alerts_stats)
stashStatsTable(out, stash_stats)
} else if formatType == "json" {
for _, val := range []interface{}{acquis_stats, parsers_stats, buckets_stats, lapi_stats, lapi_bouncer_stats, lapi_machine_stats, lapi_decisions_stats, decisions_stats, alerts_stats, stash_stats} {
x, err := json.MarshalIndent(val, "", " ")
if err != nil {
return fmt.Errorf("failed to unmarshal metrics : %v", err)
}
out.Write(x)
}
return nil
} else if formatType == "raw" {
for _, val := range []interface{}{acquis_stats, parsers_stats, buckets_stats, lapi_stats, lapi_bouncer_stats, lapi_machine_stats, lapi_decisions_stats, decisions_stats, alerts_stats, stash_stats} {
x, err := yaml.Marshal(val)
if err != nil {
return fmt.Errorf("failed to unmarshal metrics : %v", err)
}
out.Write(x)
}
appsecMetricsToTable(out, appsec_engine_stats)
appsecRulesToTable(out, appsec_rule_stats)
return nil
}
stats := make(map[string]any)
stats["acquisition"] = acquis_stats
stats["buckets"] = buckets_stats
stats["parsers"] = parsers_stats
stats["lapi"] = lapi_stats
stats["lapi_machine"] = lapi_machine_stats
stats["lapi_bouncer"] = lapi_bouncer_stats
stats["lapi_decisions"] = lapi_decisions_stats
stats["decisions"] = decisions_stats
stats["alerts"] = alerts_stats
stats["stash"] = stash_stats
switch formatType {
case "json":
x, err := json.MarshalIndent(stats, "", " ")
if err != nil {
return fmt.Errorf("failed to unmarshal metrics : %v", err)
}
out.Write(x)
case "raw":
x, err := yaml.Marshal(stats)
if err != nil {
return fmt.Errorf("failed to unmarshal metrics : %v", err)
}
out.Write(x)
default:
return fmt.Errorf("unknown format type %s", formatType)
}
return nil
}
var noUnit bool
func runMetrics(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadPrometheus(); err != nil {
return fmt.Errorf("failed to load prometheus config: %w", err)
flags := cmd.Flags()
url, err := flags.GetString("url")
if err != nil {
return err
}
if url != "" {
csConfig.Cscli.PrometheusUrl = url
}
noUnit, err = flags.GetBool("no-unit")
if err != nil {
return err
}
if csConfig.Prometheus == nil {
@ -283,22 +331,12 @@ func runMetrics(cmd *cobra.Command, args []string) error {
return fmt.Errorf("prometheus is not enabled, can't show metrics")
}
if prometheusURL == "" {
prometheusURL = csConfig.Cscli.PrometheusUrl
}
if prometheusURL == "" {
return fmt.Errorf("no prometheus url, please specify in %s or via -u", *csConfig.FilePath)
}
err := FormatPrometheusMetrics(color.Output, prometheusURL+"/metrics", csConfig.Cscli.Output)
if err != nil {
return fmt.Errorf("could not fetch prometheus metrics: %w", err)
if err = FormatPrometheusMetrics(color.Output, csConfig.Cscli.PrometheusUrl, csConfig.Cscli.Output); err != nil {
return err
}
return nil
}
func NewMetricsCmd() *cobra.Command {
cmdMetrics := &cobra.Command{
Use: "metrics",
@ -306,10 +344,12 @@ func NewMetricsCmd() *cobra.Command {
Long: `Fetch metrics from the prometheus server and display them in a human-friendly way`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: runMetrics,
RunE: runMetrics,
}
cmdMetrics.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
cmdMetrics.PersistentFlags().BoolVar(&noUnit, "no-unit", false, "Show the real number instead of formatted with units")
flags := cmdMetrics.PersistentFlags()
flags.StringP("url", "u", "", "Prometheus url (http://<ip>:<port>/metrics)")
flags.Bool("no-unit", false, "Show the real number instead of formatted with units")
return cmdMetrics
}

47
cmd/crowdsec-cli/metrics_table.go
View file

@ -90,7 +90,7 @@ func bucketStatsTable(out io.Writer, stats map[string]map[string]int) {
keys := []string{"curr_count", "overflow", "instantiation", "pour", "underflow"}
if numRows, err := metricsToTable(t, stats, keys); err != nil {
log.Warningf("while collecting acquis stats: %s", err)
log.Warningf("while collecting bucket stats: %s", err)
} else if numRows > 0 {
renderTableTitle(out, "\nBucket Metrics:")
t.Render()
@ -113,6 +113,37 @@ func acquisStatsTable(out io.Writer, stats map[string]map[string]int) {
}
}
func appsecMetricsToTable(out io.Writer, metrics map[string]map[string]int) {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Appsec Engine", "Processed", "Blocked")
t.SetAlignment(table.AlignLeft, table.AlignLeft)
keys := []string{"processed", "blocked"}
if numRows, err := metricsToTable(t, metrics, keys); err != nil {
log.Warningf("while collecting appsec stats: %s", err)
} else if numRows > 0 {
renderTableTitle(out, "\nAppsec Metrics:")
t.Render()
}
}
func appsecRulesToTable(out io.Writer, metrics map[string]map[string]map[string]int) {
for appsecEngine, appsecEngineRulesStats := range metrics {
t := newTable(out)
t.SetRowLines(false)
t.SetHeaders("Rule ID", "Triggered")
t.SetAlignment(table.AlignLeft, table.AlignLeft)
keys := []string{"triggered"}
if numRows, err := metricsToTable(t, appsecEngineRulesStats, keys); err != nil {
log.Warningf("while collecting appsec rules stats: %s", err)
} else if numRows > 0 {
renderTableTitle(out, fmt.Sprintf("\nAppsec '%s' Rules Metrics:", appsecEngine))
t.Render()
}
}
}
func parserStatsTable(out io.Writer, stats map[string]map[string]int) {
t := newTable(out)
t.SetRowLines(false)
@ -122,7 +153,7 @@ func parserStatsTable(out io.Writer, stats map[string]map[string]int) {
keys := []string{"hits", "parsed", "unparsed"}
if numRows, err := metricsToTable(t, stats, keys); err != nil {
log.Warningf("while collecting acquis stats: %s", err)
log.Warningf("while collecting parsers stats: %s", err)
} else if numRows > 0 {
renderTableTitle(out, "\nParser Metrics:")
t.Render()
@ -199,7 +230,7 @@ func lapiStatsTable(out io.Writer, stats map[string]map[string]int) {
}
if numRows > 0 {
renderTableTitle(out, "\nLocal Api Metrics:")
renderTableTitle(out, "\nLocal API Metrics:")
t.Render()
}
}
@ -213,7 +244,7 @@ func lapiMachineStatsTable(out io.Writer, stats map[string]map[string]map[string
numRows := lapiMetricsToTable(t, stats)
if numRows > 0 {
renderTableTitle(out, "\nLocal Api Machines Metrics:")
renderTableTitle(out, "\nLocal API Machines Metrics:")
t.Render()
}
}
@ -227,7 +258,7 @@ func lapiBouncerStatsTable(out io.Writer, stats map[string]map[string]map[string
numRows := lapiMetricsToTable(t, stats)
if numRows > 0 {
renderTableTitle(out, "\nLocal Api Bouncers Metrics:")
renderTableTitle(out, "\nLocal API Bouncers Metrics:")
t.Render()
}
}
@ -253,7 +284,7 @@ func lapiDecisionStatsTable(out io.Writer, stats map[string]struct {
}
if numRows > 0 {
renderTableTitle(out, "\nLocal Api Bouncers Decisions:")
renderTableTitle(out, "\nLocal API Bouncers Decisions:")
t.Render()
}
}
@ -280,7 +311,7 @@ func decisionStatsTable(out io.Writer, stats map[string]map[string]map[string]in
}
if numRows > 0 {
renderTableTitle(out, "\nLocal Api Decisions:")
renderTableTitle(out, "\nLocal API Decisions:")
t.Render()
}
}
@ -301,7 +332,7 @@ func alertStatsTable(out io.Writer, stats map[string]int) {
}
if numRows > 0 {
renderTableTitle(out, "\nLocal Api Alerts:")
renderTableTitle(out, "\nLocal API Alerts:")
t.Render()
}
}

286
cmd/crowdsec-cli/notifications.go
View file

@ -18,13 +18,19 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/csplugin"
"github.com/crowdsecurity/crowdsec/pkg/csprofiles"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/models"
)
type NotificationsCfg struct {
@ -33,35 +39,45 @@ type NotificationsCfg struct {
ids []uint
}
func NewNotificationsCmd() *cobra.Command {
var cmdNotifications = &cobra.Command{
type cliNotifications struct {}
func NewCLINotifications() *cliNotifications {
return &cliNotifications{}
}
func (cli cliNotifications) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "notifications [action]",
Short: "Helper for notification plugin configuration",
Long: "To list/inspect/test notification template",
Args: cobra.MinimumNArgs(1),
Aliases: []string{"notifications", "notification"},
DisableAutoGenTag: true,
PersistentPreRun: func(cmd *cobra.Command, args []string) {
var (
err error
)
if err = csConfig.API.Server.LoadProfiles(); err != nil {
log.Fatal(err)
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.LAPI(csConfig); err != nil {
return err
}
if csConfig.ConfigPaths.NotificationDir == "" {
log.Fatalf("config_paths.notification_dir is not set in crowdsec config")
if err := require.Profiles(csConfig); err != nil {
return err
}
if err := require.Notifications(csConfig); err != nil {
return err
}
return nil
},
}
cmdNotifications.AddCommand(NewNotificationsListCmd())
cmdNotifications.AddCommand(NewNotificationsInspectCmd())
cmdNotifications.AddCommand(NewNotificationsReinjectCmd())
cmd.AddCommand(cli.NewListCmd())
cmd.AddCommand(cli.NewInspectCmd())
cmd.AddCommand(cli.NewReinjectCmd())
cmd.AddCommand(cli.NewTestCmd())
return cmdNotifications
return cmd
}
func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
func getPluginConfigs() (map[string]csplugin.PluginConfig, error) {
pcfgs := map[string]csplugin.PluginConfig{}
wf := func(path string, info fs.FileInfo, err error) error {
if info == nil {
@ -74,6 +90,7 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
return fmt.Errorf("loading notifification plugin configuration with %s: %w", name, err)
}
for _, t := range ts {
csplugin.SetRequiredFields(&t)
pcfgs[t.Name] = t
}
}
@ -83,8 +100,15 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
if err := filepath.Walk(csConfig.ConfigPaths.NotificationDir, wf); err != nil {
return nil, fmt.Errorf("while loading notifification plugin configuration: %w", err)
}
return pcfgs, nil
}
func getProfilesConfigs() (map[string]NotificationsCfg, error) {
// A bit of a tricky stuf now: reconcile profiles and notification plugins
pcfgs, err := getPluginConfigs()
if err != nil {
return nil, err
}
ncfgs := map[string]NotificationsCfg{}
profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
if err != nil {
@ -124,16 +148,16 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
return ncfgs, nil
}
func NewNotificationsListCmd() *cobra.Command {
var cmdNotificationsList = &cobra.Command{
func (cli cliNotifications) NewListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "list",
Short: "List active notifications plugins",
Long: `List active notifications plugins`,
Short: "list active notifications plugins",
Long: `list active notifications plugins`,
Example: `cscli notifications list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, arg []string) error {
ncfgs, err := getNotificationsConfiguration()
ncfgs, err := getProfilesConfigs()
if err != nil {
return fmt.Errorf("can't build profiles configuration: %w", err)
}
@ -168,36 +192,32 @@ func NewNotificationsListCmd() *cobra.Command {
},
}
return cmdNotificationsList
return cmd
}
func NewNotificationsInspectCmd() *cobra.Command {
var cmdNotificationsInspect = &cobra.Command{
func (cli cliNotifications) NewInspectCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "inspect",
Short: "Inspect active notifications plugin configuration",
Long: `Inspect active notifications plugin and show configuration`,
Example: `cscli notifications inspect <plugin_name>`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, arg []string) error {
var (
cfg NotificationsCfg
ok bool
)
pluginName := arg[0]
if pluginName == "" {
PreRunE: func(cmd *cobra.Command, args []string) error {
if args[0] == "" {
return fmt.Errorf("please provide a plugin name to inspect")
}
ncfgs, err := getNotificationsConfiguration()
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
ncfgs, err := getProfilesConfigs()
if err != nil {
return fmt.Errorf("can't build profiles configuration: %w", err)
}
if cfg, ok = ncfgs[pluginName]; !ok {
return fmt.Errorf("plugin '%s' does not exist or is not active", pluginName)
cfg, ok := ncfgs[args[0]]
if !ok {
return fmt.Errorf("plugin '%s' does not exist or is not active", args[0])
}
if csConfig.Cscli.Output == "human" || csConfig.Cscli.Output == "raw" {
fmt.Printf(" - %15s: %15s\n", "Type", cfg.Config.Type)
fmt.Printf(" - %15s: %15s\n", "Name", cfg.Config.Name)
@ -217,78 +237,128 @@ func NewNotificationsInspectCmd() *cobra.Command {
},
}
return cmdNotificationsInspect
return cmd
}
func NewNotificationsReinjectCmd() *cobra.Command {
var remediation bool
var alertOverride string
func (cli cliNotifications) NewTestCmd() *cobra.Command {
var (
pluginBroker csplugin.PluginBroker
pluginTomb tomb.Tomb
alertOverride string
)
cmd := &cobra.Command{
Use: "test [plugin name]",
Short: "send a generic test alert to notification plugin",
Long: `send a generic test alert to a notification plugin to test configuration even if is not active`,
Example: `cscli notifications test [plugin_name]`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
PreRunE: func(cmd *cobra.Command, args []string) error {
pconfigs, err := getPluginConfigs()
if err != nil {
return fmt.Errorf("can't build profiles configuration: %w", err)
}
cfg, ok := pconfigs[args[0]]
if !ok {
return fmt.Errorf("plugin name: '%s' does not exist", args[0])
}
//Create a single profile with plugin name as notification name
return pluginBroker.Init(csConfig.PluginConfig, []*csconfig.ProfileCfg{
{
Notifications: []string{
cfg.Name,
},
},
}, csConfig.ConfigPaths)
},
RunE: func(cmd *cobra.Command, args []string) error {
pluginTomb.Go(func() error {
pluginBroker.Run(&pluginTomb)
return nil
})
alert := &models.Alert{
Capacity: ptr.Of(int32(0)),
Decisions: []*models.Decision{{
Duration: ptr.Of("4h"),
Scope: ptr.Of("Ip"),
Value: ptr.Of("10.10.10.10"),
Type: ptr.Of("ban"),
Scenario: ptr.Of("test alert"),
Origin: ptr.Of(types.CscliOrigin),
}},
Events: []*models.Event{},
EventsCount: ptr.Of(int32(1)),
Leakspeed: ptr.Of("0"),
Message: ptr.Of("test alert"),
ScenarioHash: ptr.Of(""),
Scenario: ptr.Of("test alert"),
ScenarioVersion: ptr.Of(""),
Simulated: ptr.Of(false),
Source: &models.Source{
AsName: "",
AsNumber: "",
Cn: "",
IP: "10.10.10.10",
Range: "",
Scope: ptr.Of("Ip"),
Value: ptr.Of("10.10.10.10"),
},
StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
CreatedAt: time.Now().UTC().Format(time.RFC3339),
}
if err := yaml.Unmarshal([]byte(alertOverride), alert); err != nil {
return fmt.Errorf("failed to unmarshal alert override: %w", err)
}
pluginBroker.PluginChannel <- csplugin.ProfileAlert{
ProfileID: uint(0),
Alert: alert,
}
//time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
pluginTomb.Kill(fmt.Errorf("terminating"))
pluginTomb.Wait()
return nil
},
}
cmd.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the generic alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
var cmdNotificationsReinject = &cobra.Command{
return cmd
}
func (cli cliNotifications) NewReinjectCmd() *cobra.Command {
var alertOverride string
var alert *models.Alert
cmd := &cobra.Command{
Use: "reinject",
Short: "reinject alert into notifications system",
Long: `Reinject alert into notifications system`,
Short: "reinject an alert into profiles to trigger notifications",
Long: `reinject an alert into profiles to be evaluated by the filter and sent to matched notifications plugins`,
Example: `
cscli notifications reinject <alert_id>
cscli notifications reinject <alert_id> --remediation
cscli notifications reinject <alert_id> -a '{"remediation": false,"scenario":"notification/test"}'
cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"notification/test"}'
`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
PreRunE: func(cmd *cobra.Command, args []string) error {
var err error
alert, err = FetchAlertFromArgString(args[0])
if err != nil {
return err
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
var (
pluginBroker csplugin.PluginBroker
pluginTomb tomb.Tomb
)
if len(args) != 1 {
printHelp(cmd)
return fmt.Errorf("wrong number of argument: there should be one argument")
}
//first: get the alert
id, err := strconv.Atoi(args[0])
if err != nil {
return fmt.Errorf("bad alert id %s", args[0])
}
if err := csConfig.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
}
if csConfig.API.Client == nil {
return fmt.Errorf("missing configuration on 'api_client:'")
}
if csConfig.API.Client.Credentials == nil {
return fmt.Errorf("missing API credentials in '%s'", csConfig.API.Client.CredentialsFilePath)
}
apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
if err != nil {
return fmt.Errorf("error parsing the URL of the API: %w", err)
}
client, err := apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
Password: strfmt.Password(csConfig.API.Client.Credentials.Password),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
})
if err != nil {
return fmt.Errorf("error creating the client for the API: %w", err)
}
alert, _, err := client.Alerts.GetByID(context.Background(), id)
if err != nil {
return fmt.Errorf("can't find alert with id %s: %w", args[0], err)
}
if alertOverride != "" {
if err = json.Unmarshal([]byte(alertOverride), alert); err != nil {
if err := json.Unmarshal([]byte(alertOverride), alert); err != nil {
return fmt.Errorf("can't unmarshal data in the alert flag: %w", err)
}
}
if !remediation {
alert.Remediation = true
}
// second we start plugins
err = pluginBroker.Init(csConfig.PluginConfig, csConfig.API.Server.Profiles, csConfig.ConfigPaths)
err := pluginBroker.Init(csConfig.PluginConfig, csConfig.API.Server.Profiles, csConfig.ConfigPaths)
if err != nil {
return fmt.Errorf("can't initialize plugins: %w", err)
}
@ -298,8 +368,6 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
return nil
})
//third: get the profile(s), and process the whole stuff
profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
if err != nil {
return fmt.Errorf("cannot extract profiles from configuration: %w", err)
@ -334,15 +402,39 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
break
}
}
// time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
//time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
pluginTomb.Kill(fmt.Errorf("terminating"))
pluginTomb.Wait()
return nil
},
}
cmdNotificationsReinject.Flags().BoolVarP(&remediation, "remediation", "r", false, "Set Alert.Remediation to false in the reinjected alert (see your profile filter configuration)")
cmdNotificationsReinject.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the reinjected alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
cmd.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the reinjected alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
return cmdNotificationsReinject
return cmd
}
func FetchAlertFromArgString(toParse string) (*models.Alert, error) {
id, err := strconv.Atoi(toParse)
if err != nil {
return nil, fmt.Errorf("bad alert id %s", toParse)
}
apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
if err != nil {
return nil, fmt.Errorf("error parsing the URL of the API: %w", err)
}
client, err := apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
Password: strfmt.Password(csConfig.API.Client.Credentials.Password),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
})
if err != nil {
return nil, fmt.Errorf("error creating the client for the API: %w", err)
}
alert, _, err := client.Alerts.GetByID(context.Background(), id)
if err != nil {
return nil, fmt.Errorf("can't find alert with id %d: %w", id, err)
}
return alert, nil
}

45
cmd/crowdsec-cli/papi.go
View file

@ -1,47 +1,54 @@
package main
import (
"fmt"
"time"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/tomb.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/ptr"
"github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/crowdsecurity/crowdsec/pkg/apiserver"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
func NewPapiCmd() *cobra.Command {
var cmdLapi = &cobra.Command{
type cliPapi struct {}
func NewCLIPapi() *cliPapi {
return &cliPapi{}
}
func (cli cliPapi) NewCommand() *cobra.Command {
var cmd = &cobra.Command{
Use: "papi [action]",
Short: "Manage interaction with Polling API (PAPI)",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
return fmt.Errorf("Local API is disabled, please run this command on the local API machine: %w", err)
if err := require.LAPI(csConfig); err != nil {
return err
}
if csConfig.API.Server.OnlineClient == nil {
log.Fatalf("no configuration for Central API in '%s'", *csConfig.FilePath)
if err := require.CAPI(csConfig); err != nil {
return err
}
if csConfig.API.Server.OnlineClient.Credentials.PapiURL == "" {
log.Fatalf("no PAPI URL in configuration")
if err := require.PAPI(csConfig); err != nil {
return err
}
return nil
},
}
cmdLapi.AddCommand(NewPapiStatusCmd())
cmdLapi.AddCommand(NewPapiSyncCmd())
cmd.AddCommand(cli.NewStatusCmd())
cmd.AddCommand(cli.NewSyncCmd())
return cmdLapi
return cmd
}
func NewPapiStatusCmd() *cobra.Command {
cmdCapiStatus := &cobra.Command{
func (cli cliPapi) NewStatusCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "status",
Short: "Get status of the Polling API",
Args: cobra.MinimumNArgs(0),
@ -86,11 +93,11 @@ func NewPapiStatusCmd() *cobra.Command {
},
}
return cmdCapiStatus
return cmd
}
func NewPapiSyncCmd() *cobra.Command {
cmdCapiSync := &cobra.Command{
func (cli cliPapi) NewSyncCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "sync",
Short: "Sync with the Polling API, pulling all non-expired orders for the instance",
Args: cobra.MinimumNArgs(0),
@ -134,5 +141,5 @@ func NewPapiSyncCmd() *cobra.Command {
},
}
return cmdCapiSync
return cmd
}

202
cmd/crowdsec-cli/parsers.go
View file

@ -1,202 +0,0 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewParsersCmd() *cobra.Command {
var cmdParsers = &cobra.Command{
Use: "parsers [action] [config]",
Short: "Install/Remove/Upgrade/Inspect parser(s) from hub",
Example: `cscli parsers install crowdsecurity/sshd-logs
cscli parsers inspect crowdsecurity/sshd-logs
cscli parsers upgrade crowdsecurity/sshd-logs
cscli parsers list
cscli parsers remove crowdsecurity/sshd-logs
`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"parser"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if csConfig.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
cmdParsers.AddCommand(NewParsersInstallCmd())
cmdParsers.AddCommand(NewParsersRemoveCmd())
cmdParsers.AddCommand(NewParsersUpgradeCmd())
cmdParsers.AddCommand(NewParsersInspectCmd())
cmdParsers.AddCommand(NewParsersListCmd())
return cmdParsers
}
func NewParsersInstallCmd() *cobra.Command {
var ignoreError bool
var cmdParsersInstall = &cobra.Command{
Use: "install [config]",
Short: "Install given parser(s)",
Long: `Fetch and install given parser(s) from hub`,
Example: `cscli parsers install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.PARSERS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
for _, name := range args {
t := cwhub.GetItem(cwhub.PARSERS, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.PARSERS, name)
Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
if ignoreError {
log.Errorf("Error while installing '%s': %s", name, err)
} else {
log.Fatalf("Error while installing '%s': %s", name, err)
}
}
}
},
}
cmdParsersInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdParsersInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdParsersInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple parsers")
return cmdParsersInstall
}
func NewParsersRemoveCmd() *cobra.Command {
var cmdParsersRemove = &cobra.Command{
Use: "remove [config]",
Short: "Remove given parser(s)",
Long: `Remove given parse(s) from hub`,
Aliases: []string{"delete"},
Example: `cscli parsers remove crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
if all {
cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, forceAction)
return
}
if len(args) == 0 {
log.Fatalf("Specify at least one parser to remove or '--all' flag.")
}
for _, name := range args {
cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, forceAction)
}
},
}
cmdParsersRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdParsersRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdParsersRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the parsers")
return cmdParsersRemove
}
func NewParsersUpgradeCmd() *cobra.Command {
var cmdParsersUpgrade = &cobra.Command{
Use: "upgrade [config]",
Short: "Upgrade given parser(s)",
Long: `Fetch and upgrade given parser(s) from hub`,
Example: `cscli parsers upgrade crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", forceAction)
} else {
if len(args) == 0 {
log.Fatalf("no target parser to upgrade")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, forceAction)
}
}
},
}
cmdParsersUpgrade.PersistentFlags().BoolVar(&all, "all", false, "Upgrade all the parsers")
cmdParsersUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
return cmdParsersUpgrade
}
func NewParsersInspectCmd() *cobra.Command {
var cmdParsersInspect = &cobra.Command{
Use: "inspect [name]",
Short: "Inspect given parser",
Long: `Inspect given parser`,
Example: `cscli parsers inspect crowdsec/xxx`,
DisableAutoGenTag: true,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
InspectItem(args[0], cwhub.PARSERS)
},
}
cmdParsersInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
return cmdParsersInspect
}
func NewParsersListCmd() *cobra.Command {
var cmdParsersList = &cobra.Command{
Use: "list [name]",
Short: "List all parsers or given one",
Long: `List all parsers or given one`,
Example: `cscli parsers list
cscli parser list crowdsecurity/xxx`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.PARSERS}, args, false, true, all)
},
}
cmdParsersList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmdParsersList
}

200
cmd/crowdsec-cli/postoverflows.go
View file

@ -1,200 +0,0 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewPostOverflowsInstallCmd() *cobra.Command {
var ignoreError bool
cmdPostOverflowsInstall := &cobra.Command{
Use: "install [config]",
Short: "Install given postoverflow(s)",
Long: `Fetch and install given postoverflow(s) from hub`,
Example: `cscli postoverflows install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
for _, name := range args {
t := cwhub.GetItem(cwhub.PARSERS_OVFLW, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.PARSERS_OVFLW, name)
Suggest(cwhub.PARSERS_OVFLW, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS_OVFLW, forceAction, downloadOnly); err != nil {
if ignoreError {
log.Errorf("Error while installing '%s': %s", name, err)
} else {
log.Fatalf("Error while installing '%s': %s", name, err)
}
}
}
},
}
cmdPostOverflowsInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdPostOverflowsInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdPostOverflowsInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple postoverflows")
return cmdPostOverflowsInstall
}
func NewPostOverflowsRemoveCmd() *cobra.Command {
cmdPostOverflowsRemove := &cobra.Command{
Use: "remove [config]",
Short: "Remove given postoverflow(s)",
Long: `remove given postoverflow(s)`,
Example: `cscli postoverflows remove crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
Aliases: []string{"delete"},
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
if all {
cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, "", all, purge, forceAction)
return
}
if len(args) == 0 {
log.Fatalf("Specify at least one postoverflow to remove or '--all' flag.")
}
for _, name := range args {
cwhub.RemoveMany(csConfig, cwhub.PARSERS_OVFLW, name, all, purge, forceAction)
}
},
}
cmdPostOverflowsRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdPostOverflowsRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdPostOverflowsRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the postoverflows")
return cmdPostOverflowsRemove
}
func NewPostOverflowsUpgradeCmd() *cobra.Command {
cmdPostOverflowsUpgrade := &cobra.Command{
Use: "upgrade [config]",
Short: "Upgrade given postoverflow(s)",
Long: `Fetch and Upgrade given postoverflow(s) from hub`,
Example: `cscli postoverflows upgrade crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
Run: func(cmd *cobra.Command, args []string) {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, "", forceAction)
} else {
if len(args) == 0 {
log.Fatalf("no target postoverflow to upgrade")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.PARSERS_OVFLW, name, forceAction)
}
}
},
}
cmdPostOverflowsUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the postoverflows")
cmdPostOverflowsUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
return cmdPostOverflowsUpgrade
}
func NewPostOverflowsInspectCmd() *cobra.Command {
cmdPostOverflowsInspect := &cobra.Command{
Use: "inspect [config]",
Short: "Inspect given postoverflow",
Long: `Inspect given postoverflow`,
Example: `cscli postoverflows inspect crowdsec/xxx crowdsec/xyz`,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.PARSERS_OVFLW, args, toComplete)
},
Args: cobra.MinimumNArgs(1),
Run: func(cmd *cobra.Command, args []string) {
InspectItem(args[0], cwhub.PARSERS_OVFLW)
},
}
return cmdPostOverflowsInspect
}
func NewPostOverflowsListCmd() *cobra.Command {
cmdPostOverflowsList := &cobra.Command{
Use: "list [config]",
Short: "List all postoverflows or given one",
Long: `List all postoverflows or given one`,
Example: `cscli postoverflows list
cscli postoverflows list crowdsecurity/xxx`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.PARSERS_OVFLW}, args, false, true, all)
},
}
cmdPostOverflowsList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmdPostOverflowsList
}
func NewPostOverflowsCmd() *cobra.Command {
cmdPostOverflows := &cobra.Command{
Use: "postoverflows [action] [config]",
Short: "Install/Remove/Upgrade/Inspect postoverflow(s) from hub",
Example: `cscli postoverflows install crowdsecurity/cdn-whitelist
cscli postoverflows inspect crowdsecurity/cdn-whitelist
cscli postoverflows upgrade crowdsecurity/cdn-whitelist
cscli postoverflows list
cscli postoverflows remove crowdsecurity/cdn-whitelist`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"postoverflow"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if csConfig.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
cmdPostOverflows.AddCommand(NewPostOverflowsInstallCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsRemoveCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsUpgradeCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsInspectCmd())
cmdPostOverflows.AddCommand(NewPostOverflowsListCmd())
return cmdPostOverflows
}

58
cmd/crowdsec-cli/require/branch.go Normal file
View file

@ -0,0 +1,58 @@
package require
// Set the appropriate hub branch according to config settings and crowdsec version
import (
log "github.com/sirupsen/logrus"
"golang.org/x/mod/semver"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
func chooseBranch(cfg *csconfig.Config) string {
// this was set from config.yaml or flag
if cfg.Cscli.HubBranch != "" {
log.Debugf("Hub override from config: branch '%s'", cfg.Cscli.HubBranch)
return cfg.Cscli.HubBranch
}
latest, err := cwversion.Latest()
if err != nil {
log.Warningf("Unable to retrieve latest crowdsec version: %s, using hub branch 'master'", err)
return "master"
}
csVersion := cwversion.VersionStrip()
if csVersion == latest {
log.Debugf("Latest crowdsec version (%s), using hub branch 'master'", csVersion)
return "master"
}
// if current version is greater than the latest we are in pre-release
if semver.Compare(csVersion, latest) == 1 {
log.Debugf("Your current crowdsec version seems to be a pre-release (%s), using hub branch 'master'", csVersion)
return "master"
}
if csVersion == "" {
log.Warning("Crowdsec version is not set, using hub branch 'master'")
return "master"
}
log.Warnf("A new CrowdSec release is available (%s). "+
"Your version is '%s'. Please update it to use new parsers/scenarios/collections.",
latest, csVersion)
return csVersion
}
// HubBranch sets the branch (in cscli config) and returns its value
// It can be "master", or the branch corresponding to the current crowdsec version, or the value overridden in config/flag
func HubBranch(cfg *csconfig.Config) string {
branch := chooseBranch(cfg)
cfg.Cscli.HubBranch = branch
return branch
}

99
cmd/crowdsec-cli/require/require.go Normal file
View file

@ -0,0 +1,99 @@
package require
import (
"fmt"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func LAPI(c *csconfig.Config) error {
if err := c.LoadAPIServer(); err != nil {
return fmt.Errorf("failed to load Local API: %w", err)
}
if c.DisableAPI {
return fmt.Errorf("local API is disabled -- this command must be run on the local API machine")
}
return nil
}
func CAPI(c *csconfig.Config) error {
if c.API.Server.OnlineClient == nil {
return fmt.Errorf("no configuration for Central API (CAPI) in '%s'", *c.FilePath)
}
return nil
}
func PAPI(c *csconfig.Config) error {
if c.API.Server.OnlineClient.Credentials.PapiURL == "" {
return fmt.Errorf("no PAPI URL in configuration")
}
return nil
}
func CAPIRegistered(c *csconfig.Config) error {
if c.API.Server.OnlineClient.Credentials == nil {
return fmt.Errorf("the Central API (CAPI) must be configured with 'cscli capi register'")
}
return nil
}
func DB(c *csconfig.Config) error {
if err := c.LoadDBConfig(); err != nil {
return fmt.Errorf("this command requires direct database access (must be run on the local API machine): %w", err)
}
return nil
}
func Profiles(c *csconfig.Config) error {
if err := c.API.Server.LoadProfiles(); err != nil {
return fmt.Errorf("while loading profiles: %w", err)
}
return nil
}
func Notifications(c *csconfig.Config) error {
if c.ConfigPaths.NotificationDir == "" {
return fmt.Errorf("config_paths.notification_dir is not set in crowdsec config")
}
return nil
}
// RemoteHub returns the configuration required to download hub index and items: url, branch, etc.
func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg {
// set branch in config, and log if necessary
branch := HubBranch(c)
remote := &cwhub.RemoteHubCfg {
Branch: branch,
URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s",
// URLTemplate: "http://localhost:8000/crowdsecurity/%s/hub/%s",
IndexPath: ".index.json",
}
return remote
}
// Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items.
// If no remote parameter is provided, the hub can only be used for local operations.
func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg) (*cwhub.Hub, error) {
local := c.Hub
if local == nil {
return nil, fmt.Errorf("you must configure cli before interacting with hub")
}
hub, err := cwhub.NewHub(local, remote, false)
if err != nil {
return nil, fmt.Errorf("failed to read Hub index: %w. Run 'sudo cscli hub update' to download the index again", err)
}
return hub, nil
}

197
cmd/crowdsec-cli/scenarios.go
View file

@ -1,197 +0,0 @@
package main
import (
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func NewScenariosCmd() *cobra.Command {
var cmdScenarios = &cobra.Command{
Use: "scenarios [action] [config]",
Short: "Install/Remove/Upgrade/Inspect scenario(s) from hub",
Example: `cscli scenarios list [-a]
cscli scenarios install crowdsecurity/ssh-bf
cscli scenarios inspect crowdsecurity/ssh-bf
cscli scenarios upgrade crowdsecurity/ssh-bf
cscli scenarios remove crowdsecurity/ssh-bf
`,
Args: cobra.MinimumNArgs(1),
Aliases: []string{"scenario"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if csConfig.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("while setting hub branch: %w", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
cmdScenarios.AddCommand(NewCmdScenariosInstall())
cmdScenarios.AddCommand(NewCmdScenariosRemove())
cmdScenarios.AddCommand(NewCmdScenariosUpgrade())
cmdScenarios.AddCommand(NewCmdScenariosInspect())
cmdScenarios.AddCommand(NewCmdScenariosList())
return cmdScenarios
}
func NewCmdScenariosInstall() *cobra.Command {
var ignoreError bool
var cmdScenariosInstall = &cobra.Command{
Use: "install [config]",
Short: "Install given scenario(s)",
Long: `Fetch and install given scenario(s) from hub`,
Example: `cscli scenarios install crowdsec/xxx crowdsec/xyz`,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
for _, name := range args {
t := cwhub.GetItem(cwhub.SCENARIOS, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.SCENARIOS, name)
Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError)
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
if ignoreError {
log.Errorf("Error while installing '%s': %s", name, err)
} else {
log.Fatalf("Error while installing '%s': %s", name, err)
}
}
}
},
}
cmdScenariosInstall.PersistentFlags().BoolVarP(&downloadOnly, "download-only", "d", false, "Only download packages, don't enable")
cmdScenariosInstall.PersistentFlags().BoolVar(&forceAction, "force", false, "Force install : Overwrite tainted and outdated files")
cmdScenariosInstall.PersistentFlags().BoolVar(&ignoreError, "ignore", false, "Ignore errors when installing multiple scenarios")
return cmdScenariosInstall
}
func NewCmdScenariosRemove() *cobra.Command {
var cmdScenariosRemove = &cobra.Command{
Use: "remove [config]",
Short: "Remove given scenario(s)",
Long: `remove given scenario(s)`,
Example: `cscli scenarios remove crowdsec/xxx crowdsec/xyz`,
Aliases: []string{"delete"},
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if all {
cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, forceAction)
return
}
if len(args) == 0 {
log.Fatalf("Specify at least one scenario to remove or '--all' flag.")
}
for _, name := range args {
cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, forceAction)
}
},
}
cmdScenariosRemove.PersistentFlags().BoolVar(&purge, "purge", false, "Delete source file too")
cmdScenariosRemove.PersistentFlags().BoolVar(&forceAction, "force", false, "Force remove : Remove tainted and outdated files")
cmdScenariosRemove.PersistentFlags().BoolVar(&all, "all", false, "Delete all the scenarios")
return cmdScenariosRemove
}
func NewCmdScenariosUpgrade() *cobra.Command {
var cmdScenariosUpgrade = &cobra.Command{
Use: "upgrade [config]",
Short: "Upgrade given scenario(s)",
Long: `Fetch and Upgrade given scenario(s) from hub`,
Example: `cscli scenarios upgrade crowdsec/xxx crowdsec/xyz`,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if all {
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", forceAction)
} else {
if len(args) == 0 {
log.Fatalf("no target scenario to upgrade")
}
for _, name := range args {
cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, forceAction)
}
}
},
}
cmdScenariosUpgrade.PersistentFlags().BoolVarP(&all, "all", "a", false, "Upgrade all the scenarios")
cmdScenariosUpgrade.PersistentFlags().BoolVar(&forceAction, "force", false, "Force upgrade : Overwrite tainted and outdated files")
return cmdScenariosUpgrade
}
func NewCmdScenariosInspect() *cobra.Command {
var cmdScenariosInspect = &cobra.Command{
Use: "inspect [config]",
Short: "Inspect given scenario",
Long: `Inspect given scenario`,
Example: `cscli scenarios inspect crowdsec/xxx`,
Args: cobra.MinimumNArgs(1),
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(cwhub.SCENARIOS, args, toComplete)
},
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
InspectItem(args[0], cwhub.SCENARIOS)
},
}
cmdScenariosInspect.PersistentFlags().StringVarP(&prometheusURL, "url", "u", "", "Prometheus url")
return cmdScenariosInspect
}
func NewCmdScenariosList() *cobra.Command {
var cmdScenariosList = &cobra.Command{
Use: "list [config]",
Short: "List all scenario(s) or given one",
Long: `List all scenario(s) or given one`,
Example: `cscli scenarios list
cscli scenarios list crowdsecurity/xxx`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
ListItems(color.Output, []string{cwhub.SCENARIOS}, args, false, true, all)
},
}
cmdScenariosList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")
return cmdScenariosList
}

29
cmd/crowdsec-cli/setup.go
View file

@ -6,13 +6,15 @@ import (
"os"
"os/exec"
goccyyaml "github.com/goccy/go-yaml"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
goccyyaml "github.com/goccy/go-yaml"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/setup"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
// NewSetupCmd defines the "cscli setup" command.
@ -112,6 +114,20 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
return err
}
var detectReader *os.File
switch detectConfigFile {
case "-":
log.Tracef("Reading detection rules from stdin")
detectReader = os.Stdin
default:
log.Tracef("Reading detection rules: %s", detectConfigFile)
detectReader, err = os.Open(detectConfigFile)
if err != nil {
return err
}
}
listSupportedServices, err := flags.GetBool("list-supported-services")
if err != nil {
return err
@ -171,7 +187,7 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
}
if listSupportedServices {
supported, err := setup.ListSupported(detectConfigFile)
supported, err := setup.ListSupported(detectReader)
if err != nil {
return err
}
@ -195,7 +211,7 @@ func runSetupDetect(cmd *cobra.Command, args []string) error {
SnubSystemd: snubSystemd,
}
hubSetup, err := setup.Detect(detectConfigFile, opts)
hubSetup, err := setup.Detect(detectReader, opts)
if err != nil {
return fmt.Errorf("detecting services: %w", err)
}
@ -289,7 +305,12 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error {
return fmt.Errorf("while reading file %s: %w", fromFile, err)
}
if err = setup.InstallHubItems(csConfig, input, dryRun); err != nil {
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
if err = setup.InstallHubItems(hub, input, dryRun); err != nil {
return err
}

232
cmd/crowdsec-cli/simulation.go
View file

@ -6,101 +6,21 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"golang.org/x/exp/slices"
"gopkg.in/yaml.v2"
"slices"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func addToExclusion(name string) error {
csConfig.Cscli.SimulationConfig.Exclusions = append(csConfig.Cscli.SimulationConfig.Exclusions, name)
return nil
type cliSimulation struct {}
func NewCLISimulation() *cliSimulation {
return &cliSimulation{}
}
func removeFromExclusion(name string) error {
index := indexOf(name, csConfig.Cscli.SimulationConfig.Exclusions)
// Remove element from the slice
csConfig.Cscli.SimulationConfig.Exclusions[index] = csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1] = ""
csConfig.Cscli.SimulationConfig.Exclusions = csConfig.Cscli.SimulationConfig.Exclusions[:len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
return nil
}
func enableGlobalSimulation() error {
csConfig.Cscli.SimulationConfig.Simulation = new(bool)
*csConfig.Cscli.SimulationConfig.Simulation = true
csConfig.Cscli.SimulationConfig.Exclusions = []string{}
if err := dumpSimulationFile(); err != nil {
log.Fatalf("unable to dump simulation file: %s", err)
}
log.Printf("global simulation: enabled")
return nil
}
func dumpSimulationFile() error {
newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
if err != nil {
return fmt.Errorf("unable to marshal simulation configuration: %s", err)
}
err = os.WriteFile(csConfig.ConfigPaths.SimulationFilePath, newConfigSim, 0644)
if err != nil {
return fmt.Errorf("write simulation config in '%s' failed: %s", csConfig.ConfigPaths.SimulationFilePath, err)
}
log.Debugf("updated simulation file %s", csConfig.ConfigPaths.SimulationFilePath)
return nil
}
func disableGlobalSimulation() error {
csConfig.Cscli.SimulationConfig.Simulation = new(bool)
*csConfig.Cscli.SimulationConfig.Simulation = false
csConfig.Cscli.SimulationConfig.Exclusions = []string{}
newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
if err != nil {
return fmt.Errorf("unable to marshal new simulation configuration: %s", err)
}
err = os.WriteFile(csConfig.ConfigPaths.SimulationFilePath, newConfigSim, 0644)
if err != nil {
return fmt.Errorf("unable to write new simulation config in '%s' : %s", csConfig.ConfigPaths.SimulationFilePath, err)
}
log.Printf("global simulation: disabled")
return nil
}
func simulationStatus() error {
if csConfig.Cscli.SimulationConfig == nil {
log.Printf("global simulation: disabled (configuration file is missing)")
return nil
}
if *csConfig.Cscli.SimulationConfig.Simulation {
log.Println("global simulation: enabled")
if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
log.Println("Scenarios not in simulation mode :")
for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
log.Printf(" - %s", scenario)
}
}
} else {
log.Println("global simulation: disabled")
if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
log.Println("Scenarios in simulation mode :")
for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
log.Printf(" - %s", scenario)
}
}
}
return nil
}
func NewSimulationCmds() *cobra.Command {
var cmdSimulation = &cobra.Command{
func (cli cliSimulation) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "simulation [command]",
Short: "Manage simulation status of scenarios",
Example: `cscli simulation status
@ -111,9 +31,6 @@ cscli simulation disable crowdsecurity/ssh-bf`,
if err := csConfig.LoadSimulation(); err != nil {
log.Fatal(err)
}
if csConfig.Cscli == nil {
return fmt.Errorf("you must configure cli before using simulation")
}
if csConfig.Cscli.SimulationConfig == nil {
return fmt.Errorf("no simulation configured")
}
@ -125,41 +42,38 @@ cscli simulation disable crowdsecurity/ssh-bf`,
}
},
}
cmdSimulation.Flags().SortFlags = false
cmdSimulation.PersistentFlags().SortFlags = false
cmd.Flags().SortFlags = false
cmd.PersistentFlags().SortFlags = false
cmdSimulation.AddCommand(NewSimulationEnableCmd())
cmdSimulation.AddCommand(NewSimulationDisableCmd())
cmdSimulation.AddCommand(NewSimulationStatusCmd())
cmd.AddCommand(cli.NewEnableCmd())
cmd.AddCommand(cli.NewDisableCmd())
cmd.AddCommand(cli.NewStatusCmd())
return cmdSimulation
return cmd
}
func NewSimulationEnableCmd() *cobra.Command {
func (cli cliSimulation) NewEnableCmd() *cobra.Command {
var forceGlobalSimulation bool
var cmdSimulationEnable = &cobra.Command{
cmd := &cobra.Command{
Use: "enable [scenario] [-global]",
Short: "Enable the simulation, globally or on specified scenarios",
Example: `cscli simulation enable`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if err := csConfig.LoadHub(); err != nil {
hub, err := require.Hub(csConfig, nil)
if err != nil {
log.Fatal(err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Info("Run 'sudo cscli hub update' to get the hub index")
log.Fatalf("Failed to get Hub index : %v", err)
}
if len(args) > 0 {
for _, scenario := range args {
var item = cwhub.GetItem(cwhub.SCENARIOS, scenario)
var item = hub.GetItem(cwhub.SCENARIOS, scenario)
if item == nil {
log.Errorf("'%s' doesn't exist or is not a scenario", scenario)
continue
}
if !item.Installed {
if !item.State.Installed {
log.Warningf("'%s' isn't enabled", scenario)
}
isExcluded := slices.Contains(csConfig.Cscli.SimulationConfig.Exclusions, scenario)
@ -195,15 +109,15 @@ func NewSimulationEnableCmd() *cobra.Command {
}
},
}
cmdSimulationEnable.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Enable global simulation (reverse mode)")
cmd.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Enable global simulation (reverse mode)")
return cmdSimulationEnable
return cmd
}
func NewSimulationDisableCmd() *cobra.Command {
func (cli cliSimulation) NewDisableCmd() *cobra.Command {
var forceGlobalSimulation bool
var cmdSimulationDisable = &cobra.Command{
cmd := &cobra.Command{
Use: "disable [scenario]",
Short: "Disable the simulation mode. Disable only specified scenarios",
Example: `cscli simulation disable`,
@ -244,13 +158,13 @@ func NewSimulationDisableCmd() *cobra.Command {
}
},
}
cmdSimulationDisable.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Disable global simulation (reverse mode)")
cmd.Flags().BoolVarP(&forceGlobalSimulation, "global", "g", false, "Disable global simulation (reverse mode)")
return cmdSimulationDisable
return cmd
}
func NewSimulationStatusCmd() *cobra.Command {
var cmdSimulationStatus = &cobra.Command{
func (cli cliSimulation) NewStatusCmd() *cobra.Command {
cmd := &cobra.Command{
Use: "status",
Short: "Show simulation mode status",
Example: `cscli simulation status`,
@ -264,5 +178,93 @@ func NewSimulationStatusCmd() *cobra.Command {
},
}
return cmdSimulationStatus
return cmd
}
func addToExclusion(name string) error {
csConfig.Cscli.SimulationConfig.Exclusions = append(csConfig.Cscli.SimulationConfig.Exclusions, name)
return nil
}
func removeFromExclusion(name string) error {
index := slices.Index(csConfig.Cscli.SimulationConfig.Exclusions, name)
// Remove element from the slice
csConfig.Cscli.SimulationConfig.Exclusions[index] = csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
csConfig.Cscli.SimulationConfig.Exclusions[len(csConfig.Cscli.SimulationConfig.Exclusions)-1] = ""
csConfig.Cscli.SimulationConfig.Exclusions = csConfig.Cscli.SimulationConfig.Exclusions[:len(csConfig.Cscli.SimulationConfig.Exclusions)-1]
return nil
}
func enableGlobalSimulation() error {
csConfig.Cscli.SimulationConfig.Simulation = new(bool)
*csConfig.Cscli.SimulationConfig.Simulation = true
csConfig.Cscli.SimulationConfig.Exclusions = []string{}
if err := dumpSimulationFile(); err != nil {
log.Fatalf("unable to dump simulation file: %s", err)
}
log.Printf("global simulation: enabled")
return nil
}
func dumpSimulationFile() error {
newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
if err != nil {
return fmt.Errorf("unable to marshal simulation configuration: %s", err)
}
err = os.WriteFile(csConfig.ConfigPaths.SimulationFilePath, newConfigSim, 0o644)
if err != nil {
return fmt.Errorf("write simulation config in '%s' failed: %s", csConfig.ConfigPaths.SimulationFilePath, err)
}
log.Debugf("updated simulation file %s", csConfig.ConfigPaths.SimulationFilePath)
return nil
}
func disableGlobalSimulation() error {
csConfig.Cscli.SimulationConfig.Simulation = new(bool)
*csConfig.Cscli.SimulationConfig.Simulation = false
csConfig.Cscli.SimulationConfig.Exclusions = []string{}
newConfigSim, err := yaml.Marshal(csConfig.Cscli.SimulationConfig)
if err != nil {
return fmt.Errorf("unable to marshal new simulation configuration: %s", err)
}
err = os.WriteFile(csConfig.ConfigPaths.SimulationFilePath, newConfigSim, 0o644)
if err != nil {
return fmt.Errorf("unable to write new simulation config in '%s' : %s", csConfig.ConfigPaths.SimulationFilePath, err)
}
log.Printf("global simulation: disabled")
return nil
}
func simulationStatus() error {
if csConfig.Cscli.SimulationConfig == nil {
log.Printf("global simulation: disabled (configuration file is missing)")
return nil
}
if *csConfig.Cscli.SimulationConfig.Simulation {
log.Println("global simulation: enabled")
if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
log.Println("Scenarios not in simulation mode :")
for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
log.Printf(" - %s", scenario)
}
}
} else {
log.Println("global simulation: disabled")
if len(csConfig.Cscli.SimulationConfig.Exclusions) > 0 {
log.Println("Scenarios in simulation mode :")
for _, scenario := range csConfig.Cscli.SimulationConfig.Exclusions {
log.Printf(" - %s", scenario)
}
}
}
return nil
}

92
cmd/crowdsec-cli/support.go
View file

@ -18,8 +18,9 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
@ -36,6 +37,7 @@ const (
SUPPORT_OS_INFO_PATH = "osinfo.txt"
SUPPORT_PARSERS_PATH = "hub/parsers.txt"
SUPPORT_SCENARIOS_PATH = "hub/scenarios.txt"
SUPPORT_CONTEXTS_PATH = "hub/scenarios.txt"
SUPPORT_COLLECTIONS_PATH = "hub/collections.txt"
SUPPORT_POSTOVERFLOWS_PATH = "hub/postoverflows.txt"
SUPPORT_BOUNCERS_PATH = "lapi/bouncers.txt"
@ -57,10 +59,6 @@ func stripAnsiString(str string) string {
func collectMetrics() ([]byte, []byte, error) {
log.Info("Collecting prometheus metrics")
err := csConfig.LoadPrometheus()
if err != nil {
return nil, nil, err
}
if csConfig.Cscli.PrometheusUrl == "" {
log.Warn("No Prometheus URL configured, metrics will not be collected")
@ -68,13 +66,13 @@ func collectMetrics() ([]byte, []byte, error) {
}
humanMetrics := bytes.NewBuffer(nil)
err = FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl+"/metrics", "human")
err := FormatPrometheusMetrics(humanMetrics, csConfig.Cscli.PrometheusUrl, "human")
if err != nil {
return nil, nil, fmt.Errorf("could not fetch promtheus metrics: %s", err)
}
req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl+"/metrics", nil)
req, err := http.NewRequest(http.MethodGet, csConfig.Cscli.PrometheusUrl, nil)
if err != nil {
return nil, nil, fmt.Errorf("could not create requests to prometheus endpoint: %s", err)
}
@ -131,28 +129,21 @@ func collectOSInfo() ([]byte, error) {
return w.Bytes(), nil
}
func initHub() error {
if err := csConfig.LoadHub(); err != nil {
return fmt.Errorf("cannot load hub: %s", err)
}
if csConfig.Hub == nil {
return fmt.Errorf("hub not configured")
}
func collectHubItems(hub *cwhub.Hub, itemType string) []byte {
var err error
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("cannot set hub branch: %s", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
return fmt.Errorf("no hub index found: %s", err)
}
return nil
}
func collectHubItems(itemType string) []byte {
out := bytes.NewBuffer(nil)
log.Infof("Collecting %s list", itemType)
ListItems(out, []string{itemType}, []string{}, false, true, all)
items := make(map[string][]*cwhub.Item)
if items[itemType], err = selectItems(hub, itemType, nil, true); err != nil {
log.Warnf("could not collect %s list: %s", itemType, err)
}
if err := listItems(out, []string{itemType}, items, false); err != nil {
log.Warnf("could not collect %s list: %s", itemType, err)
}
return out.Bytes()
}
@ -174,7 +165,7 @@ func collectAgents(dbClient *database.Client) ([]byte, error) {
return out.Bytes(), nil
}
func collectAPIStatus(login string, password string, endpoint string, prefix string) []byte {
func collectAPIStatus(login string, password string, endpoint string, prefix string, hub *cwhub.Hub) []byte {
if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil {
return []byte("No agent credentials found, are we LAPI ?")
}
@ -184,7 +175,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
if err != nil {
return []byte(fmt.Sprintf("cannot parse API URL: %s", err))
}
scenarios, err := cwhub.GetInstalledScenariosAsString()
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return []byte(fmt.Sprintf("could not collect scenarios: %s", err))
}
@ -247,8 +238,14 @@ func collectAcquisitionConfig() map[string][]byte {
return ret
}
func NewSupportCmd() *cobra.Command {
var cmdSupport = &cobra.Command{
type cliSupport struct {}
func NewCLISupport() *cliSupport {
return &cliSupport{}
}
func (cli cliSupport) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: "support [action]",
Short: "Provide commands to help during support",
Args: cobra.MinimumNArgs(1),
@ -258,9 +255,15 @@ func NewSupportCmd() *cobra.Command {
},
}
cmd.AddCommand(cli.NewDumpCmd())
return cmd
}
func (cli cliSupport) NewDumpCmd() *cobra.Command {
var outFile string
cmdDump := &cobra.Command{
cmd := &cobra.Command{
Use: "dump",
Short: "Dump all your configuration to a zip file for easier support",
Long: `Dump the following informations:
@ -270,6 +273,7 @@ func NewSupportCmd() *cobra.Command {
- Installed parsers list
- Installed scenarios list
- Installed postoverflows list
- Installed context list
- Bouncers list
- Machines list
- CAPI status
@ -312,13 +316,14 @@ cscli support dump -f /tmp/crowdsec-support.zip
skipAgent = true
}
err = initHub()
hub, err := require.Hub(csConfig, nil)
if err != nil {
log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
skipHub = true
infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())
infos[SUPPORT_SCENARIOS_PATH] = []byte(err.Error())
infos[SUPPORT_POSTOVERFLOWS_PATH] = []byte(err.Error())
infos[SUPPORT_CONTEXTS_PATH] = []byte(err.Error())
infos[SUPPORT_COLLECTIONS_PATH] = []byte(err.Error())
}
@ -351,10 +356,11 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_CROWDSEC_CONFIG_PATH] = collectCrowdsecConfig()
if !skipHub {
infos[SUPPORT_PARSERS_PATH] = collectHubItems(cwhub.PARSERS)
infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(cwhub.SCENARIOS)
infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.PARSERS_OVFLW)
infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(cwhub.COLLECTIONS)
infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS)
infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS)
infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
infos[SUPPORT_CONTEXTS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS)
}
if !skipDB {
@ -376,7 +382,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_CAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Server.OnlineClient.Credentials.Login,
csConfig.API.Server.OnlineClient.Credentials.Password,
csConfig.API.Server.OnlineClient.Credentials.URL,
CAPIURLPrefix)
CAPIURLPrefix,
hub)
}
if !skipLAPI {
@ -384,7 +391,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_LAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Client.Credentials.Login,
csConfig.API.Client.Credentials.Password,
csConfig.API.Client.Credentials.URL,
LAPIURLPrefix)
LAPIURLPrefix,
hub)
infos[SUPPORT_CROWDSEC_PROFILE_PATH] = collectCrowdsecProfile()
}
@ -415,7 +423,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
log.Fatalf("could not finalize zip file: %s", err)
}
err = os.WriteFile(outFile, w.Bytes(), 0600)
err = os.WriteFile(outFile, w.Bytes(), 0o600)
if err != nil {
log.Fatalf("could not write zip file to %s: %s", outFile, err)
}
@ -423,8 +431,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
log.Infof("Written zip file to %s", outFile)
},
}
cmdDump.Flags().StringVarP(&outFile, "outFile", "f", "", "File to dump the information to")
cmdSupport.AddCommand(cmdDump)
return cmdSupport
cmd.Flags().StringVarP(&outFile, "outFile", "f", "", "File to dump the information to")
return cmd
}

668
cmd/crowdsec-cli/utils.go
View file

@ -1,277 +1,24 @@
package main
import (
"encoding/csv"
"encoding/json"
"fmt"
"io"
"math"
"net"
"net/http"
"os"
"strconv"
"strings"
"time"
"github.com/fatih/color"
dto "github.com/prometheus/client_model/go"
"github.com/prometheus/prom2json"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/texttheater/golang-levenshtein/levenshtein"
"golang.org/x/exp/slices"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
const MaxDistance = 7
func printHelp(cmd *cobra.Command) {
err := cmd.Help()
if err != nil {
if err := cmd.Help(); err != nil {
log.Fatalf("unable to print help(): %s", err)
}
}
func indexOf(s string, slice []string) int {
for i, elem := range slice {
if s == elem {
return i
}
}
return -1
}
func LoadHub() error {
if err := csConfig.LoadHub(); err != nil {
log.Fatal(err)
}
if csConfig.Hub == nil {
return fmt.Errorf("unable to load hub")
}
if err := cwhub.SetHubBranch(); err != nil {
log.Warningf("unable to set hub branch (%s), default to master", err)
}
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
return fmt.Errorf("Failed to get Hub index : '%w'. Run 'sudo cscli hub update' to get the hub index", err)
}
return nil
}
func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
errMsg := ""
if score < MaxDistance {
errMsg = fmt.Sprintf("unable to find %s '%s', did you mean %s ?", itemType, baseItem, suggestItem)
} else {
errMsg = fmt.Sprintf("unable to find %s '%s'", itemType, baseItem)
}
if ignoreErr {
log.Error(errMsg)
} else {
log.Fatalf(errMsg)
}
}
func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
allItems := make([]string, 0)
nearestScore := 100
nearestItem := &cwhub.Item{}
hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
for _, item := range hubItems {
allItems = append(allItems, item.Name)
}
for _, s := range allItems {
d := levenshtein.DistanceForStrings([]rune(itemName), []rune(s), levenshtein.DefaultOptions)
if d < nearestScore {
nearestScore = d
nearestItem = cwhub.GetItem(itemType, s)
}
}
return nearestItem, nearestScore
}
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := LoadHub(); err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
for _, item := range hubItems {
if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
comp = append(comp, item.Name)
}
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := LoadHub(); err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
var items []string
var err error
switch itemType {
case cwhub.PARSERS:
items, err = cwhub.GetInstalledParsersAsString()
case cwhub.SCENARIOS:
items, err = cwhub.GetInstalledScenariosAsString()
case cwhub.PARSERS_OVFLW:
items, err = cwhub.GetInstalledPostOverflowsAsString()
case cwhub.COLLECTIONS:
items, err = cwhub.GetInstalledCollectionsAsString()
default:
return nil, cobra.ShellCompDirectiveDefault
}
if err != nil {
cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
if toComplete != "" {
for _, item := range items {
if strings.Contains(item, toComplete) {
comp = append(comp, item)
}
}
} else {
comp = items
}
cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
return comp, cobra.ShellCompDirectiveNoFileComp
}
func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) {
var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus)
for _, itemType := range itemTypes {
itemName := ""
if len(args) == 1 {
itemName = args[0]
}
hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all)
}
if csConfig.Cscli.Output == "human" {
for _, itemType := range itemTypes {
var statuses []cwhub.ItemHubStatus
var ok bool
if statuses, ok = hubStatusByItemType[itemType]; !ok {
log.Errorf("unknown item type: %s", itemType)
continue
}
listHubItemTable(out, "\n"+strings.ToUpper(itemType), statuses)
}
} else if csConfig.Cscli.Output == "json" {
x, err := json.MarshalIndent(hubStatusByItemType, "", " ")
if err != nil {
log.Fatalf("failed to unmarshal")
}
out.Write(x)
} else if csConfig.Cscli.Output == "raw" {
csvwriter := csv.NewWriter(out)
if showHeader {
header := []string{"name", "status", "version", "description"}
if showType {
header = append(header, "type")
}
err := csvwriter.Write(header)
if err != nil {
log.Fatalf("failed to write header: %s", err)
}
}
for _, itemType := range itemTypes {
var statuses []cwhub.ItemHubStatus
var ok bool
if statuses, ok = hubStatusByItemType[itemType]; !ok {
log.Errorf("unknown item type: %s", itemType)
continue
}
for _, status := range statuses {
if status.LocalVersion == "" {
status.LocalVersion = "n/a"
}
row := []string{
status.Name,
status.Status,
status.LocalVersion,
status.Description,
}
if showType {
row = append(row, itemType)
}
err := csvwriter.Write(row)
if err != nil {
log.Fatalf("failed to write raw output : %s", err)
}
}
}
csvwriter.Flush()
}
}
func InspectItem(name string, objecitemType string) {
hubItem := cwhub.GetItem(objecitemType, name)
if hubItem == nil {
log.Fatalf("unable to retrieve item.")
}
var b []byte
var err error
switch csConfig.Cscli.Output {
case "human", "raw":
b, err = yaml.Marshal(*hubItem)
if err != nil {
log.Fatalf("unable to marshal item : %s", err)
}
case "json":
b, err = json.MarshalIndent(*hubItem, "", " ")
if err != nil {
log.Fatalf("unable to marshal item : %s", err)
}
}
fmt.Printf("%s", string(b))
if csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
return
}
if prometheusURL == "" {
//This is technically wrong to do this, as the prometheus section contains a listen address, not an URL to query prometheus
//But for ease of use, we will use the listen address as the prometheus URL because it will be 127.0.0.1 in the default case
listenAddr := csConfig.Prometheus.ListenAddr
if listenAddr == "" {
listenAddr = "127.0.0.1"
}
listenPort := csConfig.Prometheus.ListenPort
if listenPort == 0 {
listenPort = 6060
}
prometheusURL = fmt.Sprintf("http://%s:%d/metrics", listenAddr, listenPort)
log.Debugf("No prometheus URL provided using: %s", prometheusURL)
}
fmt.Printf("\nCurrent metrics : \n")
ShowMetrics(hubItem)
}
func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
/*if a range is provided, change the scope*/
if *ipRange != "" {
_, _, err := net.ParseCIDR(*ipRange)
@ -300,419 +47,7 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
return nil
}
func ShowMetrics(hubItem *cwhub.Item) {
switch hubItem.Type {
case cwhub.PARSERS:
metrics := GetParserMetric(prometheusURL, hubItem.Name)
parserMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.SCENARIOS:
metrics := GetScenarioMetric(prometheusURL, hubItem.Name)
scenarioMetricsTable(color.Output, hubItem.Name, metrics)
case cwhub.COLLECTIONS:
for _, item := range hubItem.Parsers {
metrics := GetParserMetric(prometheusURL, item)
parserMetricsTable(color.Output, item, metrics)
}
for _, item := range hubItem.Scenarios {
metrics := GetScenarioMetric(prometheusURL, item)
scenarioMetricsTable(color.Output, item, metrics)
}
for _, item := range hubItem.Collections {
hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
if hubItem == nil {
log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
}
ShowMetrics(hubItem)
}
default:
log.Errorf("item of type '%s' is unknown", hubItem.Type)
}
}
// GetParserMetric is a complete rip from prom2json
func GetParserMetric(url string, itemName string) map[string]map[string]int {
stats := make(map[string]map[string]int)
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
source, ok := metric.Labels["source"]
if !ok {
log.Debugf("no source in Metric %v", metric.Labels)
} else {
if srctype, ok := metric.Labels["type"]; ok {
source = srctype + ":" + source
}
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_reader_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
stats[source]["parsed"] = 0
stats[source]["reads"] = 0
stats[source]["unparsed"] = 0
stats[source]["hits"] = 0
}
stats[source]["reads"] += ival
case "cs_parser_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_parser_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
case "cs_node_hits_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["hits"] += ival
case "cs_node_hits_ok_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["parsed"] += ival
case "cs_node_hits_ko_total":
if _, ok := stats[source]; !ok {
stats[source] = make(map[string]int)
}
stats[source]["unparsed"] += ival
default:
continue
}
}
}
return stats
}
func GetScenarioMetric(url string, itemName string) map[string]int {
stats := make(map[string]int)
stats["instantiation"] = 0
stats["curr_count"] = 0
stats["overflow"] = 0
stats["pour"] = 0
stats["underflow"] = 0
result := GetPrometheusMetric(url)
for idx, fam := range result {
if !strings.HasPrefix(fam.Name, "cs_") {
continue
}
log.Tracef("round %d", idx)
for _, m := range fam.Metrics {
metric, ok := m.(prom2json.Metric)
if !ok {
log.Debugf("failed to convert metric to prom2json.Metric")
continue
}
name, ok := metric.Labels["name"]
if !ok {
log.Debugf("no name in Metric %v", metric.Labels)
}
if name != itemName {
continue
}
value := m.(prom2json.Metric).Value
fval, err := strconv.ParseFloat(value, 32)
if err != nil {
log.Errorf("Unexpected int value %s : %s", value, err)
continue
}
ival := int(fval)
switch fam.Name {
case "cs_bucket_created_total":
stats["instantiation"] += ival
case "cs_buckets":
stats["curr_count"] += ival
case "cs_bucket_overflowed_total":
stats["overflow"] += ival
case "cs_bucket_poured_total":
stats["pour"] += ival
case "cs_bucket_underflowed_total":
stats["underflow"] += ival
default:
continue
}
}
}
return stats
}
// it's a rip of the cli version, but in silent-mode
func silenceInstallItem(name string, obtype string) (string, error) {
var item = cwhub.GetItem(obtype, name)
if item == nil {
return "", fmt.Errorf("error retrieving item")
}
it := *item
if downloadOnly && it.Downloaded && it.UpToDate {
return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
}
it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction, false)
if err != nil {
return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
}
if err := cwhub.AddItem(obtype, it); err != nil {
return "", err
}
if downloadOnly {
return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
}
it, err = cwhub.EnableItem(csConfig.Hub, it)
if err != nil {
return "", fmt.Errorf("error while enabling %s : %v", it.Name, err)
}
if err := cwhub.AddItem(obtype, it); err != nil {
return "", err
}
return fmt.Sprintf("Enabled %s", it.Name), nil
}
func GetPrometheusMetric(url string) []*prom2json.Family {
mfChan := make(chan *dto.MetricFamily, 1024)
// Start with the DefaultTransport for sane defaults.
transport := http.DefaultTransport.(*http.Transport).Clone()
// Conservatively disable HTTP keep-alives as this program will only
// ever need a single HTTP request.
transport.DisableKeepAlives = true
// Timeout early if the server doesn't even return the headers.
transport.ResponseHeaderTimeout = time.Minute
go func() {
defer trace.CatchPanic("crowdsec/GetPrometheusMetric")
err := prom2json.FetchMetricFamilies(url, mfChan, transport)
if err != nil {
log.Fatalf("failed to fetch prometheus metrics : %v", err)
}
}()
result := []*prom2json.Family{}
for mf := range mfChan {
result = append(result, prom2json.NewFamily(mf))
}
log.Debugf("Finished reading prometheus output, %d entries", len(result))
return result
}
func RestoreHub(dirPath string) error {
var err error
if err := csConfig.LoadHub(); err != nil {
return err
}
if err := cwhub.SetHubBranch(); err != nil {
return fmt.Errorf("error while setting hub branch: %s", err)
}
for _, itype := range cwhub.ItemTypes {
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
if _, err = os.Stat(itemDirectory); err != nil {
log.Infof("no %s in backup", itype)
continue
}
/*restore the upstream items*/
upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
file, err := os.ReadFile(upstreamListFN)
if err != nil {
return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
}
var upstreamList []string
err = json.Unmarshal(file, &upstreamList)
if err != nil {
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
}
for _, toinstall := range upstreamList {
label, err := silenceInstallItem(toinstall, itype)
if err != nil {
log.Errorf("Error while installing %s : %s", toinstall, err)
} else if label != "" {
log.Infof("Installed %s : %s", toinstall, label)
} else {
log.Printf("Installed %s : ok", toinstall)
}
}
/*restore the local and tainted items*/
files, err := os.ReadDir(itemDirectory)
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
}
for _, file := range files {
//this was the upstream data
if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
continue
}
if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
//we expect a stage here
if !file.IsDir() {
continue
}
stage := file.Name()
stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
}
/*find items*/
ifiles, err := os.ReadDir(itemDirectory + "/" + stage + "/")
if err != nil {
return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
}
//finally copy item
for _, tfile := range ifiles {
log.Infof("Going to restore local/tainted [%s]", tfile.Name())
sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
} else {
log.Infof("Going to restore local/tainted [%s]", file.Name())
sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
if err = CopyFile(sourceFile, destinationFile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
}
log.Infof("restored %s to %s", sourceFile, destinationFile)
}
}
}
return nil
}
func BackupHub(dirPath string) error {
var err error
var itemDirectory string
var upstreamParsers []string
for _, itemType := range cwhub.ItemTypes {
clog := log.WithFields(log.Fields{
"type": itemType,
})
itemMap := cwhub.GetItemMap(itemType)
if itemMap == nil {
clog.Infof("No %s to backup.", itemType)
continue
}
itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
}
upstreamParsers = []string{}
for k, v := range itemMap {
clog = clog.WithFields(log.Fields{
"file": v.Name,
})
if !v.Installed { //only backup installed ones
clog.Debugf("[%s] : not installed", k)
continue
}
//for the local/tainted ones, we backup the full file
if v.Tainted || v.Local || !v.UpToDate {
//we need to backup stages for parsers
if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
}
}
clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
if err = CopyFile(v.LocalPath, tfile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
}
clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
continue
}
clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
upstreamParsers = append(upstreamParsers, v.Name)
}
//write the upstream items
upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
if err != nil {
return fmt.Errorf("failed marshaling upstream parsers : %s", err)
}
err = os.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
if err != nil {
return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
}
clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
}
return nil
}
type unit struct {
value int64
symbol string
}
var ranges = []unit{
{value: 1e18, symbol: "E"},
{value: 1e15, symbol: "P"},
{value: 1e12, symbol: "T"},
{value: 1e9, symbol: "G"},
{value: 1e6, symbol: "M"},
{value: 1e3, symbol: "k"},
{value: 1, symbol: ""},
}
func formatNumber(num int) string {
goodUnit := unit{}
for _, u := range ranges {
if int64(num) >= u.value {
goodUnit = u
break
}
}
if goodUnit.value == 1 {
return fmt.Sprintf("%d%s", num, goodUnit.symbol)
}
res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
}
func getDBClient() (*database.Client, error) {
var err error
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
return nil, err
}
@ -744,5 +79,4 @@ func removeFromSlice(val string, slice []string) []string {
}
return slice
}

32
cmd/crowdsec-cli/utils_table.go
View file

@ -3,6 +3,7 @@ package main
import (
"fmt"
"io"
"strconv"
"github.com/aquasecurity/table"
"github.com/enescakir/emoji"
@ -10,14 +11,15 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func listHubItemTable(out io.Writer, title string, statuses []cwhub.ItemHubStatus) {
func listHubItemTable(out io.Writer, title string, items []*cwhub.Item) {
t := newLightTable(out)
t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
for _, status := range statuses {
t.AddRow(status.Name, status.UTF8_Status, status.LocalVersion, status.LocalPath)
for _, item := range items {
status := fmt.Sprintf("%v %s", item.State.Emoji(), item.State.Text())
t.AddRow(item.Name, status, item.State.LocalVersion, item.State.LocalPath)
}
renderTableTitle(out, title)
t.Render()
@ -31,11 +33,11 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
t.SetHeaders("Current Count", "Overflows", "Instantiated", "Poured", "Expired")
t.AddRow(
fmt.Sprintf("%d", metrics["curr_count"]),
fmt.Sprintf("%d", metrics["overflow"]),
fmt.Sprintf("%d", metrics["instantiation"]),
fmt.Sprintf("%d", metrics["pour"]),
fmt.Sprintf("%d", metrics["underflow"]),
strconv.Itoa(metrics["curr_count"]),
strconv.Itoa(metrics["overflow"]),
strconv.Itoa(metrics["instantiation"]),
strconv.Itoa(metrics["pour"]),
strconv.Itoa(metrics["underflow"]),
)
renderTableTitle(out, fmt.Sprintf("\n - (Scenario) %s:", itemName))
@ -43,23 +45,25 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
}
func parserMetricsTable(out io.Writer, itemName string, metrics map[string]map[string]int) {
skip := true
t := newTable(out)
t.SetHeaders("Parsers", "Hits", "Parsed", "Unparsed")
// don't show table if no hits
showTable := false
for source, stats := range metrics {
if stats["hits"] > 0 {
t.AddRow(
source,
fmt.Sprintf("%d", stats["hits"]),
fmt.Sprintf("%d", stats["parsed"]),
fmt.Sprintf("%d", stats["unparsed"]),
strconv.Itoa(stats["hits"]),
strconv.Itoa(stats["parsed"]),
strconv.Itoa(stats["unparsed"]),
)
skip = false
showTable = true
}
}
if !skip {
if showTable {
renderTableTitle(out, fmt.Sprintf("\n - (Parser) %s:", itemName))
t.Render()
}

9
cmd/crowdsec/Makefile
View file

@ -4,10 +4,9 @@ ifeq ($(OS), Windows_NT)
EXT = .exe
endif
# Go parameters
GOCMD = go
GOBUILD = $(GOCMD) build
GOTEST = $(GOCMD) test
GO = go
GOBUILD = $(GO) build
GOTEST = $(GO) test
CROWDSEC_BIN = crowdsec$(EXT)
# names longer than 15 chars break 'pgrep'
@ -23,7 +22,7 @@ SYSTEMD_PATH_FILE = "/etc/systemd/system/crowdsec.service"
all: clean test build
build: clean
$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(CROWDSEC_BIN)
$(GOBUILD) $(LD_OPTS) -o $(CROWDSEC_BIN)
test:
$(GOTEST) $(LD_OPTS) -v ./...

2
cmd/crowdsec/api.go
View file

@ -8,7 +8,7 @@ import (
"github.com/pkg/errors"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/pkg/apiserver"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"

28
cmd/crowdsec/crowdsec.go
View file

@ -10,9 +10,11 @@ import (
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/pkg/acquisition"
"github.com/crowdsecurity/crowdsec/pkg/appsec"
"github.com/crowdsecurity/crowdsec/pkg/alertcontext"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
@ -20,31 +22,35 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/types"
)
func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, error) {
var err error
// Populate cwhub package tools
if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
return nil, fmt.Errorf("while loading hub index: %w", err)
if err = alertcontext.LoadConsoleContext(cConfig, hub); err != nil {
return nil, fmt.Errorf("while loading context: %w", err)
}
// Start loading configs
csParsers := parser.NewParsers()
csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
return nil, fmt.Errorf("while loading parsers: %w", err)
}
if err := LoadBuckets(cConfig); err != nil {
if err := LoadBuckets(cConfig, hub); err != nil {
return nil, fmt.Errorf("while loading scenarios: %w", err)
}
if err := appsec.LoadAppsecRules(hub); err != nil {
return nil, fmt.Errorf("while loading appsec rules: %w", err)
}
if err := LoadAcquisition(cConfig); err != nil {
return nil, fmt.Errorf("while loading acquisition config: %w", err)
}
return csParsers, nil
}
func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.Hub) error {
inputEventChan = make(chan types.Event)
inputLineChan = make(chan types.Event)
@ -99,7 +105,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ {
outputsTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/runOutput")
if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials); err != nil {
if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials, hub); err != nil {
log.Fatalf("starting outputs error : %s", err)
return err
}
@ -131,7 +137,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error {
return nil
}
func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady chan bool) {
func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub.Hub, agentReady chan bool) {
crowdsecTomb.Go(func() error {
defer trace.CatchPanic("crowdsec/serveCrowdsec")
go func() {
@ -139,7 +145,7 @@ func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady
// this logs every time, even at config reload
log.Debugf("running agent after %s ms", time.Since(crowdsecT0))
agentReady <- true
if err := runCrowdsec(cConfig, parsers); err != nil {
if err := runCrowdsec(cConfig, parsers, hub); err != nil {
log.Fatalf("unable to start crowdsec routines: %s", err)
}
}()

45
cmd/crowdsec/main.go
View file

@ -70,24 +70,25 @@ type Flags struct {
WinSvc string
DisableCAPI bool
Transform string
OrderEvent bool
}
type labelsMap map[string]string
func LoadBuckets(cConfig *csconfig.Config) error {
func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error {
var (
err error
files []string
)
for _, hubScenarioItem := range cwhub.GetItemMap(cwhub.SCENARIOS) {
if hubScenarioItem.Installed {
files = append(files, hubScenarioItem.LocalPath)
for _, hubScenarioItem := range hub.GetItemMap(cwhub.SCENARIOS) {
if hubScenarioItem.State.Installed {
files = append(files, hubScenarioItem.State.LocalPath)
}
}
buckets = leakybucket.NewBuckets()
log.Infof("Loading %d scenario files", len(files))
holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, files, &bucketsTomb, buckets)
holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, hub, files, &bucketsTomb, buckets, flags.OrderEvent)
if err != nil {
return fmt.Errorf("scenario loading failed: %v", err)
@ -110,7 +111,7 @@ func LoadAcquisition(cConfig *csconfig.Config) error {
dataSources, err = acquisition.LoadAcquisitionFromDSN(flags.OneShotDSN, flags.Labels, flags.Transform)
if err != nil {
return fmt.Errorf("failed to configure datasource for %s: %w", flags.OneShotDSN, err)
return errors.Wrapf(err, "failed to configure datasource for %s", flags.OneShotDSN)
}
} else {
dataSources, err = acquisition.LoadAcquisitionFromFile(cConfig.Crowdsec)
@ -137,11 +138,13 @@ func (l *labelsMap) String() string {
}
func (l labelsMap) Set(label string) error {
split := strings.Split(label, ":")
if len(split) != 2 {
return errors.Wrapf(errors.New("Bad Format"), "for Label '%s'", label)
for _, pair := range strings.Split(label, ",") {
split := strings.Split(pair, ":")
if len(split) != 2 {
return fmt.Errorf("invalid format for label '%s', must be key:value", pair)
}
l[split[0]] = split[1]
}
l[split[0]] = split[1]
return nil
}
@ -164,6 +167,7 @@ func (f *Flags) Parse() {
flag.BoolVar(&f.DisableAgent, "no-cs", false, "disable crowdsec agent")
flag.BoolVar(&f.DisableAPI, "no-api", false, "disable local API")
flag.BoolVar(&f.DisableCAPI, "no-capi", false, "disable communication with Central API")
flag.BoolVar(&f.OrderEvent, "order-event", false, "enforce event ordering with significant performance cost")
if runtime.GOOS == "windows" {
flag.StringVar(&f.WinSvc, "winsvc", "", "Windows service Action: Install, Remove etc..")
}
@ -208,11 +212,7 @@ func newLogLevel(curLevelPtr *log.Level, f *Flags) *log.Level {
func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*csconfig.Config, error) {
cConfig, _, err := csconfig.NewConfig(configFile, disableAgent, disableAPI, quiet)
if err != nil {
return nil, err
}
if (cConfig.Common == nil || *cConfig.Common == csconfig.CommonCfg{}) {
return nil, fmt.Errorf("unable to load configuration: common section is empty")
return nil, fmt.Errorf("while loading configuration file: %w", err)
}
cConfig.Common.LogLevel = newLogLevel(cConfig.Common.LogLevel, flags)
@ -224,11 +224,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
dumpStates = true
}
// Configuration paths are dependency to load crowdsec configuration
if err := cConfig.LoadConfigurationPaths(); err != nil {
return nil, err
}
if flags.SingleFileType != "" && flags.OneShotDSN != "" {
// if we're in time-machine mode, we don't want to log to file
cConfig.Common.LogMedia = "stdout"
@ -247,13 +242,13 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
return nil, err
}
if !flags.DisableAgent {
if !cConfig.DisableAgent {
if err := cConfig.LoadCrowdsec(); err != nil {
return nil, err
}
}
if !flags.DisableAPI {
if !cConfig.DisableAPI {
if err := cConfig.LoadAPIServer(); err != nil {
return nil, err
}
@ -267,10 +262,6 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
return nil, errors.New("You must run at least the API Server or crowdsec")
}
if flags.TestMode && !cConfig.DisableAgent {
cConfig.Crowdsec.LintOnly = true
}
if flags.OneShotDSN != "" && flags.SingleFileType == "" {
return nil, errors.New("-dsn requires a -type argument")
}
@ -288,7 +279,7 @@ func LoadConfig(configFile string, disableAgent bool, disableAPI bool, quiet boo
cConfig.API.Server.OnlineClient = nil
}
/*if the api is disabled as well, just read file and exit, don't daemonize*/
if flags.DisableAPI {
if cConfig.DisableAPI {
cConfig.Common.Daemonize = false
}
log.Infof("single file mode : log_media=%s daemonize=%t", cConfig.Common.LogMedia, cConfig.Common.Daemonize)

18
cmd/crowdsec/metrics.go
View file

@ -9,8 +9,8 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/go-cs-lib/version"
v1 "github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers/v1"
"github.com/crowdsecurity/crowdsec/pkg/cache"
@ -151,14 +151,6 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
if !config.Enabled {
return
}
if config.ListenAddr == "" {
log.Warning("prometheus is enabled, but the listen address is empty, using '127.0.0.1'")
config.ListenAddr = "127.0.0.1"
}
if config.ListenPort == 0 {
log.Warning("prometheus is enabled, but the listen port is empty, using '6060'")
config.ListenPort = 6060
}
// Registering prometheus
// If in aggregated mode, do not register events associated with a source, to keep the cardinality low
@ -169,7 +161,8 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow,
v1.LapiRouteHits,
leaky.BucketsCurrentCount,
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics)
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics,
)
} else {
log.Infof("Loading prometheus collectors")
prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo,
@ -178,7 +171,8 @@ func registerPrometheus(config *csconfig.PrometheusCfg) {
v1.LapiRouteHits, v1.LapiMachineHits, v1.LapiBouncerHits, v1.LapiNilDecisions, v1.LapiNonNilDecisions, v1.LapiResponseTime,
leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, leaky.BucketsCurrentCount,
globalActiveDecisions, globalAlerts,
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics)
cache.CacheMetrics, exprhelpers.RegexpCacheMetrics,
)
}
}

24
cmd/crowdsec/output.go
View file

@ -10,7 +10,7 @@ import (
"github.com/go-openapi/strfmt"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
@ -62,7 +62,8 @@ func PushAlerts(alerts []types.RuntimeAlert, client *apiclient.ApiClient) error
var bucketOverflows []types.Event
func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky.Buckets,
postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node, apiConfig csconfig.ApiCredentialsCfg) error {
postOverflowCTX parser.UnixParserCtx, postOverflowNodes []parser.Node,
apiConfig csconfig.ApiCredentialsCfg, hub *cwhub.Hub) error {
var err error
ticker := time.NewTicker(1 * time.Second)
@ -70,7 +71,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
var cache []types.RuntimeAlert
var cacheMutex sync.Mutex
scenarios, err := cwhub.GetInstalledScenariosAsString()
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("loading list of installed hub scenarios: %w", err)
}
@ -93,7 +94,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
URL: apiURL,
PapiURL: papiURL,
VersionPrefix: "v1",
UpdateScenario: cwhub.GetInstalledScenariosAsString,
UpdateScenario: func() ([]string, error) {return hub.GetInstalledItemNames(cwhub.SCENARIOS)},
})
if err != nil {
return fmt.Errorf("new client api: %w", err)
@ -145,13 +146,6 @@ LOOP:
}
break LOOP
case event := <-overflow:
//if the Alert is nil, it's to signal bucket is ready for GC, don't track this
if dumpStates && event.Overflow.Alert != nil {
if bucketOverflows == nil {
bucketOverflows = make([]types.Event, 0)
}
bucketOverflows = append(bucketOverflows, event)
}
/*if alert is empty and mapKey is present, the overflow is just to cleanup bucket*/
if event.Overflow.Alert == nil && event.Overflow.Mapkey != "" {
buckets.Bucket_map.Delete(event.Overflow.Mapkey)
@ -163,6 +157,14 @@ LOOP:
return fmt.Errorf("postoverflow failed : %s", err)
}
log.Printf("%s", *event.Overflow.Alert.Message)
//if the Alert is nil, it's to signal bucket is ready for GC, don't track this
//dump after postoveflow processing to avoid missing whitelist info
if dumpStates && event.Overflow.Alert != nil {
if bucketOverflows == nil {
bucketOverflows = make([]types.Event, 0)
}
bucketOverflows = append(bucketOverflows, event)
}
if event.Overflow.Whitelisted {
log.Printf("[%s] is whitelisted, skip.", *event.Overflow.Alert.Message)
continue

7
cmd/crowdsec/parse.go
View file

@ -22,6 +22,13 @@ LOOP:
if !event.Process {
continue
}
/*Application security engine is going to generate 2 events:
- one that is treated as a log and can go to scenarios
- another one that will go directly to LAPI*/
if event.Type == types.APPSEC {
outputEventChan <- event
continue
}
if event.Line.Module == "" {
log.Errorf("empty event.Line.Module field, the acquisition module must set it ! : %+v", event.Line)
continue

7
cmd/crowdsec/run_in_svc.go
View file

@ -1,5 +1,4 @@
//go:build linux || freebsd || netbsd || openbsd || solaris || !windows
// +build linux freebsd netbsd openbsd solaris !windows
//go:build !windows
package main
@ -10,8 +9,8 @@ import (
log "github.com/sirupsen/logrus"
"github.com/sirupsen/logrus/hooks/writer"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/database"

4
cmd/crowdsec/run_in_svc_windows.go
View file

@ -6,8 +6,8 @@ import (
log "github.com/sirupsen/logrus"
"golang.org/x/sys/windows/svc"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/go-cs-lib/pkg/version"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/database"

45
cmd/crowdsec/serve.go
View file

@ -10,10 +10,11 @@ import (
log "github.com/sirupsen/logrus"
"gopkg.in/tomb.v2"
"github.com/crowdsecurity/go-cs-lib/pkg/csdaemon"
"github.com/crowdsecurity/go-cs-lib/pkg/trace"
"github.com/crowdsecurity/go-cs-lib/csdaemon"
"github.com/crowdsecurity/go-cs-lib/trace"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
@ -76,7 +77,12 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
}
if !cConfig.DisableAgent {
csParsers, err := initCrowdsec(cConfig)
hub, err := cwhub.NewHub(cConfig.Hub, nil, false)
if err != nil {
return nil, fmt.Errorf("while loading hub index: %w", err)
}
csParsers, err := initCrowdsec(cConfig, hub)
if err != nil {
return nil, fmt.Errorf("unable to init crowdsec: %w", err)
}
@ -93,7 +99,7 @@ func reloadHandler(sig os.Signal) (*csconfig.Config, error) {
}
agentReady := make(chan bool, 1)
serveCrowdsec(csParsers, cConfig, agentReady)
serveCrowdsec(csParsers, cConfig, hub, agentReady)
}
log.Printf("Reload is finished")
@ -141,12 +147,24 @@ func ShutdownCrowdsecRoutines() error {
time.Sleep(1 * time.Second) // ugly workaround for now
outputsTomb.Kill(nil)
if err := outputsTomb.Wait(); err != nil {
log.Warningf("Ouputs returned error : %s", err)
reterr = err
done := make(chan error, 1)
go func() {
done <- outputsTomb.Wait()
}()
// wait for outputs to finish, max 3 seconds
select {
case err := <-done:
if err != nil {
log.Warningf("Outputs returned error : %s", err)
reterr = err
}
log.Debugf("outputs are done")
case <-time.After(3 * time.Second):
// this can happen if outputs are stuck in a http retry loop
log.Warningf("Outputs didn't finish in time, some events may have not been flushed")
}
log.Debugf("outputs are done")
// He's dead, Jim.
crowdsecTomb.Kill(nil)
@ -330,21 +348,26 @@ func Serve(cConfig *csconfig.Config, apiReady chan bool, agentReady chan bool) e
}
if !cConfig.DisableAgent {
csParsers, err := initCrowdsec(cConfig)
hub, err := cwhub.NewHub(cConfig.Hub, nil, false)
if err != nil {
return fmt.Errorf("while loading hub index: %w", err)
}
csParsers, err := initCrowdsec(cConfig, hub)
if err != nil {
return fmt.Errorf("crowdsec init: %w", err)
}
// if it's just linting, we're done
if !flags.TestMode {
serveCrowdsec(csParsers, cConfig, agentReady)
serveCrowdsec(csParsers, cConfig, hub, agentReady)
}
} else {
agentReady <- true
}
if flags.TestMode {
log.Infof("test done")
log.Infof("Configuration test done")
pluginBroker.Kill()
os.Exit(0)
}

1
cmd/crowdsec/win_service.go
View file

@ -3,7 +3,6 @@
// license that can be found in the LICENSE file.
//go:build windows
// +build windows
package main

1
cmd/crowdsec/win_service_install.go
View file

@ -3,7 +3,6 @@
// license that can be found in the LICENSE file.
//go:build windows
// +build windows
package main

2
cmd/crowdsec/win_service_manage.go
View file

@ -2,7 +2,7 @@
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// +build windows
//go:build windows
package main

9
plugins/notifications/http/Makefile → cmd/notification-dummy/Makefile
View file

@ -4,14 +4,13 @@ ifeq ($(OS), Windows_NT)
EXT = .exe
endif
PLUGIN=http
BINARY_NAME = notification-$(PLUGIN)$(EXT)
GO = go
GOBUILD = $(GO) build
GOCMD = go
GOBUILD = $(GOCMD) build
BINARY_NAME = notification-dummy$(EXT)
build: clean
$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
.PHONY: clean
clean:

0
plugins/notifications/dummy/dummy.yaml → cmd/notification-dummy/dummy.yaml
View file

0
plugins/notifications/dummy/main.go → cmd/notification-dummy/main.go
View file

9
plugins/notifications/slack/Makefile → cmd/notification-email/Makefile
View file

@ -4,14 +4,13 @@ ifeq ($(OS), Windows_NT)
EXT = .exe
endif
PLUGIN=slack
BINARY_NAME = notification-$(PLUGIN)$(EXT)
GO = go
GOBUILD = $(GO) build
GOCMD = go
GOBUILD = $(GOCMD) build
BINARY_NAME = notification-email$(EXT)
build: clean
$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
.PHONY: clean
clean:

14
plugins/notifications/email/email.yaml → cmd/notification-email/email.yaml
View file

@ -15,12 +15,14 @@ timeout: 20s # Time to wait for response from the plugin before conside
# The following template receives a list of models.Alert objects
# The output goes in the email message body
format: |
<html><body>
{{range . -}}
{{$alert := . -}}
{{range .Decisions -}}
<html><body><p><a href=https://www.whois.com/whois/{{.Value}}>{{.Value}}</a> will get <b>{{.Type}}</b> for next <b>{{.Duration}}</b> for triggering <b>{{.Scenario}}</b> on machine <b>{{$alert.MachineID}}</b>.</p> <p><a href=https://app.crowdsec.net/cti/{{.Value}}>CrowdSec CTI</a></p></body></html>
<p><a href="https://www.whois.com/whois/{{.Value}}">{{.Value}}</a> will get <b>{{.Type}}</b> for next <b>{{.Duration}}</b> for triggering <b>{{.Scenario}}</b> on machine <b>{{$alert.MachineID}}</b>.</p> <p><a href="https://app.crowdsec.net/cti/{{.Value}}">CrowdSec CTI</a></p>
{{end -}}
{{end -}}
</body></html>
smtp_host: # example: smtp.gmail.com
smtp_username: # Replace with your actual username
@ -35,7 +37,15 @@ receiver_emails:
# - email2@gmail.com
# One of "ssltls", "starttls", "none"
encryption_type: ssltls
encryption_type: "ssltls"
# If you need to set the HELO hostname:
# helo_host: "localhost"
# If the email server is hitting the default timeouts (10 seconds), you can increase them here
#
# connect_timeout: 10s
# send_timeout: 10s
---

23
plugins/notifications/email/main.go → cmd/notification-email/main.go
View file

@ -4,6 +4,7 @@ import (
"context"
"fmt"
"os"
"time"
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
"github.com/hashicorp/go-hclog"
@ -47,6 +48,8 @@ type PluginConfig struct {
EncryptionType string `yaml:"encryption_type"`
AuthType string `yaml:"auth_type"`
HeloHost string `yaml:"helo_host"`
ConnectTimeout string `yaml:"connect_timeout"`
SendTimeout string `yaml:"send_timeout"`
}
type EmailPlugin struct {
@ -77,7 +80,7 @@ func (n *EmailPlugin) Configure(ctx context.Context, config *protobufs.Config) (
}
if d.ReceiverEmails == nil || len(d.ReceiverEmails) == 0 {
return nil, fmt.Errorf("Receiver emails are not set")
return nil, fmt.Errorf("receiver emails are not set")
}
n.ConfigByName[d.Name] = d
@ -108,6 +111,24 @@ func (n *EmailPlugin) Notify(ctx context.Context, notification *protobufs.Notifi
server.Authentication = AuthStringToType[cfg.AuthType]
server.Helo = cfg.HeloHost
var err error
if cfg.ConnectTimeout != "" {
server.ConnectTimeout, err = time.ParseDuration(cfg.ConnectTimeout)
if err != nil {
logger.Warn(fmt.Sprintf("invalid connect timeout '%s', using default '10s'", cfg.ConnectTimeout))
server.ConnectTimeout = 10 * time.Second
}
}
if cfg.SendTimeout != "" {
server.SendTimeout, err = time.ParseDuration(cfg.SendTimeout)
if err != nil {
logger.Warn(fmt.Sprintf("invalid send timeout '%s', using default '10s'", cfg.SendTimeout))
server.SendTimeout = 10 * time.Second
}
}
logger.Debug("making smtp connection")
smtpClient, err := server.Connect()
if err != nil {

9
plugins/notifications/splunk/Makefile → cmd/notification-http/Makefile
View file

@ -4,14 +4,13 @@ ifeq ($(OS), Windows_NT)
EXT = .exe
endif
PLUGIN=splunk
BINARY_NAME = notification-$(PLUGIN)$(EXT)
GO = go
GOBUILD = $(GO) build
GOCMD = go
GOBUILD = $(GOCMD) build
BINARY_NAME = notification-http$(EXT)
build: clean
$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
.PHONY: clean
clean:

0
plugins/notifications/http/http.yaml → cmd/notification-http/http.yaml
View file

5
plugins/notifications/http/main.go → cmd/notification-http/main.go
View file

@ -58,13 +58,12 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
if err != nil {
return nil, err
}
for headerName, headerValue := range cfg.Headers {
logger.Debug(fmt.Sprintf("adding header %s: %s", headerName, headerValue))
request.Header.Add(headerName, headerValue)
}
logger.Debug(fmt.Sprintf("making HTTP %s call to %s with body %s", cfg.Method, cfg.URL, string(notification.Text)))
resp, err := client.Do(request)
logger.Debug(fmt.Sprintf("making HTTP %s call to %s with body %s", cfg.Method, cfg.URL, notification.Text))
resp, err := client.Do(request.WithContext(ctx))
if err != nil {
logger.Error(fmt.Sprintf("Failed to make HTTP request : %s", err))
return nil, err

9
plugins/notifications/dummy/Makefile → cmd/notification-sentinel/Makefile
View file

@ -4,14 +4,13 @@ ifeq ($(OS), Windows_NT)
EXT = .exe
endif
PLUGIN = dummy
BINARY_NAME = notification-$(PLUGIN)$(EXT)
GO = go
GOBUILD = $(GO) build
GOCMD = go
GOBUILD = $(GOCMD) build
BINARY_NAME = notification-sentinel$(EXT)
build: clean
$(GOBUILD) $(LD_OPTS) $(BUILD_VENDOR_FLAGS) -o $(BINARY_NAME)
$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
.PHONY: clean
clean:

133
cmd/notification-sentinel/main.go Normal file
View file

@ -0,0 +1,133 @@
package main
import (
"context"
"crypto/hmac"
"crypto/sha256"
"encoding/base64"
"fmt"
"net/http"
"os"
"strings"
"time"
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
"github.com/hashicorp/go-hclog"
"github.com/hashicorp/go-plugin"
"gopkg.in/yaml.v3"
)
type PluginConfig struct {
Name string `yaml:"name"`
CustomerID string `yaml:"customer_id"`
SharedKey string `yaml:"shared_key"`
LogType string `yaml:"log_type"`
LogLevel *string `yaml:"log_level"`
}
type SentinelPlugin struct {
PluginConfigByName map[string]PluginConfig
}
var logger hclog.Logger = hclog.New(&hclog.LoggerOptions{
Name: "sentinel-plugin",
Level: hclog.LevelFromString("INFO"),
Output: os.Stderr,
JSONFormat: true,
})
func (s *SentinelPlugin) getAuthorizationHeader(now string, length int, pluginName string) (string, error) {
xHeaders := "x-ms-date:" + now
stringToHash := fmt.Sprintf("POST\n%d\napplication/json\n%s\n/api/logs", length, xHeaders)
decodedKey, _ := base64.StdEncoding.DecodeString(s.PluginConfigByName[pluginName].SharedKey)
h := hmac.New(sha256.New, decodedKey)
h.Write([]byte(stringToHash))
encodedHash := base64.StdEncoding.EncodeToString(h.Sum(nil))
authorization := "SharedKey " + s.PluginConfigByName[pluginName].CustomerID + ":" + encodedHash
logger.Trace("authorization header", "header", authorization)
return authorization, nil
}
func (s *SentinelPlugin) Notify(ctx context.Context, notification *protobufs.Notification) (*protobufs.Empty, error) {
if _, ok := s.PluginConfigByName[notification.Name]; !ok {
return nil, fmt.Errorf("invalid plugin config name %s", notification.Name)
}
cfg := s.PluginConfigByName[notification.Name]
if cfg.LogLevel != nil && *cfg.LogLevel != "" {
logger.SetLevel(hclog.LevelFromString(*cfg.LogLevel))
}
logger.Info("received notification for sentinel config", "name", notification.Name)
url := fmt.Sprintf("https://%s.ods.opinsights.azure.com/api/logs?api-version=2016-04-01", s.PluginConfigByName[notification.Name].CustomerID)
body := strings.NewReader(notification.Text)
//Cannot use time.RFC1123 as azure wants GMT, not UTC
now := time.Now().UTC().Format("Mon, 02 Jan 2006 15:04:05 GMT")
authorization, err := s.getAuthorizationHeader(now, len(notification.Text), notification.Name)
if err != nil {
return &protobufs.Empty{}, err
}
req, err := http.NewRequest(http.MethodPost, url, body)
if err != nil {
logger.Error("failed to create request", "error", err)
return &protobufs.Empty{}, err
}
req.Header.Set("Content-Type", "application/json")
req.Header.Set("Log-Type", s.PluginConfigByName[notification.Name].LogType)
req.Header.Set("Authorization", authorization)
req.Header.Set("x-ms-date", now)
client := &http.Client{}
resp, err := client.Do(req.WithContext(ctx))
if err != nil {
logger.Error("failed to send request", "error", err)
return &protobufs.Empty{}, err
}
defer resp.Body.Close()
logger.Debug("sent notification to sentinel", "status", resp.Status)
if resp.StatusCode != http.StatusOK {
return &protobufs.Empty{}, fmt.Errorf("failed to send notification to sentinel: %s", resp.Status)
}
return &protobufs.Empty{}, nil
}
func (s *SentinelPlugin) Configure(ctx context.Context, config *protobufs.Config) (*protobufs.Empty, error) {
d := PluginConfig{}
err := yaml.Unmarshal(config.Config, &d)
s.PluginConfigByName[d.Name] = d
return &protobufs.Empty{}, err
}
func main() {
var handshake = plugin.HandshakeConfig{
ProtocolVersion: 1,
MagicCookieKey: "CROWDSEC_PLUGIN_KEY",
MagicCookieValue: os.Getenv("CROWDSEC_PLUGIN_KEY"),
}
sp := &SentinelPlugin{PluginConfigByName: make(map[string]PluginConfig)}
plugin.Serve(&plugin.ServeConfig{
HandshakeConfig: handshake,
Plugins: map[string]plugin.Plugin{
"sentinel": &protobufs.NotifierPlugin{
Impl: sp,
},
},
GRPCServer: plugin.DefaultGRPCServer,
Logger: logger,
})
}

21
cmd/notification-sentinel/sentinel.yaml Normal file
View file

@ -0,0 +1,21 @@
type: sentinel # Don't change
name: sentinel_default # Must match the registered plugin in the profile
# One of "trace", "debug", "info", "warn", "error", "off"
log_level: info
# group_wait: # Time to wait collecting alerts before relaying a message to this plugin, eg "30s"
# group_threshold: # Amount of alerts that triggers a message before <group_wait> has expired, eg "10"
# max_retry: # Number of attempts to relay messages to plugins in case of error
# timeout: # Time to wait for response from the plugin before considering the attempt a failure, eg "10s"
#-------------------------
# plugin-specific options
# The following template receives a list of models.Alert objects
# The output goes in the http request body
format: |
{{.|toJson}}
customer_id: XXX-XXX
shared_key: XXXXXXX
log_type: crowdsec

17
cmd/notification-slack/Makefile Normal file
View file

@ -0,0 +1,17 @@
ifeq ($(OS), Windows_NT)
SHELL := pwsh.exe
.SHELLFLAGS := -NoProfile -Command
EXT = .exe
endif
GO = go
GOBUILD = $(GO) build
BINARY_NAME = notification-slack$(EXT)
build: clean
$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
.PHONY: clean
clean:
@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)

3
plugins/notifications/slack/main.go → cmd/notification-slack/main.go
View file

@ -38,10 +38,9 @@ func (n *Notify) Notify(ctx context.Context, notification *protobufs.Notificatio
if cfg.LogLevel != nil && *cfg.LogLevel != "" {
logger.SetLevel(hclog.LevelFromString(*cfg.LogLevel))
}
logger.Info(fmt.Sprintf("found notify signal for %s config", notification.Name))
logger.Debug(fmt.Sprintf("posting to %s webhook, message %s", cfg.Webhook, notification.Text))
err := slack.PostWebhook(n.ConfigByName[notification.Name].Webhook, &slack.WebhookMessage{
err := slack.PostWebhookContext(ctx, n.ConfigByName[notification.Name].Webhook, &slack.WebhookMessage{
Text: notification.Text,
})
if err != nil {

0
plugins/notifications/slack/slack.yaml → cmd/notification-slack/slack.yaml
View file

17
cmd/notification-splunk/Makefile Normal file
View file

@ -0,0 +1,17 @@
ifeq ($(OS), Windows_NT)
SHELL := pwsh.exe
.SHELLFLAGS := -NoProfile -Command
EXT = .exe
endif
GO = go
GOBUILD = $(GO) build
BINARY_NAME = notification-splunk$(EXT)
build: clean
$(GOBUILD) $(LD_OPTS) -o $(BINARY_NAME)
.PHONY: clean
clean:
@$(RM) $(BINARY_NAME) $(WIN_IGNORE_ERR)

6
plugins/notifications/splunk/main.go → cmd/notification-splunk/main.go
View file

@ -58,19 +58,19 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio
return &protobufs.Empty{}, err
}
req, err := http.NewRequest("POST", cfg.URL, strings.NewReader(string(data)))
req, err := http.NewRequest(http.MethodPost, cfg.URL, strings.NewReader(string(data)))
if err != nil {
return &protobufs.Empty{}, err
}
req.Header.Add("Authorization", fmt.Sprintf("Splunk %s", cfg.Token))
logger.Debug(fmt.Sprintf("posting event %s to %s", string(data), req.URL))
resp, err := s.Client.Do(req)
resp, err := s.Client.Do(req.WithContext(ctx))
if err != nil {
return &protobufs.Empty{}, err
}
if resp.StatusCode != 200 {
if resp.StatusCode != http.StatusOK {
content, err := io.ReadAll(resp.Body)
if err != nil {
return &protobufs.Empty{}, fmt.Errorf("got non 200 response and failed to read error %s", err)

0
plugins/notifications/splunk/splunk.yaml → cmd/notification-splunk/splunk.yaml
View file

1
config/config.yaml
View file

@ -6,7 +6,6 @@ common:
log_max_size: 20
compress_logs: true
log_max_files: 10
working_dir: .
config_paths:
config_dir: /etc/crowdsec/
data_dir: /var/lib/crowdsec/data/

1
config/config_win.yaml
View file

@ -3,7 +3,6 @@ common:
log_media: file
log_level: info
log_dir: C:\ProgramData\CrowdSec\log\
working_dir: .
config_paths:
config_dir: C:\ProgramData\CrowdSec\config\
data_dir: C:\ProgramData\CrowdSec\data\

1
config/config_win_no_lapi.yaml
View file

@ -3,7 +3,6 @@ common:
log_media: file
log_level: info
log_dir: C:\ProgramData\CrowdSec\log\
working_dir: .
config_paths:
config_dir: C:\ProgramData\CrowdSec\config\
data_dir: C:\ProgramData\CrowdSec\data\

2
config/crowdsec.service
View file

@ -5,7 +5,7 @@ After=syslog.target network.target remote-fs.target nss-lookup.target
[Service]
Type=notify
Environment=LC_ALL=C LANG=C
ExecStartPre=/usr/local/bin/crowdsec -c /etc/crowdsec/config.yaml -t
ExecStartPre=/usr/local/bin/crowdsec -c /etc/crowdsec/config.yaml -t -error
ExecStart=/usr/local/bin/crowdsec -c /etc/crowdsec/config.yaml
#ExecStartPost=/bin/sleep 0.1
ExecReload=/bin/kill -HUP $MAINPID

1
config/dev.yaml
View file

@ -2,7 +2,6 @@ common:
daemonize: true
log_media: stdout
log_level: info
working_dir: .
config_paths:
config_dir: ./config
data_dir: ./data/

1
config/user.yaml
View file

@ -3,7 +3,6 @@ common:
log_media: stdout
log_level: info
log_dir: /var/log/
working_dir: .
config_paths:
config_dir: /etc/crowdsec/
data_dir: /var/lib/crowdsec/data

2
debian/crowdsec.service vendored
View file

@ -5,7 +5,7 @@ After=syslog.target network.target remote-fs.target nss-lookup.target
[Service]
Type=notify
Environment=LC_ALL=C LANG=C
ExecStartPre=/usr/bin/crowdsec -c /etc/crowdsec/config.yaml -t
ExecStartPre=/usr/bin/crowdsec -c /etc/crowdsec/config.yaml -t -error
ExecStart=/usr/bin/crowdsec -c /etc/crowdsec/config.yaml
#ExecStartPost=/bin/sleep 0.1
ExecReload=/bin/kill -HUP $MAINPID

Some files were not shown because too many files have changed in this diff Show more