beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Fix documentation errors (#496)

Browse source
This commit is contained in:
AlteredCoder 2020-12-01 17:04:13 +01:00 committed by GitHub
parent b7190c9ecc
commit 8707140fb2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
31 changed files with 331 additions and 463 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmd/crowdsec-cli/capi.go
View file

@ -96,7 +96,7 @@ func NewCapiCmd() *cobra.Command {
fmt.Printf("%s\n", string(apiConfigDump))
}
log.Warningf("Run 'systemctl reload crowdsec' for the new configuration to be effective")
log.Warningf("Run 'sudo systemctl reload crowdsec' for the new configuration to be effective")
},
}
cmdCapiRegister.Flags().StringVarP(&outputFile, "file", "f", "", "output file destination")

2
cmd/crowdsec-cli/collections.go
View file

@ -31,7 +31,7 @@ func NewCollectionsCmd() *cobra.Command {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof("Run 'systemctl reload crowdsec' for the new configuration to be effective.")
log.Infof("Run 'sudo systemctl reload crowdsec' for the new configuration to be effective.")
},
}

2
cmd/crowdsec-cli/lapi.go
View file

@ -107,7 +107,7 @@ Keep in mind the machine needs to be validated by an administrator on LAPI side
} else {
fmt.Printf("%s\n", string(apiConfigDump))
}
log.Warningf("Run 'systemctl reload crowdsec' for the new configuration to be effective")
log.Warningf("Run 'sudo systemctl reload crowdsec' for the new configuration to be effective")
},
}
cmdLapiRegister.Flags().StringVarP(&apiURL, "url", "u", "", "URL of the API (ie. http://127.0.0.1)")

2
cmd/crowdsec-cli/parsers.go
View file

@ -35,7 +35,7 @@ cscli parsers remove crowdsecurity/sshd-logs
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof("Run 'systemctl reload crowdsec' for the new configuration to be effective.")
log.Infof("Run 'sudo systemctl reload crowdsec' for the new configuration to be effective.")
},
}

2
cmd/crowdsec-cli/postoverflows.go
View file

@ -34,7 +34,7 @@ func NewPostOverflowsCmd() *cobra.Command {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof("Run 'systemctl reload crowdsec' for the new configuration to be effective.")
log.Infof("Run 'sudo systemctl reload crowdsec' for the new configuration to be effective.")
},
}

2
cmd/crowdsec-cli/scenarios.go
View file

@ -35,7 +35,7 @@ cscli scenarios remove crowdsecurity/ssh-bf
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof("Run 'systemctl reload crowdsec' for the new configuration to be effective.")
log.Infof("Run 'sudo systemctl reload crowdsec' for the new configuration to be effective.")
},
}

2
cmd/crowdsec-cli/simulation.go
View file

@ -112,7 +112,7 @@ cscli simulation disable crowdsecurity/ssh-bf`,
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() != "status" {
log.Infof("Run 'systemctl reload crowdsec' for the new configuration to be effective.")
log.Infof("Run 'sudo systemctl reload crowdsec' for the new configuration to be effective.")
}
},
}

2
config/profiles.yaml
View file

@ -4,5 +4,5 @@ filters:
- Alert.Remediation == true && Alert.GetScope() == "Ip"
decisions:
- type: ban
duration: 1h
duration: 4h
on_success: break

2
docs/v1.X/docs/bouncers/index.md
View file

@ -16,7 +16,7 @@ You can explore [available {{v1X.bouncers.name}} on the hub]({{v1X.hub.bouncers_
To be able for your {{v1X.bouncers.Name}} to communicate with the local API, you have to generate an API token with `cscli` and put it in your {{v1X.bouncers.Name}} configuration file:
```bash
$ cscli bouncers add testBouncer
$ sudo cscli bouncers add testBouncer
Api key for 'testBouncer':
6dcfe93f18675265e905aef390330a35

188
docs/v1.X/docs/getting_started/crowdsec-tour.md
View file

@ -2,12 +2,11 @@
## List installed configurations
```bash
{{v1X.cli.bin}} hub list
sudo {{v1X.cli.bin}} hub list
```
On the machine where you deployed {{v1X.crowdsec.name}}, type `{{v1X.cli.bin}} hub list` to see install configurations.
This list represents the parsers, scenarios and/or collections that you deployed. They represent what your {{v1X.crowdsec.name}} setup can read (logs) and detect (scenarios). `{{v1X.cli.bin}} hub list -a` will list all available configurations in the hub.
On the machine where you deployed {{v1X.crowdsec.name}}, type `sudo {{v1X.cli.bin}} hub list` to see install configurations.
This list represents the parsers, scenarios and/or collections that you deployed. They represent what your {{v1X.crowdsec.name}} setup can read (logs) and detect (scenarios). `sudo {{v1X.cli.bin}} hub list -a` will list all available configurations in the hub.
Check [{{v1X.cli.name}} configuration](/Crowdsec/v1/user_guide/cscli/) management for more !
@ -15,36 +14,41 @@ Check [{{v1X.cli.name}} configuration](/Crowdsec/v1/user_guide/cscli/) managemen
<details>
<summary>output example</summary>
```bash
$ ./cscli -c dev.yaml hub list
INFO[0000] Loaded 13 collecs, 17 parsers, 20 scenarios, 3 post-overflow parsers
INFO[0000] unmanaged items : 7 local, 0 tainted
$ sudo cscli hub list
INFO[0000] Loaded 13 collecs, 17 parsers, 21 scenarios, 3 post-overflow parsers
INFO[0000] unmanaged items : 23 local, 0 tainted
INFO[0000] PARSERS:
----------------------------------------------------------------------------------------------------------------------------------------------------------------
NAME 📦 STATUS VERSION LOCAL PATH
----------------------------------------------------------------------------------------------------------------------------------------------------------------
crowdsecurity/syslog-logs ✔️ enabled 0.1 /.../config/parsers/s00-raw/syslog-logs.yaml
crowdsecurity/dateparse-enrich ✔️ enabled 0.1 /.../config/parsers/s02-enrich/dateparse-enrich.yaml
crowdsecurity/geoip-enrich ✔️ enabled 0.2 /.../config/parsers/s02-enrich/geoip-enrich.yaml
crowdsecurity/sshd-logs ✔️ enabled 0.1 /.../config/parsers/s01-parse/sshd-logs.yaml
----------------------------------------------------------------------------------------------------------------------------------------------------------------
--------------------------------------------------------------------------------------------------------------
NAME 📦 STATUS VERSION LOCAL PATH
--------------------------------------------------------------------------------------------------------------
crowdsecurity/mysql-logs ✔️ enabled 0.1 /etc/crowdsec/parsers/s01-parse/mysql-logs.yaml
crowdsecurity/sshd-logs ✔️ enabled 0.1 /etc/crowdsec/parsers/s01-parse/sshd-logs.yaml
crowdsecurity/dateparse-enrich ✔️ enabled 0.1 /etc/crowdsec/parsers/s02-enrich/dateparse-enrich.yaml
crowdsecurity/whitelists ✔️ enabled 0.1 /etc/crowdsec/parsers/s02-enrich/whitelists.yaml
crowdsecurity/geoip-enrich ✔️ enabled 0.2 /etc/crowdsec/parsers/s02-enrich/geoip-enrich.yaml
crowdsecurity/syslog-logs ✔️ enabled 0.1 /etc/crowdsec/parsers/s00-raw/syslog-logs.yaml
--------------------------------------------------------------------------------------------------------------
INFO[0000] SCENARIOS:
-----------------------------------------------------------------------------------------------------------------------------------
NAME 📦 STATUS VERSION LOCAL PATH
-----------------------------------------------------------------------------------------------------------------------------------
crowdsecurity/ssh-bf ✔️ enabled 0.1 /.../config/scenarios/ssh-bf.yaml
-----------------------------------------------------------------------------------------------------------------------------------
-------------------------------------------------------------------------------------
NAME 📦 STATUS VERSION LOCAL PATH
-------------------------------------------------------------------------------------
crowdsecurity/mysql-bf ✔️ enabled 0.1 /etc/crowdsec/scenarios/mysql-bf.yaml
crowdsecurity/ssh-bf ✔️ enabled 0.1 /etc/crowdsec/scenarios/ssh-bf.yaml
-------------------------------------------------------------------------------------
INFO[0000] COLLECTIONS:
-----------------------------------------------------------------------------------------------------------------------------------
NAME 📦 STATUS VERSION LOCAL PATH
-----------------------------------------------------------------------------------------------------------------------------------
crowdsecurity/sshd ✔️ enabled 0.1 /.../config/collections/sshd.yaml
crowdsecurity/linux ✔️ enabled 0.2 /.../config/collections/linux.yaml
-----------------------------------------------------------------------------------------------------------------------------------
---------------------------------------------------------------------------------
NAME 📦 STATUS VERSION LOCAL PATH
---------------------------------------------------------------------------------
crowdsecurity/mysql ✔️ enabled 0.1 /etc/crowdsec/collections/mysql.yaml
crowdsecurity/sshd ✔️ enabled 0.1 /etc/crowdsec/collections/sshd.yaml
crowdsecurity/linux ✔️ enabled 0.2 /etc/crowdsec/collections/linux.yaml
---------------------------------------------------------------------------------
INFO[0000] POSTOVERFLOWS:
--------------------------------------
NAME 📦 STATUS VERSION LOCAL PATH
--------------------------------------
--------------------------------------
```
</details>
@ -52,7 +56,7 @@ INFO[0000] POSTOVERFLOWS:
```bash
{{v1X.cli.bin}} decisions list
sudo {{v1X.cli.bin}} decisions list
```
If you just deployed {{v1X.crowdsec.name}}, the list might be empty, but don't worry, it simply means you haven't yet been attacked, congrats!
@ -63,28 +67,29 @@ Check [{{v1X.cli.name}} decisions](/Crowdsec/v1/user_guide/decision_management/)
<details>
<summary>output example</summary>
```bash
$ cscli decisions list
+----+----------+-------------+----------------------+--------+---------+----+--------+------------------+
| ID | SOURCE | SCOPE:VALUE | REASON | ACTION | COUNTRY | AS | EVENTS | EXPIRATION |
+----+----------+-------------+----------------------+--------+---------+----+--------+------------------+
| 1 | crowdsec | Ip:1.2.3.6 | crowdsecurity/ssh-bf | ban | US | | 6 | 59m48.467053872s |
| 2 | cscli | Ip:1.2.3.4 | | ban | | | 1 | 3h59m57.671401352s |
+----+----------+-------------+----------------------+--------+---------+----+--------+--------------------+
$ sudo cscli decisions list
+-----+-----------+-------------+------------------------------------+--------+---------+----+--------+--------------------+----------+
| ID | SOURCE | SCOPE:VALUE | REASON | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | ALERT ID |
+-----+-----------+-------------+------------------------------------+--------+---------+----+--------+--------------------+----------+
| 802 | cscli | Ip:1.2.3.5 | manual 'ban' from | ban | | | 1 | 3h50m58.10039043s | 802 |
| | | | 'b76cc7b1bbdc489e93909d2043031de8' | | | | | | |
| 801 | crowdsec | Ip:1.2.3.4 | crowdsecurity/ssh-bf | ban | | | 6 | 3h59m45.100387557s | 801 |
+-----+-----------+-------------+------------------------------------+--------+---------+----+--------+--------------------+----------+
```
</details>
There are different bans sources:
There are different decisions `SOURCE`:
- crowdsec : bans triggered locally
- api : bans fetched from the API as part of the global consensus
- csli : bans added via `{{v1X.cli.bin}} decisions add`
- crowdsec : decisions triggered locally by the crowdsec agent
- CAPI : decisions fetched from the Crowdsec Central API
- csli : decisions added via `sudo {{v1X.cli.bin}} decisions add`
## List alerts
```bash
{{v1X.cli.bin}} alerts list
sudo {{v1X.cli.bin}} alerts list
```
While decisions won't be shown anymore once they expire (or are manually deleted), the alerts will stay visible, allowing you to keep track of past decisions.
@ -93,13 +98,12 @@ You will here see the alerts, even if the associated decisions expired.
<details>
<summary>output example</summary>
```bash
$ cscli alerts list --since 1h
$ sudo cscli alerts list --since 1h
+----+-------------+----------------------------+---------+----+-----------+---------------------------+
| ID | SCOPE:VALUE | REASON | COUNTRY | AS | DECISIONS | CREATED AT |
+----+-------------+----------------------------+---------+----+-----------+---------------------------+
| 5 | Ip:1.2.3.6 | crowdsecurity/ssh-bf (0.1) | US | | ban:1 | 2020-10-29T11:33:36+01:00 |
+----+-------------+----------------------------+---------+----+-----------+---------------------------+
```
</details>
@ -107,7 +111,7 @@ $ cscli alerts list --since 1h
## Monitor on-going activity (prometheus)
```bash
{{v1X.cli.bin}} metrics
sudo {{v1X.cli.bin}} metrics
```
The metrics displayed are extracted from {{v1X.crowdsec.name}} prometheus.
@ -122,40 +126,66 @@ The indicators are grouped by scope :
<summary>output example</summary>
```bash
$ {{v1X.cli.bin}} metrics
INFO[0000] Buckets Metrics:
+--------------------------------+---------------+-----------+--------------+--------+---------+
| BUCKET | CURRENT COUNT | OVERFLOWS | INSTANCIATED | POURED | EXPIRED |
+--------------------------------+---------------+-----------+--------------+--------+---------+
| crowdsecurity/ssh-bf | 1 | 1 | 2 | 10 | - |
| crowdsecurity/ssh-bf_user-enum | 1 | - | 1 | 1 | - |
+--------------------------------+---------------+-----------+--------------+--------+---------+
INFO[0000] Acquisition Metrics:
+-------------------+------------+--------------+----------------+------------------------+
| SOURCE | LINES READ | LINES PARSED | LINES UNPARSED | LINES POURED TO BUCKET |
+-------------------+------------+--------------+----------------+------------------------+
| /tmp/test.log | 10 | 10 | - | 11 |
| /var/log/auth.log | 2 | - | 2 | - |
| /var/log/syslog | 4 | - | 4 | - |
+-------------------+------------+--------------+----------------+------------------------+
INFO[0000] Parser Metrics:
+--------------------------------+------+--------+----------+
| PARSERS | HITS | PARSED | UNPARSED |
+--------------------------------+------+--------+----------+
| child-crowdsecurity/sshd-logs | 10 | 10 | - |
| crowdsecurity/dateparse-enrich | 10 | 10 | - |
| crowdsecurity/geoip-enrich | 10 | 10 | - |
| crowdsecurity/sshd-logs | 10 | 10 | - |
| crowdsecurity/syslog-logs | 16 | 16 | - |
+--------------------------------+------+--------+----------+
INFO[0000] Local Api Metrics:
+--------------------+--------+------+
| ROUTE | METHOD | HITS |
+--------------------+--------+------+
| /v1/alerts | GET | 2 |
| /v1/alerts | POST | 2 |
| /v1/watchers/login | POST | 4 |
+--------------------+--------+------+
$ sudo {{v1X.cli.bin}} metrics
INFO[0000] Buckets Metrics:
+--------------------------------------+---------------+-----------+--------------+--------+---------+
| BUCKET | CURRENT COUNT | OVERFLOWS | INSTANCIATED | POURED | EXPIRED |
+--------------------------------------+---------------+-----------+--------------+--------+---------+
| crowdsecurity/http-bad-user-agent | - | - | 7 | 7 | 7 |
| crowdsecurity/http-crawl-non_statics | - | - | 82 | 107 | 82 |
| crowdsecurity/http-probing | - | - | 2 | 2 | 2 |
| crowdsecurity/http-sensitive-files | - | - | 1 | 1 | 1 |
| crowdsecurity/ssh-bf | 16 | 5562 | 7788 | 41542 | 2210 |
| crowdsecurity/ssh-bf_user-enum | 8 | - | 6679 | 12571 | 6671 |
+--------------------------------------+---------------+-----------+--------------+--------+---------+
INFO[0000] Acquisition Metrics:
+---------------------------+------------+--------------+----------------+------------------------+
| SOURCE | LINES READ | LINES PARSED | LINES UNPARSED | LINES POURED TO BUCKET |
+---------------------------+------------+--------------+----------------+------------------------+
| /var/log/auth.log | 92978 | 41542 | 51436 | 54113 |
| /var/log/messages | 2 | - | 2 | - |
| /var/log/nginx/access.log | 124 | 99 | 25 | 88 |
| /var/log/nginx/error.log | 287 | 63 | 224 | 29 |
| /var/log/syslog | 27271 | - | 27271 | - |
+---------------------------+------------+--------------+----------------+------------------------+
INFO[0000] Parser Metrics:
+--------------------------------+--------+--------+----------+
| PARSERS | HITS | PARSED | UNPARSED |
+--------------------------------+--------+--------+----------+
| child-crowdsecurity/http-logs | 486 | 232 | 254 |
| child-crowdsecurity/nginx-logs | 723 | 162 | 561 |
| child-crowdsecurity/sshd-logs | 381792 | 41542 | 340250 |
| crowdsecurity/dateparse-enrich | 41704 | 41704 | - |
| crowdsecurity/geoip-enrich | 41641 | 41641 | - |
| crowdsecurity/http-logs | 162 | 59 | 103 |
| crowdsecurity/nginx-logs | 411 | 162 | 249 |
| crowdsecurity/non-syslog | 411 | 411 | - |
| crowdsecurity/sshd-logs | 92126 | 41542 | 50584 |
| crowdsecurity/syslog-logs | 120251 | 120249 | 2 |
| crowdsecurity/whitelists | 41704 | 41704 | - |
+--------------------------------+--------+--------+----------+
INFO[0000] Local Api Metrics:
+----------------------+--------+------+
| ROUTE | METHOD | HITS |
+----------------------+--------+------+
| /v1/alerts | GET | 3 |
| /v1/alerts | POST | 4673 |
| /v1/decisions/stream | GET | 6498 |
| /v1/watchers/login | POST | 23 |
+----------------------+--------+------+
INFO[0000] Local Api Machines Metrics:
+----------------------------------+------------+--------+------+
| MACHINE | ROUTE | METHOD | HITS |
+----------------------------------+------------+--------+------+
| 7f0607a3469243139699bf2f30321fc4 | /v1/alerts | POST | 4673 |
| 7f0607a3469243139699bf2f30321fc4 | /v1/alerts | GET | 3 |
+----------------------------------+------------+--------+------+
INFO[0000] Local Api Bouncers Metrics:
+------------------------------+----------------------+--------+------+
| BOUNCER | ROUTE | METHOD | HITS |
+------------------------------+----------------------+--------+------+
| cs-firewall-bouncer-n3W19Qua | /v1/decisions/stream | GET | 6498 |
+------------------------------+----------------------+--------+------+
```
</details>
@ -163,7 +193,7 @@ INFO[0000] Local Api Metrics:
## Deploy dashboard
```bash
cscli dashboard setup --listen 0.0.0.0
sudo cscli dashboard setup --listen 0.0.0.0
```
A docker metabase {{v1X.metabase.Htmlname}} container can be deployed with `cscli dashboard`.
@ -172,7 +202,7 @@ It requires docker, [installation instructions are available here](https://docs.
## Logs
```bash
tail -f /var/log/crowdsec.log
sudo tail -f /var/log/crowdsec.log
```
- `/var/log/crowdsec.log` is the main log, it shows ongoing decisions and acquisition/parsing/scenario errors.
@ -181,7 +211,7 @@ tail -f /var/log/crowdsec.log
## Installing collections
```bash
cscli collections install crowdsecurity/nginx
sudo cscli collections install crowdsecurity/nginx
```
Collections are bundles of parsers/scenarios that form a coherent ensemble to analyze/detect attacks for a specific service. It is the most common way to deploy configurations.

2
docs/v1.X/docs/getting_started/installation.md
View file

@ -78,4 +78,4 @@ make release
This will create you a directory (`crowdsec-vXXX/`) and an archive (`crowdsec-release.tgz`) that are release built from your local code source.
Now, you can install either with [interactive wizard](#using-the-interactive-wizard) or the [unattended mode](#using-unattended-mode).
Now, you can install either with [interactive wizard](#using-the-interactive-wizard) or the [unattended mode](#using-unattended-mode).

19
docs/v1.X/docs/localAPI/index.md
View file

@ -7,7 +7,7 @@ The Local API (LAPI) is a core component of {{v1X.crowdsec.name}} and has a few
- Allow `cscli` to view add or delete decisions
[You can find the swagger documentation here](https://crowdsecurity.github.io/api_doc/index.html?urls.primaryName=LAPI)
You can find the swagger documentation [here](https://crowdsecurity.github.io/api_doc/index.html?urls.primaryName=LAPI).
## Authentication
@ -23,7 +23,7 @@ There is two kinds of authentication to the local API :
To register a bouncer to your API, you need to run the following command on the server where the API is installed:
```bash
$ cscli bouncers add testBouncer
$ sudo cscli bouncers add testBouncer
```
and keep the generated API token to use it in your {{v1X.bouncers.Name}} configuration file.
@ -37,7 +37,7 @@ There is two ways to register a crowdsec to a local API.
* You can create a machine directly on the API server that will be automatically validated, by running the following command on the server where the API is installed:
```bash
$ cscli machines add testMachine
$ sudo cscli machines add testMachine
```
If your crowdsec run on the same server that the local API, then your credentials file will be generated automatically, else you will have to copy/paste them in your remote crowdsec credentials file (`/etc/crowdsec/local_api_credentials.yaml`)
@ -45,13 +45,13 @@ If your crowdsec run on the same server that the local API, then your credential
* You can use `cscli` to register to the API server:
```
cscli lapi register -u <api_url>
sudo cscli lapi register -u <api_url>
```
And validate it with `cscli` on the server where the API is installed:
```
cscli machines validate <machineName>
sudo cscli machines validate <machineName>
```
!!! tips
@ -68,13 +68,18 @@ By default, `crowdsec` and `cscli` use `127.0.0.1:8080` as a default local API.
* On the remote crowdsec server, run:
```
$ cscli lapi register -u http://<remote_api>:<port>
$ sudo cscli lapi register -u http://<remote_api>:<port>
```
* On the local API server, validate the machine by running the command:
```bash
$ sudo cscli machines list # to get the name of the new registered machine
```
$ cscli machines validate <machineName>
```
$ sudo cscli machines validate <machineName>
```

89
docs/v1.X/docs/observability/command_line.md
View file

@ -1,5 +1,5 @@
```bash
{{v1X.cli.name}} metrics
sudo {{v1X.cli.name}} metrics
```
This command provides an overview of {{v1X.crowdsec.name}} statistics provided by [prometheus client](/Crowdsec/v1/observability/prometheus/). By default it assumes that the {{v1X.crowdsec.name}} is installed on the same machine.
@ -22,40 +22,67 @@ The metrics are split in 3 main sections :
<details>
<summary>{{v1X.cli.name}} metrics example</summary>
```bash
INFO[0000] Buckets Metrics:
+-----------------------------------------+-----------+--------------+--------+---------+
| BUCKET | OVERFLOWS | INSTANTIATED | POURED | EXPIRED |
+-----------------------------------------+-----------+--------------+--------+---------+
| crowdsecurity/http-scan-uniques_404 | - | 8 | 9 | 8 |
| crowdsecurity/iptables-scan-multi_ports | 1 | 8306 | 9097 | 8288 |
| crowdsecurity/ssh-bf | 42 | 281 | 1434 | 238 |
| crowdsecurity/ssh-bf_user-enum | 13 | 659 | 777 | 646 |
| crowdsecurity/http-crawl-non_statics | - | 10 | 12 | 10 |
+-----------------------------------------+-----------+--------------+--------+---------+
INFO[0000] Acquisition Metrics:
+------------------------------------------+------------+--------------+----------------+------------------------+
| SOURCE | LINES READ | LINES PARSED | LINES UNPARSED | LINES POURED TO BUCKET |
+------------------------------------------+------------+--------------+----------------+------------------------+
| /var/log/nginx/https.access.log | 25 | 25 | - | 7 |
| /var/log/kern.log | 18078 | 18078 | - | 4066 |
| /var/log/syslog | 18499 | 18078 | 421 | 5031 |
| /var/log/auth.log | 6086 | 1434 | 4652 | 2211 |
| /var/log/nginx/error.log | 170243 | 169632 | 611 | - |
| /var/log/nginx/http.access.log | 44 | 44 | - | 14 |
+------------------------------------------+------------+--------------+----------------+------------------------+
INFO[0000] Parser Metrics:
$ sudo cscli metrics
INFO[0000] Buckets Metrics:
+--------------------------------------+---------------+-----------+--------------+--------+---------+
| BUCKET | CURRENT COUNT | OVERFLOWS | INSTANCIATED | POURED | EXPIRED |
+--------------------------------------+---------------+-----------+--------------+--------+---------+
| crowdsecurity/http-bad-user-agent | - | - | 10 | 10 | 10 |
| crowdsecurity/http-crawl-non_statics | - | - | 91 | 119 | 91 |
| crowdsecurity/http-probing | - | - | 2 | 2 | 2 |
| crowdsecurity/http-sensitive-files | - | - | 1 | 1 | 1 |
| crowdsecurity/ssh-bf | 13 | 6314 | 8768 | 46772 | 2441 |
| crowdsecurity/ssh-bf_user-enum | 6 | - | 7646 | 14406 | 7640 |
+--------------------------------------+---------------+-----------+--------------+--------+---------+
INFO[0000] Acquisition Metrics:
+---------------------------+------------+--------------+----------------+------------------------+
| SOURCE | LINES READ | LINES PARSED | LINES UNPARSED | LINES POURED TO BUCKET |
+---------------------------+------------+--------------+----------------+------------------------+
| /var/log/auth.log | 105476 | 46772 | 58704 | 61178 |
| /var/log/messages | 2 | - | 2 | - |
| /var/log/nginx/access.log | 138 | 111 | 27 | 100 |
| /var/log/nginx/error.log | 312 | 68 | 244 | 32 |
| /var/log/syslog | 31919 | - | 31919 | - |
+---------------------------+------------+--------------+----------------+------------------------+
INFO[0000] Parser Metrics:
+--------------------------------+--------+--------+----------+
| PARSERS | HITS | PARSED | UNPARSED |
+--------------------------------+--------+--------+----------+
| crowdsecurity/geoip-enrich | 37659 | 37659 | 0 |
| crowdsecurity/http-logs | 169701 | 27 | 169674 |
| crowdsecurity/iptables-logs | 36156 | 36156 | 0 |
| crowdsecurity/nginx-logs | 170316 | 169701 | 615 |
| crowdsecurity/non-syslog | 170312 | 170312 | 0 |
| crowdsecurity/sshd-logs | 6053 | 1434 | 4619 |
| crowdsecurity/syslog-logs | 42663 | 42663 | 0 |
| crowdsecurity/dateparse-enrich | 207291 | 207291 | 0 |
| child-crowdsecurity/http-logs | 537 | 257 | 280 |
| child-crowdsecurity/nginx-logs | 789 | 179 | 610 |
| child-crowdsecurity/sshd-logs | 436048 | 46772 | 389276 |
| crowdsecurity/dateparse-enrich | 46951 | 46951 | - |
| crowdsecurity/geoip-enrich | 46883 | 46883 | - |
| crowdsecurity/http-logs | 179 | 66 | 113 |
| crowdsecurity/nginx-logs | 450 | 179 | 271 |
| crowdsecurity/non-syslog | 450 | 450 | - |
| crowdsecurity/sshd-logs | 104386 | 46772 | 57614 |
| crowdsecurity/syslog-logs | 137397 | 137395 | 2 |
| crowdsecurity/whitelists | 46951 | 46951 | - |
+--------------------------------+--------+--------+----------+
INFO[0000] Local Api Metrics:
+----------------------+--------+------+
| ROUTE | METHOD | HITS |
+----------------------+--------+------+
| /v1/alerts | GET | 4 |
| /v1/alerts | POST | 5400 |
| /v1/decisions/stream | GET | 7694 |
| /v1/watchers/login | POST | 27 |
+----------------------+--------+------+
INFO[0000] Local Api Machines Metrics:
+----------------------------------+------------+--------+------+
| MACHINE | ROUTE | METHOD | HITS |
+----------------------------------+------------+--------+------+
| 7f0607a3469243139699bf2f30321fc4 | /v1/alerts | GET | 4 |
| 7f0607a3469243139699bf2f30321fc4 | /v1/alerts | POST | 5400 |
+----------------------------------+------------+--------+------+
INFO[0000] Local Api Bouncers Metrics:
+------------------------------+----------------------+--------+------+
| BOUNCER | ROUTE | METHOD | HITS |
+------------------------------+----------------------+--------+------+
| cs-firewall-bouncer-n3W19Qua | /v1/decisions/stream | GET | 7694 |
+------------------------------+----------------------+--------+------+
```
</details>

10
docs/v1.X/docs/observability/dashboard.md
View file

@ -11,7 +11,7 @@ The {{v1X.cli.name}} command `{{v1X.cli.bin}} dashboard setup` will use [docker]
> Setup and Start crowdsec metabase dashboard
```bash
{{v1X.cli.bin}} dashboard setup
sudo {{v1X.cli.bin}} dashboard setup
```
Optional arguments:
@ -51,14 +51,14 @@ Now you can connect to your dashboard, sign-in with your saved credentials then
Dashboard docker image can be managed by {{v1X.cli.name}} and docker cli also. Look at the {{v1X.cli.name}} help command using
```bash
{{v1X.cli.bin}} dashboard -h
sudo {{v1X.cli.bin}} dashboard -h
```
## Remove the dashboard
> Remove crowdsec metabase dashboard
```bash
{{v1X.cli.bin}} dashboard remove [-f]
sudo {{v1X.cli.bin}} dashboard remove [-f]
```
Optional arguments:
@ -68,13 +68,13 @@ Optional arguments:
> Stop crowdsec metabase dashboard
```bash
{{v1X.cli.bin}} dashboard stop
sudo {{v1X.cli.bin}} dashboard stop
```
## Start the dashboard
> Start crowdsec metabase dashboard
```bash
{{v1X.cli.bin}} dashboard start
sudo {{v1X.cli.bin}} dashboard start
```

2
docs/v1.X/docs/references/enrichers.md
View file

@ -17,7 +17,7 @@ It exposes three methods : `GeoIpCity` `GeoIpASN` and `IpToRange` that are used
Enrichers can be installed as any other parsers with the following command:
```
{{v1X.cli.bin}} install parser crowdsecurity/geoip-enrich
sudo {{v1X.cli.bin}} parsers install crowdsecurity/geoip-enrich
```
Take a tour at the {{v1X.hub.htmlname}} to find them !

7
docs/v1.X/docs/references/events.md
View file

@ -1,6 +1,11 @@
# Events
An `Event` is the runtime representation of an item being processed by crowdsec : It be a Log line being parsed, or an Overflow being reprocessed.
An `Event` is the runtime representation of an item being processed by crowdsec, it can be:
- a log line being parsed
- an overflow being reprocessed
The `Event` object is modified by parsers, scenarios, and directly via user [statics expressions](/Crowdsec/v1/references/parsers/#statics) (for example).

12
docs/v1.X/docs/references/expressions.md
View file

@ -23,39 +23,39 @@ If the `debug` is enabled (in the scenario or parser where expr is used), additi
In order to makes its use in {{v1X.crowdsec.name}} more efficient, we added a few helpers that are documented bellow.
## Atof(string) float64
## `Atof(string) float64`
Parses a string representation of a float number to an actual float number (binding on `strconv.ParseFloat`)
> Atof(evt.Parsed.tcp_port)
## JsonExtract(JsonBlob, FieldName) string
## `JsonExtract(JsonBlob, FieldName) string`
Extract the `FieldName` from the `JsonBlob` and returns it as a string. (binding on [jsonparser](https://github.com/buger/jsonparser/))
> JsonExtract(evt.Parsed.some_json_blob, "foo.bar[0].one_item")
## File(FileName) []string
## `File(FileName) []string`
Returns the content of `FileName` as an array of string, while providing cache mechanism.
> evt.Parsed.some_field in File('some_patterns.txt')
> any(File('rdns_seo_bots.txt'), { evt.Enriched.reverse_dns endsWith #})
## RegexpInFile(StringToMatch, FileName) bool
## `RegexpInFile(StringToMatch, FileName) bool`
Returns `true` if the `StringToMatch` is matched by one of the expressions contained in `FileName` (uses RE2 regexp engine).
> RegexpInFile( evt.Enriched.reverse_dns, 'my_legit_seo_whitelists.txt')
## Upper(string) string
## `Upper(string) string`
Returns the uppercase version of the string
> Upper("yop")
## IpInRange(IPStr, RangeStr) bool
## `IpInRange(IPStr, RangeStr) bool`
Returns true if the IP `IPStr` is contained in the IP range `RangeStr` (uses `net.ParseCIDR`)

178
docs/v1.X/docs/references/plugins_api.md
View file

@ -1,178 +0,0 @@
## Foreword
Output plugins handle Signal Occurences resulting from bucket overflows.
This allows to either make a simple notification/alerting plugin or fully manage a backend (this is what {{v1X.crowdsec.name}} uses to manage SQLite and MySQL).
You can create your own plugins to perform specific actions when a scenario is triggered.
The plugin itself will be compiled into a `.so` and will have its dedicated configuration.
## Interface
Plugins are created in golang and must conform to the following interface :
```go
type Backend interface {
Insert(types.SignalOccurence) error
ReadAT(time.Time) ([]map[string]string, error)
Delete(string) (int, error)
Init(map[string]string) error
Flush() error
Shutdown() error
DeleteAll() error
StartAutoCommit() error
}
```
> Startup/shutdown methods
- `Init` : called at startup time and receives the custom configuration as a string map. Errors aren't fatal, but plugin will be discarded.
- `Shutdown` : called when {{v1X.crowdsec.Name}} is shutting down or restarting
> Writing/Deleting events
- `Insert` : called every time an overflow happens, receives the `SignalOccurence` as a single parameter. Returned errors are non-fatal and will be logged in warning level.
- `Delete` : called to delete existing bans. Receives the exact `ip_text` (ban target) to delete. Only used by `cscli ban del`, only relevant for read/write plugins such as database ones.
- `DeleteAll` : called to delete *all* existing bans. Only used by `cscli ban flush`, only relevant for read/write plugins such as database ones)
> Reading events
- `ReadAT` : returns the list of bans that where active at the given time. The following keys are relevant in the list returned : source, iptext, reason, bancount, action, cn, as, events_count, until. Only used by `cscli ban list`, only relevant for read/write plugins such as database ones)
> Backend
- `Flush` is called regulary by crowdsec for each plugin that received events. For example it will be called after each write in `cscli` (as it's one-shot) and every few hundreds of ms / few events in {{v1X.crowdsec.name}} itself. It might be a good place to deal with slower write operations.
## Configurations
Each plugin has its own configuration file :
```bash
$ cat config/plugins/backend/dummy.yaml
# name of the plugin, is used by profiles.yaml
name: dummy
# path to the .so
path: ./plugins/backend/dummy.so
# your plugin specific configuration
config:
some_parameter: some value
other_parameter: more data
token: fooobarjajajajaja
```
## Dummy plugin
```go
package main
import (
"time"
"github.com/crowdsecurity/crowdsec/pkg/types"
log "github.com/sirupsen/logrus"
)
//This is where you would hold your plugin-specific context
type pluginDummy struct {
//some persistent data
}
func (p *pluginDummy) Shutdown() error {
return nil
}
func (p *pluginDummy) StartAutoCommit() error {
return nil
}
func (p *pluginDummy) Init(config map[string]string) error {
log.Infof("pluginDummy config : %+v ", config)
return nil
}
func (p *pluginDummy) Delete(target string) (int, error) {
return 0, nil
}
func (p *pluginDummy) DeleteAll() error {
return nil
}
func (p *pluginDummy) Insert(sig types.SignalOccurence) error {
log.Infof("insert signal : %+v", sig)
return nil
}
func (p *pluginDummy) Flush() error {
return nil
}
func (p *pluginDummy) ReadAT(timeAT time.Time) ([]map[string]string, error) {
return nil, nil
}
// New is used by the plugin system to get the context
func New() interface{} {
return &pluginDummy
{}
}
// empty main function is mandatory since we are in a main package
func main() {}
```
## Building plugin
```bash
$ go build -buildmode=plugin -o dummy.so
```
## Testing plugin
<details open>
<summary>Get a test env from fresh crowdsec release</summary>
```bash
$ cd crowdsec-v0.3.0
$ ./test_env.sh
$ cd tests
```
</details>
```bash
$ cp ../../plugins/backend/dummy/dummy.so ./plugins/backend/
$ cat > config/plugins/backend/dummy.yaml
name: dummy
path: ./plugins/backend/dummy.so
config:
some_parameter: some value
other_parameter: more data
token: fooobarjajajajaja
$ ./crowdsec -c dev.yaml -file test.log -type mylog
...
INFO[06-08-2020 17:21:30] pluginDummy config : map[flush:false max_records:10000 max_records_age:720h other_parameter:more data some_parameter:some value token:fooobarjajajajaja]
...
INFO[06-08-2020 17:21:30] Starting processing routines
...
INFO[06-08-2020 17:21:30] Processing Overflow ...
INFO[06-08-2020 17:21:30] insert signal : {Model:{ID:0 CreatedAt:0001-01-01 00:00:00 +0000 UTC UpdatedAt:0001-01-01 00:00:00 +0000 UTC DeletedAt:<nil>} MapKey:97872dfae02c523577eff8ec8e19706eec5fa21e Scenario:trigger on stuff Bucket_id:summer-field Alert_message:0.0.0.0 performed 'trigger on stuff' (1 events over 59ns) at 2020-08-06 17:21:30.491000439 +0200 CEST m=+0.722674306 Events_count:1 Events_sequence:[{Model:{ID:0 CreatedAt:0001-01-01 00:00:00 +0000 UTC UpdatedAt:0001-01-01 00:00:00 +0000 UTC DeletedAt:<nil>} Time:2020-08-06 17:21:30.491000368 +0200 CEST m=+0.722674247 Source:{Model:{ID:0 CreatedAt:0001-01-01 00:00:00 +0000 UTC UpdatedAt:0001-01-01 00:00:00 +0000 UTC DeletedAt:<nil>} Ip:0.0.0.0 Range:{IP:<nil> Mask:<nil>} AutonomousSystemNumber:0 AutonomousSystemOrganization: Country: Latitude:0 Longitude:0 Flags:map[]} Source_ip:0.0.0.0 Source_range: Source_AutonomousSystemNumber:0 Source_AutonomousSystemOrganization: Source_Country: SignalOccurenceID:0 Serialized:{"ASNNumber":"0","IsInEU":"false","command":"...","cwd":"...":"...","orig_uid":"...","orig_user":"...","parent":"bash","service":"...","source_ip":"...","user":"..."}}] Start_at:2020-08-06 17:21:30.491000368 +0200 CEST m=+0.722674247 BanApplications:[] Stop_at:2020-08-06 17:21:30.491000439 +0200 CEST m=+0.722674306 Source:0xc000248410 Source_ip:0.0.0.0 Source_range:<nil> Source_AutonomousSystemNumber:0 Source_AutonomousSystemOrganization: Source_Country: Source_Latitude:0 Source_Longitude:0 Sources:map[0.0.0.0:{Model:{ID:0 CreatedAt:0001-01-01 00:00:00 +0000 UTC UpdatedAt:0001-01-01 00:00:00 +0000 UTC DeletedAt:<nil>} Ip:0.0.0.0 Range:{IP:<nil> Mask:<nil>} AutonomousSystemNumber:0 AutonomousSystemOrganization: Country: Latitude:0 Longitude:0 Flags:map[]}] Dest_ip: Capacity:0 Leak_speed:0s Whitelisted:false Simulation:false Reprocess:false Labels:map[type:foobar]}
...
```
## Notes
- All the calls to the plugin methods are blocking. If you need to perform long running operations, it's the plugin's task to handle the background processing with [tombs](https://godoc.org/gopkg.in/tomb.v2) or such.
- Due to [a golang limitation](https://github.com/golang/go/issues/31354) you might have to build crowdsec in the same environment as the plugins.

25
docs/v1.X/docs/references/profiles.md
View file

@ -5,30 +5,19 @@ The profiles configuration (`/etc/crowdsec/profiles.yaml`) allow to configure wh
The configuration file is a yaml file that looks like :
```yaml
name: enforce_mfa
#debug: true
filters:
- 'Alert.Remediation == true && Alert.GetScenario() == "crowdsecurity/ssh-enforce-mfa" && Alert.GetScope() == "username"'
decisions: #remediation vs decision
- type: enforce_mfa
scope: "username"
duration: 1h
on_success: continue
---
name: default_ip_remediation
#debug: true
filters:
# try types.Ip here :)
- Alert.Remediation == true && Alert.GetScope() == "Ip"
decisions:
- type: ban
duration: 1h
duration: 4h
on_success: break
```
Each YAML object in the file contains a list of `models.Decision` that contains :
## Name
## `name`
```yaml
name: foobar
@ -36,7 +25,7 @@ name: foobar
A label for the profile (used in logging)
## Debug
## `debug`
```yaml
debug: true
@ -44,7 +33,7 @@ debug: true
A boolean flag that provides contextual debug.
## Filters
## `filters`
```yaml
filters:
@ -54,7 +43,7 @@ filters:
If any `filter` of the list returns `true`, the profile is elligible and the `decisions` will be applied.
## Decisions
## `decisions`
```yaml
decisions:
@ -74,7 +63,7 @@ It is a list of `models.Decision` objects. The following fields, when present, a
- `type` : defines the type of the remediation that will be applied by available {{v1X.bouncers.htmlname}}, for example `ban`, `captcha`
- `value` : define a hardcoded value for the decision (ie. `1.2.3.4`)
## on_success
## `on_success`
```yaml
on_success: break
@ -82,7 +71,7 @@ on_success: break
If the profile applies and `on_success` is set to `break`, decisions processing will stop here and it won't evaluate against following profiles.
## on_failure
## `on_failure`
```yaml
on_failure: break

2
docs/v1.X/docs/references/scenarios.md
View file

@ -405,7 +405,7 @@ format: 2.0
Running `cscli version` will show you such compatibility matrix :
```bash
$ cscli version
$ sudo cscli version
2020/11/05 09:35:05 version: v0.3.6-183e34c966c475e0d2cdb3c60d0b7426499aa573
2020/11/05 09:35:05 Codename: beta
2020/11/05 09:35:05 BuildDate: 2020-11-04_17:56:46

22
docs/v1.X/docs/user_guide/bouncer_machine_management.md
View file

@ -18,20 +18,20 @@ There are two kind of access to the local api :
The `cscli bouncers` command interacts directly with the database (bouncers add and delete are not implemented in API), and thus it must have correct database configuration.
```bash
$ cscli bouncers list
$ sudo cscli bouncers list
```
You can view the registered bouncers with `list`, as well as add or delete them :
```bash
$ cscli bouncers add mybouncersname
$ sudo cscli bouncers add mybouncersname
Api key for 'mybouncersname':
23........b5a0c
Please keep this key since will not be able to retrive it!
$ cscli bouncers delete mybouncersname
$ sudo cscli bouncers delete mybouncersname
```
The API KEY must be kept and given to the {{v1X.bouncers.htmlname}}.
@ -80,10 +80,10 @@ $ cscli machines list
You can view the registered machines with `list`, as well as add or delete them :
```bash
$ cscli machines add -m mytestmachine -a
$ sudo cscli machines add mytestmachine -a
INFO[0004] Machine 'mytestmachine' created successfully
INFO[0004] API credentials dumped to '/etc/crowdsec/local_api_credentials.yaml'
$ cscli machines delete -m 82929df7ee394b73b81252fe3b4e5020
$ sudo cscli machines delete 82929df7ee394b73b81252fe3b4e5020
```
@ -91,13 +91,13 @@ $ cscli machines delete -m 82929df7ee394b73b81252fe3b4e5020
<summary>cscli machines example</summary>
```bash
$ cscli machines list
$ sudo cscli machines list
----------------------------------------------------------------------------------------------------------------------------------
NAME IP ADDRESS LAST UPDATE STATUS VERSION
----------------------------------------------------------------------------------------------------------------------------------
82929df7ee394b73b81252fe3b4e5020 127.0.0.1 2020-10-31T14:06:32+01:00 ✔️ v0.3.6-3d6ce33908409f2a830af6551a7f5e37f2a4728f
----------------------------------------------------------------------------------------------------------------------------------
$ cscli machines add -m mytestmachine -a
$ sudo cscli machines add -m mytestmachine -a
INFO[0004] Machine 'mytestmachine' created successfully
INFO[0004] API credentials dumped to '/etc/crowdsec/local_api_credentials.yaml'
$ sudo cscli machines list
@ -105,17 +105,15 @@ $ sudo cscli machines list
NAME IP ADDRESS LAST UPDATE STATUS VERSION
----------------------------------------------------------------------------------------------------------------------------------
82929df7ee394b73b81252fe3b4e5020 127.0.0.1 2020-10-31T14:06:32+01:00 ✔️ v0.3.6-3d6ce33908409f2a830af6551a7f5e37f2a4728f
mytestmachine 127.0.0.1 2020-11-01T11:37:19+01:00 ✔️ v0.3.6-6a18458badf8ae5fed8d5f1bb96fc7a59c96163c
mytestmachine 127.0.0.1 2020-11-01T11:37:19+01:00 ✔️ v0.3.6-6a18458badf8ae5fed8d5f1bb96fc7a59c96163c
----------------------------------------------------------------------------------------------------------------------------------
$ cscli machines delete -m 82929df7ee394b73b81252fe3b4e5020
$ cscli machines list
$ sudo cscli machines delete -m 82929df7ee394b73b81252fe3b4e5020
$ sudo cscli machines list
---------------------------------------------------------------------------------------------------------
NAME IP ADDRESS LAST UPDATE STATUS VERSION
---------------------------------------------------------------------------------------------------------
mytestmachine 127.0.0.1 2020-11-01T11:37:19+01:00 ✔️ v0.3.6-6a18458badf8ae5fed8d5f1bb96fc7a59c96163c
---------------------------------------------------------------------------------------------------------
```
</details>

6
docs/v1.X/docs/user_guide/configurations_management/acquisition.md
View file

@ -54,7 +54,7 @@ This allows you to see how many lines are coming from each source, and if they a
You can see those metrics with the following command:
```
{{v1X.cli.bin}} metrics
sudo {{v1X.cli.bin}} metrics
```
@ -62,7 +62,8 @@ You can see those metrics with the following command:
<summary>{{v1X.cli.name}} metrics example</summary>
```bash
## {{v1X.cli.bin}} metrics
$ sudo {{v1X.cli.bin}} metrics
...
...
INFO[0000] Acquisition Metrics:
+--------------------------------------+------------+--------------+----------------+------------------------+
@ -72,6 +73,7 @@ INFO[0000] Acquisition Metrics:
| journalctl-_SYSTEMD_UNIT=ssh.service | 36 | 12 | 24 | 17 |
+--------------------------------------+------------+--------------+----------------+------------------------+
...
...
```
</details>

20
docs/v1.X/docs/user_guide/configurations_management/collections.md
View file

@ -4,14 +4,14 @@
## Installing collections
```bash
$ cscli collections install crowdsecurity/whitelist-good-actors
$ sudo cscli collections install crowdsecurity/whitelist-good-actors
```
<details>
<summary>{{v1X.cli.name}} collection install example</summary>
```bash
$ cscli collections install crowdsecurity/whitelist-good-actors
$ sudo cscli collections install crowdsecurity/whitelist-good-actors
INFO[0000] crowdsecurity/seo-bots-whitelist : OK
INFO[0000] downloading data 'https://raw.githubusercontent.com/crowdsecurity/sec-lists/master/whitelists/benign_bots/search_engine_crawlers/rdns_seo_bots.txt' in '/var/lib/crowdsec/data/rdns_seo_bots.txt'
INFO[0001] downloading data 'https://raw.githubusercontent.com/crowdsecurity/sec-lists/master/whitelists/benign_bots/search_engine_crawlers/rnds_seo_bots.regex' in '/var/lib/crowdsec/data/rdns_seo_bots.regex'
@ -36,14 +36,14 @@ $ systemctl reload crowdsec
## Listing installed collections
```bash
$ {{v1X.cli.bin}} collections list
$ sudo {{v1X.cli.bin}} collections list
```
<details>
<summary>cscli collections list example</summary>
```bash
$ cscli collections list
$ sudo cscli collections list
-------------------------------------------------------------------------------------------------------------
NAME 📦 STATUS VERSION LOCAL PATH
-------------------------------------------------------------------------------------------------------------
@ -59,8 +59,8 @@ $ cscli collections list
## Upgrading installed collections
```bash
$ {{v1X.cli.bin}} hub update
$ {{v1X.cli.bin}} collections upgrade crowdsecurity/sshd
$ sudo {{v1X.cli.bin}} hub update
$ sudo {{v1X.cli.bin}} collections upgrade crowdsecurity/sshd
```
Collection upgrade allows you to upgrade an existing collection (and its items) to the latest version.
@ -70,7 +70,7 @@ Collection upgrade allows you to upgrade an existing collection (and its items)
<summary>cscli collections upgrade example</summary>
```bash
$ cscli collections upgrade crowdsecurity/sshd
$ sudo cscli collections upgrade crowdsecurity/sshd
INFO[0000] crowdsecurity/sshd : up-to-date
WARN[0000] crowdsecurity/sshd-logs : overwrite
WARN[0000] crowdsecurity/ssh-bf : overwrite
@ -87,7 +87,7 @@ $ systemctl reload crowdsec
## Monitoring collections
```bash
$ cscli collections inspect crowdsecurity/sshd
$ sudo cscli collections inspect crowdsecurity/sshd
```
Collections inspect will give you detailed information about a given collection, including versioning information *and* runtime metrics (fetched from prometheus).
@ -96,7 +96,7 @@ Collections inspect will give you detailed information about a given collection,
<summary>cscli collections inspect example</summary>
```bash
$ cscli collections inspect crowdsecurity/sshd
$ sudo cscli collections inspect crowdsecurity/sshd
type: collections
name: crowdsecurity/sshd
filename: sshd.yaml
@ -131,7 +131,7 @@ Current metrics :
```
<details>
</details>
## Reference documentation

2
docs/v1.X/docs/user_guide/configurations_management/enrichers.md
View file

@ -15,7 +15,7 @@ It exposes three methods : `GeoIpCity` `GeoIpASN` and `IpToRange` that are used
Enrichers can be installed as any other parsers with the following command:
```
{{v1X.cli.bin}} install parser crowdsecurity/geoip-enrich
sudo {{v1X.cli.bin}} parsers install crowdsecurity/geoip-enrich
```
Take a tour at the {{v1X.hub.htmlname}} to find them !

58
docs/v1.X/docs/user_guide/configurations_management/parsers.md
View file

@ -3,14 +3,14 @@
## Installing parsers
```bash
$ cscli parsers install crowdsecurity/sshd-logs
$ sudo cscli parsers install crowdsecurity/sshd-logs
```
<details>
<summary>cscli parsers install example</summary>
```bash
$ cscli parsers install crowdsecurity/iptables-logs
$ sudo cscli parsers install crowdsecurity/iptables-logs
INFO[0000] crowdsecurity/iptables-logs : OK
INFO[0000] Enabled parsers : crowdsecurity/iptables-logs
INFO[0000] Enabled crowdsecurity/iptables-logs
@ -21,19 +21,17 @@ INFO[0000] Run 'systemctl reload crowdsec' for the new configuration to be effec
## Listing installed parsers
```bash
cscli parsers list
sudo cscli parsers list
```
{{v1X.parsers.Htmlname}} are yaml files in `{{v1X.config.crowdsec_dir}}parsers/<STAGE>/parser.yaml`.
<details>
<summary>cscli parsers list example</summary>
```bash
$ cscli parsers list
$ sudo cscli parsers list
--------------------------------------------------------------------------------------------------------------
NAME 📦 STATUS VERSION LOCAL PATH
--------------------------------------------------------------------------------------------------------------
@ -55,7 +53,7 @@ $ cscli parsers list
## Upgrading installed parsers
```bash
$ {{v1X.cli.bin}} parsers upgrade crowdsecurity/sshd-logs
$ sudo {{v1X.cli.bin}} parsers upgrade crowdsecurity/sshd-logs
```
Parsers upgrade allows you to upgrade an existing parser to the latest version.
@ -64,7 +62,7 @@ Parsers upgrade allows you to upgrade an existing parser to the latest version.
<summary>cscli parsers upgrade example</summary>
```bash
$ cscli collections upgrade crowdsecurity/sshd
$ sudo cscli parsers upgrade crowdsecurity/sshd-logs
INFO[0000] crowdsecurity/sshd : up-to-date
WARN[0000] crowdsecurity/sshd-logs : overwrite
WARN[0000] crowdsecurity/ssh-bf : overwrite
@ -80,48 +78,44 @@ INFO[0000] Run 'systemctl reload crowdsec' for the new configuration to be effec
## Monitoring parsers
```bash
$ cscli collections inspect crowdsecurity/sshd
$ sudo cscli parsers inspect crowdsecurity/sshd-logs
```
Collections inspect will give you detailed information about a given collection, including versioning information *and* runtime metrics (fetched from prometheus).
Parsers inspect will give you detailed information about a given parser, including versioning information *and* runtime metrics (fetched from prometheus).
<!--TBD: refaire l'output apres avoir fix le 'parsers inspect XXXX'-->
<details>
<summary>cscli collections inspect example</summary>
<summary>cscli parsers inspect example</summary>
```bash
$ cscli collections inspect crowdsecurity/sshd
type: collections
name: crowdsecurity/sshd
filename: sshd.yaml
description: 'sshd support : parser and brute-force detection'
$ sudo cscli parsers inspect crowdsecurity/sshd-logs
type: parsers
stage: s01-parse
name: crowdsecurity/sshd-logs
filename: sshd-logs.yaml
description: Parse openSSH logs
author: crowdsecurity
belongs_to_collections:
- crowdsecurity/linux
- crowdsecurity/linux
remote_path: collections/crowdsecurity/sshd.yaml
- crowdsecurity/sshd
remote_path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml
version: "0.1"
local_path: /etc/crowdsec/collections/sshd.yaml
local_path: /etc/crowdsec/parsers/s01-parse/sshd-logs.yaml
localversion: "0.1"
localhash: 21159aeb87529efcf1a5033f720413d5321a6451bab679a999f7f01a7aa972b3
localhash: ecd40cb8cd95e2bad398824ab67b479362cdbf0e1598b8833e2f537ae3ce2f93
installed: true
downloaded: true
uptodate: true
tainted: false
local: false
parsers:
- crowdsecurity/sshd-logs
scenarios:
- crowdsecurity/ssh-bf
Current metrics :
Current metrics :
- (Scenario) crowdsecurity/ssh-bf:
+---------------+-----------+--------------+--------+---------+
| CURRENT COUNT | OVERFLOWS | INSTANCIATED | POURED | EXPIRED |
+---------------+-----------+--------------+--------+---------+
| 0 | 1 | 2 | 10 | 1 |
+---------------+-----------+--------------+--------+---------+
- (Parser) crowdsecurity/sshd-logs:
+-------------------+-------+--------+----------+
| PARSERS | HITS | PARSED | UNPARSED |
+-------------------+-------+--------+----------+
| /var/log/auth.log | 94138 | 42404 | 51734 |
+-------------------+-------+--------+----------+
```

51
docs/v1.X/docs/user_guide/configurations_management/scenarios.md
View file

@ -3,14 +3,14 @@
## Installing scenarios
```bash
$ cscli scenarios install crowdsecurity/http-bf-wordpress_bf
$ sudo cscli scenarios install crowdsecurity/http-bf-wordpress_bf
```
<details>
<summary>cscli scenarios install example</summary>
```bash
$ cscli scenarios install crowdsecurity/http-bf-wordpress_bf
$ sudo cscli scenarios install crowdsecurity/http-bf-wordpress_bf
INFO[0000] crowdsecurity/http-bf-wordpress_bf : OK
INFO[0000] Enabled scenarios : crowdsecurity/http-bf-wordpress_bf
INFO[0000] Enabled crowdsecurity/http-bf-wordpress_bf
@ -24,7 +24,7 @@ $ systemctl reload crowdsec
## Listing installed scenarios
```bash
cscli scenarios list
sudo cscli scenarios list
```
{{v1X.scenarios.Htmlname}} are yaml files in `{{v1X.config.crowdsec_dir}}scenarios/`.
@ -34,7 +34,7 @@ cscli scenarios list
<summary>cscli scenarios list example</summary>
```bash
$ cscli scenarios list
$ sudo cscli scenarios list
---------------------------------------------------------------------------------------------------------------------------
NAME 📦 STATUS VERSION LOCAL PATH
---------------------------------------------------------------------------------------------------------------------------
@ -58,7 +58,7 @@ $ cscli scenarios list
## Upgrading installed scenarios
```bash
$ cscli scenarios upgrade crowdsecurity/sshd-bf
$ sudo cscli scenarios upgrade crowdsecurity/sshd-bf
```
Scenarios upgrade allows you to upgrade an existing scenario to the latest version.
@ -67,7 +67,7 @@ Scenarios upgrade allows you to upgrade an existing scenario to the latest versi
<summary>cscli scenarios upgrade example</summary>
```bash
$ cscli scenarios upgrade crowdsecurity/ssh-bf
$ sudo cscli scenarios upgrade crowdsecurity/ssh-bf
INFO[0000] crowdsecurity/ssh-bf : up-to-date
WARN[0000] crowdsecurity/ssh-bf : overwrite
INFO[0000] 📦 crowdsecurity/ssh-bf : updated
@ -80,49 +80,44 @@ INFO[0000] Run 'systemctl reload crowdsec' for the new configuration to be effec
## Monitoring scenarios
```bash
$ cscli scenarios inspect crowdsecurity/ssh-bf
$ sudo cscli scenarios inspect crowdsecurity/ssh-bf
```
Collections inspect will give you detailed information about a given collection, including versioning information *and* runtime metrics (fetched from prometheus).
Scenarios inspect will give you detailed information about a given scenario, including versioning information *and* runtime metrics (fetched from prometheus).
<!--TBD: refaire l'output apres avoir fix le 'parsers inspect XXXX'-->
<details>
<summary>cscli collections inspect example</summary>
<summary>cscli scenarios inspect example</summary>
```bash
$ cscli collections inspect crowdsecurity/sshd
type: collections
name: crowdsecurity/sshd
filename: sshd.yaml
description: 'sshd support : parser and brute-force detection'
$ sudo cscli scenarios inspect crowdsecurity/ssh-bf
type: scenarios
name: crowdsecurity/ssh-bf
filename: ssh-bf.yaml
description: Detect ssh bruteforce
author: crowdsecurity
references:
- http://wikipedia.com/ssh-bf-is-bad
belongs_to_collections:
- crowdsecurity/linux
- crowdsecurity/linux
remote_path: collections/crowdsecurity/sshd.yaml
- crowdsecurity/sshd
remote_path: scenarios/crowdsecurity/ssh-bf.yaml
version: "0.1"
local_path: /etc/crowdsec/collections/sshd.yaml
local_path: /etc/crowdsec/scenarios/ssh-bf.yaml
localversion: "0.1"
localhash: 21159aeb87529efcf1a5033f720413d5321a6451bab679a999f7f01a7aa972b3
localhash: 4441dcff07020f6690d998b7101e642359ba405c2abb83565bbbdcee36de280f
installed: true
downloaded: true
uptodate: true
tainted: false
local: false
parsers:
- crowdsecurity/sshd-logs
scenarios:
- crowdsecurity/ssh-bf
Current metrics :
Current metrics :
- (Scenario) crowdsecurity/ssh-bf:
- (Scenario) crowdsecurity/ssh-bf:
+---------------+-----------+--------------+--------+---------+
| CURRENT COUNT | OVERFLOWS | INSTANCIATED | POURED | EXPIRED |
+---------------+-----------+--------------+--------+---------+
| 0 | 1 | 2 | 10 | 1 |
| 14 | 5700 | 7987 | 42572 | 2273 |
+---------------+-----------+--------------+--------+---------+
```
<details>

37
docs/v1.X/docs/user_guide/decision_management.md
View file

@ -1,28 +1,24 @@
!!! info
Please see your local `{{v1X.cli.bin}} help decisions` for up-to-date documentation.
Please see your local `sudo {{v1X.cli.bin}} help decisions` for up-to-date documentation.
## List active decisions
```bash
{{v1X.cli.bin}} decisions list
sudo {{v1X.cli.bin}} decisions list
```
<details>
<summary>example</summary>
```bash
bui@sd:~$ cscli decisions list
+-----+-----------+-------------+----------------------------------+--------+---------+-------------------------+--------+--------------------+
| ID | SOURCE | SCOPE:VALUE | REASON | ACTION | COUNTRY | AS | EVENTS | EXPIRATION |
+-----+-----------+------------------------------------------------+--------+---------+-------------------------+--------+--------------------+
| 1 | crowdsec | Ip:1.2.3.4 | crowdsecurity/ssh-bf (v0.5) | ban | CN | No.31,Jin-rong Street | 6 | 3h59m14.803995692s |
| 2 | crowdsec | Ip:1.2.3.4 | crowdsecurity/ssh-bf (v0.5) | ban | CN | No.31,Jin-rong Street | 6 | 3h59m14.803995692s |
| 3 | cscli | Ip:1.2.3.4 | manual ban | ban | | | 1 | 3h59m14.803995692s |
| 4 | cscli | Ip:1.2.3.5 | manual ban | ban | | | 1 | 3h59m58.986924109s |
+-----+-----------+-------------+----------------------------------+--------+---------+-------------------------+--------+--------------------+
$ sudo cscli decisions list
+-----+-----------+-------------+------------------------------------+--------+---------+----+--------+--------------------+----------+
| ID | SOURCE | SCOPE:VALUE | REASON | ACTION | COUNTRY | AS | EVENTS | EXPIRATION | ALERT ID |
+-----+-----------+-------------+------------------------------------+--------+---------+----+--------+--------------------+----------+
| 802 | cscli | Ip:1.2.3.5 | manual 'ban' from | ban | | | 1 | 3h50m58.10039043s | 802 |
| | | | 'b76cc7b1bbdc489e93909d2043031de8' | | | | | | |
| 801 | crowdsec | Ip:1.2.3.4 | crowdsecurity/ssh-bf | ban | | | 6 | 3h59m45.100387557s | 801 |
+-----+-----------+-------------+------------------------------------+--------+---------+----+--------+--------------------+----------+
```
</details>
@ -38,6 +34,7 @@ bui@sd:~$ cscli decisions list
- `COUNTRY` and `AS` are provided by GeoIP enrichment if present
- `EVENTS` number of event that triggered this decison
- `EXPIRATION` is the time left on remediation
- `ALERT ID` is the ID of the corresponding alert
Check [command usage](/Crowdsec/v1/cscli/cscli_decisions_list/) for additional filtering and output control flags.
@ -51,20 +48,20 @@ Check [command usage](/Crowdsec/v1/cscli/cscli_decisions_list/) for additional f
> Add a decision (ban) on IP `1.2.3.4` for 24 hours, with reason 'web bruteforce'
```bash
{{v1X.cli.bin}} decisions add --ip 1.2.3.4 --duration 24h --reason "web bruteforce"
sudo {{v1X.cli.bin}} decisions add --ip 1.2.3.4 --duration 24h --reason "web bruteforce"
```
> Add a decision (ban) on range `1.2.3.0/24` for 4 hours, with reason 'web bruteforce'
```bash
{{v1X.cli.bin}} decisions add --range 1.2.3.0/24 --reason "web bruteforce"
sudo {{v1X.cli.bin}} decisions add --range 1.2.3.0/24 --reason "web bruteforce"
```
> Add a decision (captcha) on ip `1.2.3.4` for 4hours (default duration), with reason 'web bruteforce'
```bash
{{v1X.cli.bin}} decisions add --ip 1.2.3.4 --reason "web bruteforce" --type captcha
sudo {{v1X.cli.bin}} decisions add --ip 1.2.3.4 --reason "web bruteforce" --type captcha
```
@ -74,13 +71,13 @@ Check [command usage](/Crowdsec/v1/cscli/cscli_decisions_list/) for additional f
> delete the decision on IP `1.2.3.4`
```bash
{{v1X.cli.bin}} decisions delete --ip 1.2.3.4
sudo {{v1X.cli.bin}} decisions delete --ip 1.2.3.4
```
> delete the decision on range 1.2.3.0/24
```bash
{{v1X.cli.bin}} decisions delete --range 1.2.3.0/24
sudo {{v1X.cli.bin}} decisions delete --range 1.2.3.0/24
```
@ -92,7 +89,7 @@ Check [command usage](/Crowdsec/v1/cscli/cscli_decisions_list/) for additional f
> Flush all the existing bans
```bash
{{v1X.cli.bin}} decisions delete --all
sudo {{v1X.cli.bin}} decisions delete --all
```
!!! warning

16
docs/v1.X/docs/user_guide/forensic_mode.md
View file

@ -9,21 +9,21 @@ When doing so, {{v1X.crowdsec.name}} will read the logs, extract timestamps from
you can run :
```bash
crowdsec -c /etc/crowdsec/user.yaml -file /path/to/your/log/file.log -type log_file_type
sudo crowdsec -c /etc/crowdsec/user.yaml -file /path/to/your/log/file.log -type log_file_type
```
Where `-file` points to the log file you want to process, and the `-type` is similar to what you would put in your acquisition's label field, for example :
```bash
crowdsec -c /etc/crowdsec/user.yaml -file /var/log/nginx/2019.log -type nginx
crowdsec -c /etc/crowdsec/user.yaml -file /var/log/sshd-2019.log -type syslog
crowdsec -c /etc/crowdsec/user.yaml -jfilter "_SYSTEMD_UNIT=ssh.service --since yesterday" -type syslog
sudo crowdsec -c /etc/crowdsec/user.yaml -file /var/log/nginx/2019.log -type nginx
sudo crowdsec -c /etc/crowdsec/user.yaml -file /var/log/sshd-2019.log -type syslog
sudo crowdsec -c /etc/crowdsec/user.yaml -jfilter "_SYSTEMD_UNIT=ssh.service --since yesterday" -type syslog
```
When running crowdsec in forensic mode, the alerts will be displayed to stdout, and as well pushed to database :
```bash
# crowdsec -c /etc/crowdsec/user.yaml -file /var/log/nginx/nginx-2019.log.1 -type nginx
$ sudo crowdsec -c /etc/crowdsec/user.yaml -file /var/log/nginx/nginx-2019.log.1 -type nginx
...
INFO[13-11-2020 13:05:23] Ip 123.206.50.249 performed 'crowdsecurity/http-probing' (11 events over 6s) at 2019-01-01 01:37:32 +0100 CET
INFO[13-11-2020 13:05:23] Ip 123.206.50.249 performed 'crowdsecurity/http-backdoors-attempts' (2 events over 1s) at 2019-01-01 01:37:33 +0100 CET
@ -40,7 +40,7 @@ And as these alerts are as well pushed to database, it mean you can view them in
If you already have a running crowdsec/Local API running and want to inject events into existing database, you can run crowdsec directly :
```bash
crowdsec -file ~/logs/nginx/access.log -type nginx --no-api
sudo crowdsec -file ~/logs/nginx/access.log -type nginx --no-api
```
Crowdsec will process `~/logs/nginx/access.log` and push alerts to the Local API configured in your default configuration file (`/etc/crowdsec/config.yaml`, see `api.client.credentials_path`)
@ -50,7 +50,7 @@ Crowdsec will process `~/logs/nginx/access.log` and push alerts to the Local API
If you don't have a service currently running, you can run crowdsec directly :
```bash
crowdsec -file ~/logs/nginx/access.log -type nginx
sudo crowdsec -file ~/logs/nginx/access.log -type nginx
```
Crowdsec will start a Local API and process `~/logs/nginx/access.log`.
@ -63,7 +63,7 @@ If you have a local instance running and you don't want to pollute your existing
Let's copy the existing configuration to edit it :
```bash
$ cp /etc/crowdsec/config.yaml ./forensic.yaml
$ sudo cp /etc/crowdsec/config.yaml ./forensic.yaml
$ emacs ./forensic.yaml
```

14
docs/v1.X/docs/user_guide/simulation_mode.md
View file

@ -1,7 +1,7 @@
# Simulation
```bash
$ cscli simulation status
$ sudo cscli simulation status
INFO[0000] global simulation: disabled
INFO[0000] Scenarios in simulation mode :
INFO[0000] - crowdsecurity/ssh-bf
@ -12,14 +12,16 @@ INFO[0000] - crowdsecurity/ssh-bf
You can add and remove scenarios to the simulation list :
```bash
$ cscli simulation enable crowdsecurity/ssh-bf
$ sudo cscli simulation enable crowdsecurity/ssh-bf
INFO[0000] simulation mode for 'crowdsecurity/ssh-bf' enabled
INFO[0000] Run 'systemctl reload crowdsec' for the new configuration to be effective.
$ systemctl reload crowdsec
$ tail -f /var/log/crowdsec.log
...
INFO[0000] Run 'sudo systemctl reload crowdsec' for the new configuration to be effective.
$ sudo systemctl reload crowdsec
$ sudo tail -f /var/log/crowdsec.log
....
time="01-11-2020 14:08:58" level=info msg="Ip 1.2.3.6 performed 'crowdsecurity/ssh-bf' (6 events over 986.769µs) at 2020-11-01 14:08:58.575885389 +0100 CET m=+437.524832750"
time="01-11-2020 14:08:58" level=info msg="Ip 1.2.3.6 decision : 1h (simulation) ban"
....
$ cscli decisions list
+----+----------+--------------+-----------------------------------+------------+---------+----+--------+------------------+
| ID | SOURCE | SCOPE:VALUE | REASON | ACTION | COUNTRY | AS | EVENTS | EXPIRATION |

4
docs/v1.X/docs/write_configurations/parsers.md
View file

@ -103,7 +103,9 @@ May 11 16:23:50 sd-126005 kernel: [47615902.763137] IN=enp1s0 OUT= MAC=00:08:a2:
Using an [online grok debugger](https://grokdebug.herokuapp.com/) or an [online regex debugger](https://www.debuggex.com/), we come up with the following grok pattern :
`\[%{DATA}\]+.*(%{WORD:action})? IN=%{WORD:int_eth} OUT= MAC=%{IP}:%{MAC} SRC=%{IP:src_ip} DST=%{IP:dst_ip} LEN=%{INT:length}.*PROTO=%{WORD:proto} SPT=%{INT:src_port} DPT=%{INT:dst_port}.*`
```
\[%{DATA}\]+.*(%{WORD:action})? IN=%{WORD:int_eth} OUT= MAC=%{IP}:%{MAC} SRC=%{IP:src_ip} DST=%{IP:dst_ip} LEN=%{INT:length}.*PROTO=%{WORD:proto} SPT=%{INT:src_port} DPT=%{INT:dst_port}.*
```
!!! warning
Check if the pattern you are looking for is not already present in [patterns configuration](https://github.com/crowdsecurity/crowdsec/tree/master/config/patterns).

12
wizard.sh
View file

@ -397,7 +397,7 @@ main() {
if [[ "$1" == "restore_from_dir" ]];
then
if ! [ $(id -u) = 0 ]; then
log_err "Please run it as root"
log_err "Please run the wizard as root or with sudo"
exit 1
fi
restore_from_dir
@ -407,7 +407,7 @@ main() {
if [[ "$1" == "binupgrade" ]];
then
if ! [ $(id -u) = 0 ]; then
log_err "Please run it as root"
log_err "Please run the wizard as root or with sudo"
exit 1
fi
update_bins
@ -417,7 +417,7 @@ main() {
if [[ "$1" == "upgrade" ]];
then
if ! [ $(id -u) = 0 ]; then
log_err "Please run it as root"
log_err "Please run the wizard as root or with sudo"
exit 1
fi
update_full
@ -427,7 +427,7 @@ main() {
if [[ "$1" == "uninstall" ]];
then
if ! [ $(id -u) = 0 ]; then
log_err "Please run it as root"
log_err "Please run the wizard as root or with sudo"
exit 1
fi
uninstall_crowdsec
@ -438,7 +438,7 @@ main() {
if [[ "$1" == "bininstall" ]];
then
if ! [ $(id -u) = 0 ]; then
log_err "Please run it as root"
log_err "Please run the wizard as root or with sudo"
exit 1
fi
log_info "installing crowdsec"
@ -450,7 +450,7 @@ main() {
if [[ "$1" == "install" ]];
then
if ! [ $(id -u) = 0 ]; then
log_err "Please run it as root"
log_err "Please run the wizard as root or with sudo"
exit 1
fi