diff --git a/README.md b/README.md
index 8bb6a2a75a19932e2d9cf4c763f539e53e1ee1d0..2fcb78d9261d368161287d8d60b23499a76fe7ac 100644
--- a/README.md
+++ b/README.md
@@ -35,6 +35,39 @@ One of the advantages of Crowdsec when compared to other solutions is its crowde
Besides detecting and stopping attacks in real time based on your logs, it allows you to preemptively block known bad actors from accessing your information system.
+
+## Install it !
+
+Find the [latest release](https://github.com/crowdsecurity/crowdsec/releases/latest)
+
+Ensure you have dependencies :
+
+ for Debian based distributions
+
+```bash
+apt-get install bash gettext whiptail curl wget
+```
+
+
+
+ for RedHat based distributions
+
+```bash
+yum install bash gettext newt curl wget
+ ```
+
+
+
+
+```bash
+curl -s https://api.github.com/repos/crowdsecurity/crowdsec/releases/latest | grep browser_download_url| cut -d '"' -f 4 | wget -i -
+tar xvzf crowdsec-release.tgz
+cd crowdsec-v*
+sudo ./wizard.sh -i
+```
+
+
+
## Key points
### Fast assisted installation, no technical barrier
diff --git a/cmd/crowdsec-cli/api.go b/cmd/crowdsec-cli/api.go
index bbaa19d3863aca11613468a9cde5abaa23bcb861..41ea0c49a840f53e79056972bb6e2d8eeba738d1 100644
--- a/cmd/crowdsec-cli/api.go
+++ b/cmd/crowdsec-cli/api.go
@@ -102,7 +102,6 @@ func pullTOP() error {
if _, ok := item["scenario"]; !ok {
continue
}
- item["scenario"] = fmt.Sprintf("api: %s", item["scenario"])
if _, ok := item["action"]; !ok {
continue
diff --git a/cmd/crowdsec-cli/ban.go b/cmd/crowdsec-cli/ban.go
index 3a65d0d67f1d51a621c46e44b0e599ee244fc101..b31f744914ea2849e8943cca35a42d5fe7ff3d14 100644
--- a/cmd/crowdsec-cli/ban.go
+++ b/cmd/crowdsec-cli/ban.go
@@ -20,7 +20,11 @@ import (
var remediationType string
var atTime string
-var all bool
+
+//user supplied filters
+var ipFilter, rangeFilter, reasonFilter, countryFilter, asFilter string
+var displayLimit int
+var displayAPI, displayALL bool
func simpleBanToSignal(targetIP string, reason string, expirationStr string, action string, asName string, asNum string, country string, banSource string) (types.SignalOccurence, error) {
var signalOcc types.SignalOccurence
@@ -84,6 +88,102 @@ func simpleBanToSignal(targetIP string, reason string, expirationStr string, act
return signalOcc, nil
}
+func filterBans(bans []map[string]string) ([]map[string]string, error) {
+
+ var retBans []map[string]string
+
+ for _, ban := range bans {
+ var banIP net.IP
+ var banRange *net.IPNet
+ var keep bool = true
+ var err error
+
+ if ban["iptext"] != "" {
+ if strings.Contains(ban["iptext"], "/") {
+ log.Debugf("%s is a range", ban["iptext"])
+ banIP, banRange, err = net.ParseCIDR(ban["iptext"])
+ if err != nil {
+ log.Warningf("failed to parse range '%s' from database : %s", ban["iptext"], err)
+ }
+ } else {
+ log.Debugf("%s is IP", ban["iptext"])
+ banIP = net.ParseIP(ban["iptext"])
+ }
+ }
+
+ if ipFilter != "" {
+ var filterBinIP net.IP = net.ParseIP(ipFilter)
+
+ if banRange != nil {
+ if banRange.Contains(filterBinIP) {
+ log.Debugf("[keep] ip filter is set, and range contains ip")
+ keep = true
+ } else {
+ log.Debugf("[discard] ip filter is set, and range doesn't contain ip")
+ keep = false
+ }
+ } else {
+ if ipFilter == ban["iptext"] {
+ log.Debugf("[keep] (ip) %s == %s", ipFilter, ban["iptext"])
+ keep = true
+ } else {
+ log.Debugf("[discard] (ip) %s == %s", ipFilter, ban["iptext"])
+ keep = false
+ }
+ }
+ }
+ if rangeFilter != "" {
+ _, filterBinRange, err := net.ParseCIDR(rangeFilter)
+ if err != nil {
+ return nil, fmt.Errorf("failed to parse range '%s' : %s", rangeFilter, err)
+ }
+ if filterBinRange.Contains(banIP) {
+ log.Debugf("[keep] range filter %s contains %s", rangeFilter, banIP.String())
+ keep = true
+ } else {
+ log.Debugf("[discard] range filter %s doesn't contain %s", rangeFilter, banIP.String())
+ keep = false
+ }
+ }
+ if reasonFilter != "" {
+ if strings.Contains(ban["reason"], reasonFilter) {
+ log.Debugf("[keep] reason filter %s matches %s", reasonFilter, ban["reason"])
+ keep = true
+ } else {
+ log.Debugf("[discard] reason filter %s doesn't match %s", reasonFilter, ban["reason"])
+ keep = false
+ }
+ }
+
+ if countryFilter != "" {
+ if ban["cn"] == countryFilter {
+ log.Debugf("[keep] country filter %s matches %s", countryFilter, ban["cn"])
+ keep = true
+ } else {
+ log.Debugf("[discard] country filter %s matches %s", countryFilter, ban["cn"])
+ keep = false
+ }
+ }
+
+ if asFilter != "" {
+ if strings.Contains(ban["as"], asFilter) {
+ log.Debugf("[keep] AS filter %s matches %s", asFilter, ban["as"])
+ keep = true
+ } else {
+ log.Debugf("[discard] AS filter %s doesn't match %s", asFilter, ban["as"])
+ keep = false
+ }
+ }
+
+ if keep {
+ retBans = append(retBans, ban)
+ } else {
+ log.Debugf("[discard] discard %v", ban)
+ }
+ }
+ return retBans, nil
+}
+
func BanList() error {
at := time.Now()
if atTime != "" {
@@ -96,6 +196,10 @@ func BanList() error {
if err != nil {
return fmt.Errorf("unable to get records from Database : %v", err)
}
+ ret, err = filterBans(ret)
+ if err != nil {
+ log.Errorf("Error while filtering : %s", err)
+ }
if config.output == "raw" {
fmt.Printf("source,ip,reason,bans,action,country,as,events_count,expiration\n")
for _, rm := range ret {
@@ -113,10 +217,9 @@ func BanList() error {
table.SetHeader([]string{"Source", "Ip", "Reason", "Bans", "Action", "Country", "AS", "Events", "Expiration"})
dispcount := 0
- totcount := 0
apicount := 0
for _, rm := range ret {
- if !all && rm["source"] == "api" {
+ if !displayAPI && rm["source"] == "api" {
apicount++
if _, ok := uniqAS[rm["as"]]; !ok {
uniqAS[rm["as"]] = true
@@ -124,27 +227,55 @@ func BanList() error {
if _, ok := uniqCN[rm["cn"]]; !ok {
uniqCN[rm["cn"]] = true
}
- continue
}
- if dispcount < 20 {
- table.Append([]string{rm["source"], rm["iptext"], rm["reason"], rm["bancount"], rm["action"], rm["cn"], rm["as"], rm["events_count"], rm["until"]})
+ if displayALL {
+ if rm["source"] == "api" {
+ if displayAPI {
+ table.Append([]string{rm["source"], rm["iptext"], rm["reason"], rm["bancount"], rm["action"], rm["cn"], rm["as"], rm["events_count"], rm["until"]})
+ dispcount++
+ continue
+ }
+ } else {
+ table.Append([]string{rm["source"], rm["iptext"], rm["reason"], rm["bancount"], rm["action"], rm["cn"], rm["as"], rm["events_count"], rm["until"]})
+ dispcount++
+ continue
+ }
+ } else if dispcount < displayLimit {
+ if displayAPI {
+ if rm["source"] == "api" {
+ table.Append([]string{rm["source"], rm["iptext"], rm["reason"], rm["bancount"], rm["action"], rm["cn"], rm["as"], rm["events_count"], rm["until"]})
+ dispcount++
+ continue
+ }
+ } else {
+ if rm["source"] != "api" {
+ table.Append([]string{rm["source"], rm["iptext"], rm["reason"], rm["bancount"], rm["action"], rm["cn"], rm["as"], rm["events_count"], rm["until"]})
+ dispcount++
+ continue
+ }
+ }
}
- totcount++
- dispcount++
-
}
if dispcount > 0 {
- if !all {
- fmt.Printf("%d local decisions:\n", totcount)
+ if !displayAPI {
+ fmt.Printf("%d local decisions:\n", dispcount)
+ } else if displayAPI && !displayALL {
+ fmt.Printf("%d decision from API\n", dispcount)
+ } else if displayALL && displayAPI {
+ fmt.Printf("%d decision from crowdsec and API\n", dispcount)
}
table.Render() // Send output
- if dispcount > 20 {
+ if dispcount > displayLimit && !displayALL {
fmt.Printf("Additional records stripped.\n")
}
} else {
- fmt.Printf("No local decisions.\n")
+ if displayAPI {
+ fmt.Println("No API decisions")
+ } else {
+ fmt.Println("No local decisions")
+ }
}
- if !all {
+ if !displayAPI {
fmt.Printf("And %d records from API, %d distinct AS, %d distinct countries\n", apicount, len(uniqAS), len(uniqCN))
}
}
@@ -167,7 +298,7 @@ func BanAdd(target string, duration string, reason string, action string) error
if err != nil {
return err
}
- log.Infof("Wrote ban to database.")
+ log.Infof("%s %s for %s (%s)", action, target, duration, reason)
return nil
}
@@ -225,7 +356,11 @@ cscli ban add range 1.2.3.0/24 24h "the whole range"`,
Run: func(cmd *cobra.Command, args []string) {
reason := strings.Join(args[2:], " ")
if err := BanAdd(args[0], args[1], reason, remediationType); err != nil {
+<<<<<<< HEAD
log.Fatalf("failed to add ban to database : %v", err)
+=======
+ log.Fatalf("failed to add ban to sqlite : %v", err)
+>>>>>>> master
}
},
}
@@ -239,7 +374,11 @@ cscli ban add range 1.2.3.0/24 24h "the whole range"`,
Run: func(cmd *cobra.Command, args []string) {
reason := strings.Join(args[2:], " ")
if err := BanAdd(args[0], args[1], reason, remediationType); err != nil {
+<<<<<<< HEAD
log.Fatalf("failed to add ban to database : %v", err)
+=======
+ log.Fatalf("failed to add ban to sqlite : %v", err)
+>>>>>>> master
}
},
}
@@ -301,7 +440,8 @@ cscli ban del range 1.2.3.0/24`,
Short: "List local or api bans/remediations",
Long: `List the bans, by default only local decisions.
-If --all/-a is specified, api-provided bans will be displayed too.
+If --all/-a is specified, bans will be displayed without limit (--limit).
+Default limit is 50.
Time can be specified with --at and support a variety of date formats:
- Jan 2 15:04:05
@@ -312,6 +452,10 @@ Time can be specified with --at and support a variety of date formats:
- 2006-01-02
- 2006-01-02 15:04
`,
+ Example: `ban list --range 0.0.0.0/0 : will list all
+ ban list --country CN
+ ban list --reason crowdsecurity/http-probing
+ ban list --as OVH`,
Args: cobra.ExactArgs(0),
Run: func(cmd *cobra.Command, args []string) {
if err := BanList(); err != nil {
@@ -320,7 +464,15 @@ Time can be specified with --at and support a variety of date formats:
},
}
cmdBanList.PersistentFlags().StringVar(&atTime, "at", "", "List bans at given time")
- cmdBanList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List as well bans received from API")
+ cmdBanList.PersistentFlags().BoolVarP(&displayALL, "all", "a", false, "List bans without limit")
+ cmdBanList.PersistentFlags().BoolVarP(&displayAPI, "api", "", false, "List as well bans received from API")
+ cmdBanList.PersistentFlags().StringVar(&ipFilter, "ip", "", "List bans for given IP")
+ cmdBanList.PersistentFlags().StringVar(&rangeFilter, "range", "", "List bans belonging to given range")
+ cmdBanList.PersistentFlags().StringVar(&reasonFilter, "reason", "", "List bans containing given reason")
+ cmdBanList.PersistentFlags().StringVar(&countryFilter, "country", "", "List bans belonging to given country code")
+ cmdBanList.PersistentFlags().StringVar(&asFilter, "as", "", "List bans belonging to given AS name")
+ cmdBanList.PersistentFlags().IntVar(&displayLimit, "limit", 50, "Limit of bans to display (default 50)")
+
cmdBan.AddCommand(cmdBanList)
return cmdBan
}
diff --git a/cmd/crowdsec-cli/install.go b/cmd/crowdsec-cli/install.go
index f0632c592405f3fc9087d94aa7c44f51bb10706a..dc76a7eb01dbd81d2a7e11f698d46d4aa3ad8378 100644
--- a/cmd/crowdsec-cli/install.go
+++ b/cmd/crowdsec-cli/install.go
@@ -71,7 +71,7 @@ you should [update cscli](./cscli_update.md).
var cmdInstallParser = &cobra.Command{
Use: "parser [config]",
- Short: "Install given log parser",
+ Short: "Install given parser",
Long: `Fetch and install given parser from hub`,
Example: `cscli install parser crowdsec/xxx`,
Args: cobra.MinimumNArgs(1),
@@ -79,7 +79,9 @@ you should [update cscli](./cscli_update.md).
if err := cwhub.GetHubIdx(); err != nil {
log.Fatalf("failed to get Hub index : %v", err)
}
- InstallItem(args[0], cwhub.PARSERS)
+ for _, name := range args {
+ InstallItem(name, cwhub.PARSERS)
+ }
},
}
cmdInstall.AddCommand(cmdInstallParser)
@@ -93,7 +95,9 @@ you should [update cscli](./cscli_update.md).
if err := cwhub.GetHubIdx(); err != nil {
log.Fatalf("failed to get Hub index : %v", err)
}
- InstallItem(args[0], cwhub.SCENARIOS)
+ for _, name := range args {
+ InstallItem(name, cwhub.SCENARIOS)
+ }
},
}
cmdInstall.AddCommand(cmdInstallScenario)
@@ -108,7 +112,9 @@ you should [update cscli](./cscli_update.md).
if err := cwhub.GetHubIdx(); err != nil {
log.Fatalf("failed to get Hub index : %v", err)
}
- InstallItem(args[0], cwhub.COLLECTIONS)
+ for _, name := range args {
+ InstallItem(name, cwhub.COLLECTIONS)
+ }
},
}
cmdInstall.AddCommand(cmdInstallCollection)
@@ -124,7 +130,9 @@ As a reminder, postoverflows are parsing configuration that will occur after the
if err := cwhub.GetHubIdx(); err != nil {
log.Fatalf("failed to get Hub index : %v", err)
}
- InstallItem(args[0], cwhub.PARSERS_OVFLW)
+ for _, name := range args {
+ InstallItem(name, cwhub.PARSERS_OVFLW)
+ }
},
}
cmdInstall.AddCommand(cmdInstallPostoverflow)
diff --git a/cmd/crowdsec-cli/remove.go b/cmd/crowdsec-cli/remove.go
index 984b536e0ee07adfe6f8011a5f22fd76948682fa..55c1e9baf4218f218624f5326c24629bcd147436 100644
--- a/cmd/crowdsec-cli/remove.go
+++ b/cmd/crowdsec-cli/remove.go
@@ -71,15 +71,13 @@ func NewRemoveCmd() *cobra.Command {
log.Fatalf("Failed to get Hub index : %v", err)
}
- if remove_all && len(args) == 0 {
+ if remove_all {
RemoveMany(cwhub.PARSERS, "")
- } else if len(args) == 1 {
- RemoveMany(cwhub.PARSERS, args[0])
} else {
- _ = cmd.Help()
- return
+ for _, name := range args {
+ RemoveMany(cwhub.PARSERS, name)
+ }
}
- //fmt.Println("remove/disable parser: " + strings.Join(args, " "))
},
}
cmdRemove.AddCommand(cmdRemoveParser)
@@ -92,13 +90,12 @@ func NewRemoveCmd() *cobra.Command {
if err := cwhub.GetHubIdx(); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
}
- if remove_all && len(args) == 0 {
+ if remove_all {
RemoveMany(cwhub.SCENARIOS, "")
- } else if len(args) == 1 {
- RemoveMany(cwhub.SCENARIOS, args[0])
} else {
- _ = cmd.Help()
- return
+ for _, name := range args {
+ RemoveMany(cwhub.SCENARIOS, name)
+ }
}
},
}
@@ -112,13 +109,12 @@ func NewRemoveCmd() *cobra.Command {
if err := cwhub.GetHubIdx(); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
}
- if remove_all && len(args) == 0 {
+ if remove_all {
RemoveMany(cwhub.COLLECTIONS, "")
- } else if len(args) == 1 {
- RemoveMany(cwhub.COLLECTIONS, args[0])
} else {
- _ = cmd.Help()
- return
+ for _, name := range args {
+ RemoveMany(cwhub.COLLECTIONS, name)
+ }
}
},
}
@@ -133,13 +129,12 @@ func NewRemoveCmd() *cobra.Command {
if err := cwhub.GetHubIdx(); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
}
- if remove_all && len(args) == 0 {
+ if remove_all {
RemoveMany(cwhub.PARSERS_OVFLW, "")
- } else if len(args) == 1 {
- RemoveMany(cwhub.PARSERS_OVFLW, args[0])
} else {
- _ = cmd.Help()
- return
+ for _, name := range args {
+ RemoveMany(cwhub.PARSERS_OVFLW, name)
+ }
}
},
}
diff --git a/cmd/crowdsec-cli/upgrade.go b/cmd/crowdsec-cli/upgrade.go
index da0bafc08c5ae52da12d0d9f0ace4fbcb73f96ba..6fdacb03a6c970007ed21c3c7da56a6c04ea8732 100644
--- a/cmd/crowdsec-cli/upgrade.go
+++ b/cmd/crowdsec-cli/upgrade.go
@@ -124,14 +124,14 @@ cscli upgrade --force # Overwrite tainted configuration
if err := cwhub.GetHubIdx(); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
}
- if len(args) == 1 {
- UpgradeConfig(cwhub.PARSERS, args[0])
- //UpgradeConfig(cwhub.PARSERS_OVFLW, "")
- } else if upgrade_all {
+ if upgrade_all {
UpgradeConfig(cwhub.PARSERS, "")
} else {
- _ = cmd.Help()
+ for _, name := range args {
+ UpgradeConfig(cwhub.PARSERS, name)
+ }
}
+
},
}
cmdUpgrade.AddCommand(cmdUpgradeParser)
@@ -146,12 +146,12 @@ cscli upgrade --force # Overwrite tainted configuration
if err := cwhub.GetHubIdx(); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
}
- if len(args) == 1 {
- UpgradeConfig(cwhub.SCENARIOS, args[0])
- } else if upgrade_all {
+ if upgrade_all {
UpgradeConfig(cwhub.SCENARIOS, "")
} else {
- _ = cmd.Help()
+ for _, name := range args {
+ UpgradeConfig(cwhub.SCENARIOS, name)
+ }
}
},
}
@@ -168,12 +168,12 @@ cscli upgrade --force # Overwrite tainted configuration
if err := cwhub.GetHubIdx(); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
}
- if len(args) == 1 {
- UpgradeConfig(cwhub.COLLECTIONS, args[0])
- } else if upgrade_all {
+ if upgrade_all {
UpgradeConfig(cwhub.COLLECTIONS, "")
} else {
- _ = cmd.Help()
+ for _, name := range args {
+ UpgradeConfig(cwhub.COLLECTIONS, name)
+ }
}
},
}
@@ -191,12 +191,12 @@ cscli upgrade --force # Overwrite tainted configuration
if err := cwhub.GetHubIdx(); err != nil {
log.Fatalf("Failed to get Hub index : %v", err)
}
- if len(args) == 1 {
- UpgradeConfig(cwhub.PARSERS_OVFLW, args[0])
- } else if upgrade_all {
+ if upgrade_all {
UpgradeConfig(cwhub.PARSERS_OVFLW, "")
} else {
- _ = cmd.Help()
+ for _, name := range args {
+ UpgradeConfig(cwhub.PARSERS_OVFLW, name)
+ }
}
},
}
diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go
index 643e1027d0d7d129013d4d82f358ddd557a09d27..c4ae8eb70a422593a705d86607fc0e29776983ad 100644
--- a/cmd/crowdsec/main.go
+++ b/cmd/crowdsec/main.go
@@ -10,6 +10,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/acquisition"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
+ "github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/outputs"
"github.com/crowdsecurity/crowdsec/pkg/parser"
@@ -282,6 +283,11 @@ func main() {
go runTachymeter(cConfig.HTTPListen)
}
+ err = exprhelpers.Init()
+ if err != nil {
+ log.Fatalf("Failed to init expr helpers : %s", err)
+ }
+
// Start loading configs
if err := LoadParsers(cConfig); err != nil {
log.Fatalf("Failed to load parsers: %s", err)
diff --git a/cmd/crowdsec/output.go b/cmd/crowdsec/output.go
index 925c2f41ad32dbea4334f30bdcc9aa87a86fd7bd..f4c8d23345d17d9a06267432f69ba798cda481ad 100644
--- a/cmd/crowdsec/output.go
+++ b/cmd/crowdsec/output.go
@@ -1,6 +1,8 @@
package main
import (
+ "fmt"
+
log "github.com/sirupsen/logrus"
"time"
@@ -40,6 +42,12 @@ LOOP:
input <- event
}
+ /* process post overflow parser nodes */
+ event, err := parser.Parse(poctx, event, ponodes)
+ if err != nil {
+ return fmt.Errorf("postoverflow failed : %s", err)
+ }
+
if event.Overflow.Scenario == "" && event.Overflow.MapKey != "" {
//log.Infof("Deleting expired entry %s", event.Overflow.MapKey)
buckets.Bucket_map.Delete(event.Overflow.MapKey)
diff --git a/config/plugins/backend/sqlite.yaml b/config/plugins/backend/sqlite.yaml
new file mode 100644
index 0000000000000000000000000000000000000000..0d04e7664500491f4da223c43fc6f7ebf746bfb0
--- /dev/null
+++ b/config/plugins/backend/sqlite.yaml
@@ -0,0 +1,4 @@
+name: sqlite
+path: /usr/local/lib/crowdsec/plugins/backend/sqlite.so
+config:
+ db_path: /var/lib/crowdsec/data/crowdsec.db
diff --git a/config/profiles.yaml b/config/profiles.yaml
index 73d5ad5cfa4c0e1db3c2c4083ab66282117debf6..5ad8e4b583dee7f2140ef9123f03ae326be5429c 100644
--- a/config/profiles.yaml
+++ b/config/profiles.yaml
@@ -1,5 +1,5 @@
profile: default_remediation
-filter: "sig.Labels.remediation == 'true'"
+filter: "sig.Labels.remediation == 'true' && not sig.Whitelisted"
api: true # If no api: specified, will use the default config in default.yaml
remediation:
ban: true
@@ -16,3 +16,11 @@ api: false
outputs:
- plugin: database # If we do not want to push, we can remove this line and the next one
store: false
+---
+profile: send_false_positif_to_API
+filter: "sig.Whitelisted == true && sig.Labels.remediation == 'true'"
+#remediation is empty, it means non taken
+api: true
+outputs:
+ - plugin: sqlite # If we do not want to push, we can remove this line and the next one
+ store: false
\ No newline at end of file
diff --git a/docs/assets/images/crowdsec_architecture.png b/docs/assets/images/crowdsec_architecture.png
index 8ba749de297cbb3aab8c23651e50dbd4f3ee48fe..5e5e6184de90cbc9d6e47aa62fb374fc5234324a 100644
Binary files a/docs/assets/images/crowdsec_architecture.png and b/docs/assets/images/crowdsec_architecture.png differ
diff --git a/docs/index.md b/docs/index.md
index 092a32553e4e25f2fb272f5f79b29510c67917a9..9d59cb58a0cd97156b2b10504f0359c89cd07c08 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -27,6 +27,14 @@ Besides detecting and stopping attacks in real time based on your logs, it allow
+
+## Core concepts
+
+{{crowdsec.name}} relies on {{parsers.htmlname}} to normalize and enrich logs, and {{scenarios.htmlname}} to detect attacks, often bundled together in {{collections.htmlname}} to form a coherent configuration set. For example the collection [`crowdsecurity/nginx`](https://hub.crowdsec.net/author/crowdsecurity/collections/nginx) contains all the necessary parsers and scenarios to deal with nginx logs and the common attacks that can be seen on http servers.
+
+All of those are represented as YAML files, that can be found, shared and kept up-to-date thanks to the {{hub.htmlname}}, or [easily hand-crafted](/write_configurations/scenarios/) to address specific needs.
+
+
## Moving forward
To learn more about {{crowdsec.name}} and give it a try, please see :
diff --git a/docs/references/parsers.md b/docs/references/parsers.md
index 0a4638ff012e478cb29757ea765b9f611e7220bf..d887fbe1f8857821c88df31a18d197532e0cc1c9 100644
--- a/docs/references/parsers.md
+++ b/docs/references/parsers.md
@@ -151,10 +151,14 @@ It is meant to help understanding parser node behaviour by providing contextual
filter: expression
```
-`filter` must be a valid {{expr.htmlname}} expression that will be evaluated against the {{event.name}}.
+`filter` must be a valid {{expr.htmlname}} expression that will be evaluated against the {{event.htmlname}}.
+
If `filter` evaluation returns true or is absent, node will be processed.
+
If `filter` returns `false` or a non-boolean, node won't be processed.
+Here is the [expr documentation](https://github.com/antonmedv/expr/tree/master/docs).
+
Examples :
- `filter: "evt.Meta.foo == 'test'"`
@@ -278,6 +282,30 @@ statics:
expression: evt.Meta.target_field + ' this_is' + ' a dynamic expression'
```
+### data
+
+```
+data:
+ - source_url: https://URL/TO/FILE
+ dest_file: LOCAL_FILENAME
+ [type: regexp]
+```
+
+`data` allows user to specify an external source of data.
+This section is only relevant when `cscli` is used to install parser from hub, as it will download the `source_url` and store it to `dest_file`. When the parser is not installed from the hub, {{crowdsec.name}} won't download the URL, but the file must exist for the parser to be loaded correctly.
+
+If `type` is set to `regexp`, the content of the file must be one valid (re2) regular expression per line.
+Those regexps will be compiled and kept in cache.
+
+
+```yaml
+name: crowdsecurity/cdn-whitelist
+...
+data:
+ - source_url: https://www.cloudflare.com/ips-v4
+ dest_file: cloudflare_ips.txt
+```
+
## Parser concepts
diff --git a/docs/references/scenarios.md b/docs/references/scenarios.md
index 01c13bfef286d5eeb92d39e5ee4a49adb9e09b69..51634bfacbff0c2291bb40e4d435bc9712b55d52 100644
--- a/docs/references/scenarios.md
+++ b/docs/references/scenarios.md
@@ -87,12 +87,16 @@ The name must be unique (and will define the scenario's name in the hub), and th
### filter
```yaml
-filter: evt.Meta.log_type == 'telnet_new_session'
+filter: expression
```
+`filter` must be a valid {{expr.htmlname}} expression that will be evaluated against the {{event.htmlname}}.
-an {{expr.htmlname}} that must return true if the event is eligible for the bucket.
+If `filter` evaluation returns true or is absent, event will be pour in the bucket.
+If `filter` returns `false` or a non-boolean, the event will be skip for this bucket.
+
+Here is the [expr documentation](https://github.com/antonmedv/expr/tree/master/docs).
Examples :
@@ -343,3 +347,28 @@ overflow_filter: any(queue.Queue, { .Enriched.IsInEU == "true" })
If this expression is present and returns false, the overflow will be discarded.
+### data
+
+```
+data:
+ - source_url: https://URL/TO/FILE
+ dest_file: LOCAL_FILENAME
+ [type: regexp]
+```
+
+`data` allows user to specify an external source of data.
+This section is only relevant when `cscli` is used to install scenario from hub, as ill download the `source_url` and store it to `dest_file`. When the scenario is not installed from the hub, {{crowdsec.name}} won't download the URL, but the file must exist for the scenario to be loaded correctly.
+
+If `type` is set to `regexp`, the content of the file must be one valid (re2) regular expression per line.
+Those regexps will be compiled and kept in cache.
+
+
+```yaml
+name: crowdsecurity/cdn-whitelist
+...
+data:
+ - source_url: https://www.cloudflare.com/ips-v4
+ dest_file: cloudflare_ips.txt
+```
+
+
diff --git a/docs/write_configurations/expressions.md b/docs/write_configurations/expressions.md
new file mode 100644
index 0000000000000000000000000000000000000000..09204f736ad43ff8d512f1ebff77b90b8c243c55
--- /dev/null
+++ b/docs/write_configurations/expressions.md
@@ -0,0 +1,52 @@
+# Expressions
+
+> {{expr.htmlname}} : Expression evaluation engine for Go: fast, non-Turing complete, dynamic typing, static typing
+
+
+Several places of {{crowdsec.name}}'s configuration use {{expr.htmlname}} :
+
+ - {{filter.Htmlname}} that are used to determine events eligibility in {{parsers.htmlname}} and {{scenarios.htmlname}} or `profiles`
+ - {{statics.Htmlname}} use expr in the `expression` directive, to compute complex values
+ - {{whitelists.Htmlname}} rely on `expression` directive to allow more complex whitelists filters
+
+To learn more about {{expr.htmlname}}, [check the github page of the project](https://github.com/antonmedv/expr/blob/master/docs/Language-Definition.md).
+
+In order to makes its use in {{crowdsec.name}} more efficient, we added a few helpers that are documented bellow.
+
+## Atof(string) float64
+
+Parses a string representation of a float number to an actual float number (binding on `strconv.ParseFloat`)
+
+> Atof(evt.Parsed.tcp_port)
+
+
+## JsonExtract(JsonBlob, FieldName) string
+
+Extract the `FieldName` from the `JsonBlob` and returns it as a string. (binding on [jsonparser](https://github.com/buger/jsonparser/))
+
+> JsonExtract(evt.Parsed.some_json_blob, "foo.bar[0].one_item")
+
+## File(FileName) []string
+
+Returns the content of `FileName` as an array of string, while providing cache mechanism.
+
+> evt.Parsed.some_field in File('some_patterns.txt')
+> any(File('rdns_seo_bots.txt'), { evt.Enriched.reverse_dns endsWith #})
+
+## RegexpInFile(StringToMatch, FileName) bool
+
+Returns `true` if the `StringToMatch` is matched by one of the expressions contained in `FileName` (uses RE2 regexp engine).
+
+> RegexpInFile( evt.Enriched.reverse_dns, 'my_legit_seo_whitelists.txt')
+
+## Upper(string) string
+
+Returns the uppercase version of the string
+
+> Upper("yop")
+
+## IpInRange(IPStr, RangeStr) bool
+
+Returns true if the IP `IPStr` is contained in the IP range `RangeStr` (uses `net.ParseCIDR`)
+
+> IpInRange("1.2.3.4", "1.2.3.0/24")
diff --git a/docs/write_configurations/parsers.md b/docs/write_configurations/parsers.md
index 593009ba2e7c50b85cf4fe9ff0010e1f81d628e6..ed5917eb1030ae5afabd9f84bddc089ab5d22c68 100644
--- a/docs/write_configurations/parsers.md
+++ b/docs/write_configurations/parsers.md
@@ -3,6 +3,12 @@
!!! info
Please ensure that you have working env or setup test environment before writing your parser.
+!!! warning "Parser dependency"
+
+The crowdsecurity/syslog-logs parsers is needed by the core parsing
+engine. Deletion or modification of this could result of {{crowdsec.name}}
+being unable to parse logs, so this should be done very carefully.
+
> In the current example, we'll write a parser for the logs produced by `iptables` (netfilter) with the `-j LOG` target.
> This document aims at detailing the process of writing and testing new parsers.
@@ -410,4 +416,4 @@ statics:
- meta: http_path
expression: "evt.Parsed.request"
```
- -->
\ No newline at end of file
+ -->
diff --git a/docs/write_configurations/whitelist.md b/docs/write_configurations/whitelist.md
index 2f91f56245ea8fa18ea97c94bc802ed30c6d5344..85ac1909cfc795ae7ec7d7cb048e4b3d007be035 100644
--- a/docs/write_configurations/whitelist.md
+++ b/docs/write_configurations/whitelist.md
@@ -1,15 +1,28 @@
-## Where are whitelists
+# What are whitelists
-Whitelists are, as for most configuration, YAML files, and allow you to "discard" signals based on :
+Whitelists are special parsers that allow you to "discard" events, and can exist at two different steps :
- - ip adress or the fact that it belongs to a specific range
- - a {{expr.name}} expression
+ - *Parser whitelists* : Allows you to discard an event at parse time, so that it never hits the buckets.
+ - *PostOverflow whitelists* : Those are whitelists that are checked *after* the overflow happens. It is usually best for whitelisting process that can be expensive (such as performing reverse DNS on an IP, or performing a `whois` of an IP).
-Here is an example :
+!!! info
+ While the whitelists are the same for parser or postoverflows, beware that field names might change.
+ Source ip is usually in `evt.Meta.source_ip` when it's a log, but `evt.Overflow.Source_ip` when it's an overflow
+
+
+The whitelist can be based on several criteria :
+
+ - specific ip address : if the event/overflow IP is the same, event is whitelisted
+ - ip ranges : if the event/overflow IP belongs to this range, event is whitelisted
+ - a list of {{expr.htmlname}} expressions : if any expression returns true, event is whitelisted
+
+Here is an example showcasing configuration :
```yaml
name: crowdsecurity/my-whitelists
description: "Whitelist events from my ipv4 addresses"
+#it's a normal parser, so we can restrict its scope with filter
+filter: "1 == 1"
whitelist:
reason: "my ipv4 ranges"
ip:
@@ -19,67 +32,75 @@ whitelist:
- "10.0.0.0/8"
- "172.16.0.0/12"
expression:
- - "'mycorp.com' in evt.Meta.source_ip_rdns"
+ #beware, this one will work *only* if you enabled the reverse dns (crowdsecurity/rdns) enrichment postoverflow parser
+ - evt.Enriched.reverse_dns endsWith ".mycoolorg.com."
+ #this one will work *only* if you enabled the geoip (crowdsecurity/geoip-enrich) enrichment parser
+ - evt.Enriched.IsoCode == 'FR'
```
-## Hands on
-Let's assume we have a setup with a `crowdsecurity/base-http-scenarios` scenario enabled and no whitelists.
+# Whitelists in parsing
+
+When a whitelist is present in parsing `/etc/crowdsec/config/parsers/...`, it will be checked/discarded before being poured to any bucket. These whitelists intentionally generate no logs and are useful to discard noisy false positive sources.
+
+## Whitelist by ip
+
+Let's assume we have a setup with a `crowdsecurity/nginx` collection enabled and no whitelists.
Thus, if I "attack" myself :
```bash
-nikto -host 127.0.0.1
+nikto -host myfqdn.com
```
my own IP will be flagged as being an attacker :
```bash
$ tail -f /var/log/crowdsec.log
-time="07-05-2020 09:23:03" level=warning msg="127.0.0.1 triggered a 4h0m0s ip ban remediation for [crowdsecurity/http-scan-uniques_404]" bucket_id=old-surf event_time="2020-05-07 09:23:03.322277347 +0200 CEST m=+57172.732939890" scenario=crowdsecurity/http-scan-uniques_404 source_ip=127.0.0.1
-time="07-05-2020 09:23:03" level=warning msg="127.0.0.1 triggered a 4h0m0s ip ban remediation for [crowdsecurity/http-crawl-non_statics]" bucket_id=lingering-sun event_time="2020-05-07 09:23:03.345341864 +0200 CEST m=+57172.756004380" scenario=crowdsecurity/http-crawl-non_statics source_ip=127.0.0.1
+ime="07-07-2020 16:13:16" level=warning msg="80.x.x.x triggered a 4h0m0s ip ban remediation for [crowdsecurity/http-bad-user-agent]" bucket_id=cool-smoke event_time="2020-07-07 16:13:16.579581642 +0200 CEST m=+358819.413561109" scenario=crowdsecurity/http-bad-user-agent source_ip=80.x.x.x
+time="07-07-2020 16:13:16" level=warning msg="80.x.x.x triggered a 4h0m0s ip ban remediation for [crowdsecurity/http-probing]" bucket_id=green-silence event_time="2020-07-07 16:13:16.737579458 +0200 CEST m=+358819.571558901" scenario=crowdsecurity/http-probing source_ip=80.x.x.x
+time="07-07-2020 16:13:17" level=warning msg="80.x.x.x triggered a 4h0m0s ip ban remediation for [crowdsecurity/http-crawl-non_statics]" bucket_id=purple-snowflake event_time="2020-07-07 16:13:17.353641625 +0200 CEST m=+358820.187621068" scenario=crowdsecurity/http-crawl-non_statics source_ip=80.x.x.x
+time="07-07-2020 16:13:18" level=warning msg="80.x.x.x triggered a 4h0m0s ip ban remediation for [crowdsecurity/http-sensitive-files]" bucket_id=small-hill event_time="2020-07-07 16:13:18.005919055 +0200 CEST m=+358820.839898498" scenario=crowdsecurity/http-sensitive-files source_ip=80.x.x.x
^C
$ {{cli.bin}} ban list
-1 local decisions:
-+--------+-----------+-------------------------------------+------+--------+---------+----+--------+------------+
-| SOURCE | IP | REASON | BANS | ACTION | COUNTRY | AS | EVENTS | EXPIRATION |
-+--------+-----------+-------------------------------------+------+--------+---------+----+--------+------------+
-| local | 127.0.0.1 | crowdsecurity/http-scan-uniques_404 | 2 | ban | | 0 | 47 | 3h55m57s |
-+--------+-----------+-------------------------------------+------+--------+---------+----+--------+------------+
+4 local decisions:
++--------+---------------+-----------------------------------+------+--------+---------+---------------------------+--------+------------+
+| SOURCE | IP | REASON | BANS | ACTION | COUNTRY | AS | EVENTS | EXPIRATION |
++--------+---------------+-----------------------------------+------+--------+---------+---------------------------+--------+------------+
+| local | 80.x.x.x | crowdsecurity/http-bad-user-agent | 4 | ban | FR | 21502 SFR SA | 60 | 3h59m3s |
+...
```
-## Create the whitelist by IP
-Let's create a `/etc/crowdsec/crowdsec/parsers/s02-enrich/whitelists.yaml` file with the following content :
+### Create the whitelist by IP
+
+Let's create a `/etc/crowdsec/crowdsec/parsers/s02-enrich/mywhitelists.yaml` file with the following content :
```yaml
name: crowdsecurity/whitelists
-description: "Whitelist events from private ipv4 addresses"
+description: "Whitelist events from my ip addresses"
whitelist:
- reason: "private ipv4 ranges"
- ip:
- - "127.0.0.1"
-
+ reason: "my ip ranges"
+ ip:
+ - "80.x.x.x"
```
-and restart {{crowdsec.name}} : `sudo systemctl restart {{crowdsec.name}}`
+and reload {{crowdsec.name}} : `sudo systemctl restart crowdsec`
-## Test the whitelist
+### Test the whitelist
Thus, if we restart our attack :
```bash
-nikto -host 127.0.0.1
+nikto -host myfqdn.com
```
-And we don't get bans, instead :
+And we don't get bans :
```bash
$ tail -f /var/log/crowdsec.log
...
-time="07-05-2020 09:30:13" level=info msg="Event from [127.0.0.1] is whitelisted by Ips !" filter= name=lively-firefly stage=s02-enrich
-...
^C
$ {{cli.bin}} ban list
No local decisions.
@@ -87,11 +108,12 @@ And 21 records from API, 15 distinct AS, 12 distinct countries
```
+Here, we don't get *any* logs, as the event have been discarded at parsing time.
## Create whitelist by expression
-Now, let's make something more tricky : let's whitelist a **specific** user-agent (of course, it's just an example, don't do this at home !).
+Now, let's make something more tricky : let's whitelist a **specific** user-agent (of course, it's just an example, don't do this at home !). The [hub's taxonomy](https://hub.crowdsec.net/fields) will helps us to find which data is present in which field.
Let's change our whitelist to :
@@ -109,7 +131,7 @@ again, let's restart {{crowdsec.name}} !
For the record, I edited nikto's configuration to use 'MySecretUserAgent' as user-agent, and thus :
```bash
-nikto -host 127.0.0.1
+nikto -host myfqdn.com
```
```bash
@@ -120,3 +142,43 @@ time="07-05-2020 09:39:09" level=info msg="Event is whitelisted by Expr !" filte
```
+# Whitelist in PostOverflows
+
+Whitelists in PostOverflows are applied *after* the bucket overflow happens.
+It has the advantage of being triggered only once we are about to take decision about an IP or Range, and thus happens a lot less often.
+
+A good example is the [crowdsecurity/whitelist-good-actors](https://hub.crowdsec.net/author/crowdsecurity/collections/whitelist-good-actors) collection.
+
+But let's craft ours based on our previous example !
+First of all, install the [crowdsecurity/rdns postoverflow](https://hub.crowdsec.net/author/crowdsecurity/configurations/rdns) : it will be in charge of enriching overflows with reverse dns information of the offending IP.
+
+Let's put the following file in `/etc/crowdsec/config/postoverflows/s01-whitelists/mywhitelists.yaml` :
+
+```yaml
+name: me/my_cool_whitelist
+description: lets whitelist our own reverse dns
+whitelist:
+ reason: dont ban my ISP
+ expression:
+ #this is the reverse of my ip, you can get it by performing a "host" command on your public IP for example
+ - evt.Enriched.reverse_dns endsWith '.asnieres.rev.numericable.fr.'
+```
+
+After reloading {{crowdsec.name}}, and launching (again!) nikto :
+
+```bash
+nikto -host myfqdn.com
+```
+
+
+```bash
+$ tail -f /var/log/crowdsec.log
+ime="07-07-2020 17:11:09" level=info msg="Ban for 80.x.x.x whitelisted, reason [dont ban my ISP]" id=cold-sunset name=me/my_cool_whitelist stage=s01
+time="07-07-2020 17:11:09" level=info msg="node warning : no remediation" bucket_id=blue-cloud event_time="2020-07-07 17:11:09.175068053 +0200 CEST m=+2308.040825320" scenario=crowdsecurity/http-probing source_ip=80.x.x.x
+time="07-07-2020 17:11:09" level=info msg="Processing Overflow with no decisions 80.x.x.x performed 'crowdsecurity/http-probing' (11 events over 313.983994ms) at 2020-07-07 17:11:09.175068053 +0200 CEST m=+2308.040825320" bucket_id=blue-cloud event_time="2020-07-07 17:11:09.175068053 +0200 CEST m=+2308.040825320" scenario=crowdsecurity/http-probing source_ip=80.x.x.x
+...
+
+```
+
+This time, we can see that logs are being produced when the event is discarded.
+
diff --git a/mkdocs.yml b/mkdocs.yml
index 0cbc9af04c64e30c5d7576f75fdb5cbbc322a7a3..63d567855d25dbc81d2cbb310e61ea0b4767a644 100644
--- a/mkdocs.yml
+++ b/mkdocs.yml
@@ -17,6 +17,7 @@ nav:
- Cheat Sheets:
- Ban Management: cheat_sheets/ban-mgmt.md
- Configuration Management: cheat_sheets/config-mgmt.md
+ - Hub's taxonomy: https://hub.crowdsec.net/fields
- Observability:
- Overview: observability/overview.md
- Logs: observability/logs.md
@@ -31,7 +32,8 @@ nav:
- Acquisition: write_configurations/acquisition.md
- Parsers: write_configurations/parsers.md
- Scenarios: write_configurations/scenarios.md
- - Whitelist: write_configurations/whitelist.md
+ - Whitelists: write_configurations/whitelist.md
+ - Expressions: write_configurations/expressions.md
- Blockers:
- Overview : blockers/index.md
- Nginx:
@@ -204,6 +206,11 @@ extra:
Name: Overflow
htmlname: "[overflow](/getting_started/glossary/#overflow-or-signaloccurence)"
Htmlname: "[Overflow](/getting_started/glossary/#overflow-or-signaloccurence)"
+ whitelists:
+ name: whitelists
+ Name: Whitelists
+ htmlname: "[whitelists](/write_configurations/whitelist/)"
+ Htmlname: "[Whitelists](/write_configurations/whitelist/)"
signal:
name: signal
Name: Signal
diff --git a/pkg/cwhub/hubMgmt.go b/pkg/cwhub/hubMgmt.go
index 6321e78a03d9cafb9c41a78cccda0dc9985242ed..62ba902c406db4735f3bf3b62e037cd8baa9a628 100644
--- a/pkg/cwhub/hubMgmt.go
+++ b/pkg/cwhub/hubMgmt.go
@@ -813,9 +813,6 @@ func HubStatus(itype string, name string, list_all bool) []map[string]string {
log.Errorf("type %s doesn't exist", itype)
return nil
}
- if list_all {
- log.Printf("only enabled ones")
- }
var mli []map[string]string
/*remember, you do it for the user :)*/
diff --git a/pkg/exprhelpers/expr_test.go b/pkg/exprhelpers/expr_test.go
index e62e051d0acd4d28dc55c0b2d8441e7d1a368c21..b44fbd919ce1bc510e00065dec5be3a4348b583e 100644
--- a/pkg/exprhelpers/expr_test.go
+++ b/pkg/exprhelpers/expr_test.go
@@ -5,6 +5,7 @@ import (
"testing"
"github.com/antonmedv/expr"
+ "github.com/stretchr/testify/require"
"gotest.tools/assert"
)
@@ -113,3 +114,22 @@ func TestFile(t *testing.T) {
assert.Equal(t, test.result, result)
}
}
+
+func TestIpInRange(t *testing.T) {
+ env := map[string]interface{}{
+ "ip": "192.168.0.1",
+ "ipRange": "192.168.0.0/24",
+ "IpInRange": IpInRange,
+ }
+ code := "IpInRange(ip, ipRange)"
+ log.Printf("Running filter : %s", code)
+
+ program, err := expr.Compile(code, expr.Env(env))
+ require.NoError(t, err)
+
+ output, err := expr.Run(program, env)
+ require.NoError(t, err)
+
+ require.Equal(t, true, output)
+
+}
diff --git a/pkg/exprhelpers/exprlib.go b/pkg/exprhelpers/exprlib.go
index e1ae0eef249ac42762f2184b6b91cfa6e12d0d84..b32cef42c9632a761fa8985f6bdcdd414739c194 100644
--- a/pkg/exprhelpers/exprlib.go
+++ b/pkg/exprhelpers/exprlib.go
@@ -3,6 +3,7 @@ package exprhelpers
import (
"bufio"
"fmt"
+ "net"
"os"
"path"
"regexp"
@@ -36,6 +37,7 @@ func GetExprEnv(ctx map[string]interface{}) map[string]interface{} {
"File": File,
"RegexpInFile": RegexpInFile,
"Upper": Upper,
+ "IpInRange": IpInRange,
}
for k, v := range ctx {
ExprLib[k] = v
@@ -50,6 +52,7 @@ func Init() error {
}
func FileInit(fileFolder string, filename string, fileType string) error {
+ log.Debugf("init (folder:%s) (file:%s) (type:%s)", fileFolder, filename, fileType)
filepath := path.Join(fileFolder, filename)
file, err := os.Open(filepath)
if err != nil {
@@ -65,6 +68,9 @@ func FileInit(fileFolder string, filename string, fileType string) error {
}
scanner := bufio.NewScanner(file)
for scanner.Scan() {
+ if strings.HasPrefix(scanner.Text(), "#") { // allow comments
+ continue
+ }
switch fileType {
case "regex", "regexp":
dataFileRegex[filename] = append(dataFileRegex[filename], regexp.MustCompile(scanner.Text()))
@@ -85,7 +91,7 @@ func File(filename string) []string {
if _, ok := dataFile[filename]; ok {
return dataFile[filename]
}
- log.Errorf("file '%s' not found for expr library", filename)
+ log.Errorf("file '%s' (type:string) not found in expr library", filename)
return []string{}
}
@@ -97,7 +103,27 @@ func RegexpInFile(data string, filename string) bool {
}
}
} else {
- log.Errorf("file '%s' not found for expr library", filename)
+ log.Errorf("file '%s' (type:regexp) not found in expr library", filename)
+ }
+ return false
+}
+
+func IpInRange(ip string, ipRange string) bool {
+ var err error
+ var ipParsed net.IP
+ var ipRangeParsed *net.IPNet
+
+ ipParsed = net.ParseIP(ip)
+ if ipParsed == nil {
+ log.Errorf("'%s' is not a valid IP", ip)
+ return false
+ }
+ if _, ipRangeParsed, err = net.ParseCIDR(ipRange); err != nil {
+ log.Errorf("'%s' is not a valid IP Range", ipRange)
+ return false
+ }
+ if ipRangeParsed.Contains(ipParsed) {
+ return true
}
return false
}
diff --git a/pkg/leakybucket/buckets_test.go b/pkg/leakybucket/buckets_test.go
index cc968cc7fe6e6a2d952aff146aec3ddb948ce2cc..b490c46345be6a5a2fd90d6c286f659592297ac7 100644
--- a/pkg/leakybucket/buckets_test.go
+++ b/pkg/leakybucket/buckets_test.go
@@ -10,6 +10,7 @@ import (
"testing"
"time"
+ "github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/davecgh/go-spew/spew"
@@ -25,6 +26,10 @@ type TestFile struct {
func TestBucket(t *testing.T) {
var envSetting = os.Getenv("TEST_ONLY")
+ err := exprhelpers.Init()
+ if err != nil {
+ log.Fatalf("exprhelpers init failed: %s", err)
+ }
if envSetting != "" {
if err := testOneBucket(t, envSetting); err != nil {
diff --git a/pkg/leakybucket/manager.go b/pkg/leakybucket/manager.go
index c8caa22e37b0ee838ba8400be9b8a784031df85d..f6ec8c8cab6be728b97bb164254cd0c03ca40387 100644
--- a/pkg/leakybucket/manager.go
+++ b/pkg/leakybucket/manager.go
@@ -112,10 +112,6 @@ func LoadBuckets(files []string, dataFolder string) ([]BucketFactory, chan types
)
var seed namegenerator.Generator = namegenerator.NewNameGenerator(time.Now().UTC().UnixNano())
- err := exprhelpers.Init()
- if err != nil {
- return nil, nil, err
- }
response = make(chan types.Event, 1)
for _, f := range files {
diff --git a/pkg/outputs/ouputs.go b/pkg/outputs/ouputs.go
index aa6c215968f05f4735c3f219913480923a8460b0..0d4953777eb5eb21c68fcd5c58a39b52ac49cb91 100644
--- a/pkg/outputs/ouputs.go
+++ b/pkg/outputs/ouputs.go
@@ -176,7 +176,7 @@ func (o *Output) ProcessOutput(sig types.SignalOccurence, profiles []types.Profi
return err
}
if warn != nil {
- logger.Infof("node warning : %s", warn)
+ logger.Debugf("node warning : %s", warn)
}
if ordr != nil {
bans, err := types.OrderToApplications(ordr)
diff --git a/pkg/parser/enrich_dns.go b/pkg/parser/enrich_dns.go
index 39a6e30794dcc2dbb05a054f832eacd1df4600db..86944a77432de3a37be3d194debf553055c1335c 100644
--- a/pkg/parser/enrich_dns.go
+++ b/pkg/parser/enrich_dns.go
@@ -18,7 +18,7 @@ func reverse_dns(field string, p *types.Event, ctx interface{}) (map[string]stri
}
rets, err := net.LookupAddr(field)
if err != nil {
- log.Infof("failed to resolve '%s'", field)
+ log.Debugf("failed to resolve '%s'", field)
return nil, nil
}
//When using the host C library resolver, at most one result will be returned. To bypass the host resolver, use a custom Resolver.
diff --git a/pkg/parser/node.go b/pkg/parser/node.go
index 13048872118b9fd97b5a0283db2f83061ec4b87f..157da54b79e1d5e77b51d458f0c136240d103893 100644
--- a/pkg/parser/node.go
+++ b/pkg/parser/node.go
@@ -137,14 +137,15 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
NodeState = true
clog.Debugf("eval(TRUE) '%s'", n.Filter)
} else {
- clog.Tracef("Node has not filter, enter")
+ clog.Debugf("Node has not filter, enter")
NodeState = true
}
if n.Name != "" {
NodesHits.With(prometheus.Labels{"source": p.Line.Src, "name": n.Name}).Inc()
}
- set := false
+ isWhitelisted := false
+ hasWhitelist := false
var src net.IP
/*overflow and log don't hold the source ip in the same field, should be changed */
/* perform whitelist checks for ips, cidr accordingly */
@@ -160,24 +161,28 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
if v.Equal(src) {
clog.Debugf("Event from [%s] is whitelisted by Ips !", src)
p.Whitelisted = true
- set = true
+ isWhitelisted = true
+ } else {
+ clog.Debugf("whitelist: %s is not eq [%s]", src, v)
}
+ hasWhitelist = true
}
for _, v := range n.Whitelist.B_Cidrs {
if v.Contains(src) {
clog.Debugf("Event from [%s] is whitelisted by Cidrs !", src)
p.Whitelisted = true
- set = true
+ isWhitelisted = true
} else {
clog.Debugf("whitelist: %s not in [%s]", src, v)
}
+ hasWhitelist = true
}
} else {
clog.Debugf("no ip in event, cidr/ip whitelists not checked")
}
/* run whitelist expression tests anyway */
- for _, e := range n.Whitelist.B_Exprs {
+ for eidx, e := range n.Whitelist.B_Exprs {
output, err := expr.Run(e, exprhelpers.GetExprEnv(map[string]interface{}{"evt": p}))
if err != nil {
clog.Warningf("failed to run whitelist expr : %v", err)
@@ -190,11 +195,14 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
if out {
clog.Debugf("Event is whitelisted by Expr !")
p.Whitelisted = true
- set = true
+ isWhitelisted = true
}
+ hasWhitelist = true
+ default:
+ log.Errorf("unexpected type %t (%v) while running '%s'", output, output, n.Whitelist.Exprs[eidx])
}
}
- if set {
+ if isWhitelisted {
p.WhiteListReason = n.Whitelist.Reason
/*huglily wipe the ban order if the event is whitelisted and it's an overflow */
if p.Type == types.OVFLW { /*don't do this at home kids */
@@ -202,6 +210,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
//Break this for now. Souldn't have been done this way, but that's not taht serious
/*only display logs when we discard ban to avoid spam*/
clog.Infof("Ban for %s whitelisted, reason [%s]", p.Overflow.Source.Ip.String(), n.Whitelist.Reason)
+ p.Overflow.Whitelisted = true
}
}
@@ -295,9 +304,9 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
if n.Name != "" {
NodesHitsOk.With(prometheus.Labels{"source": p.Line.Src, "name": n.Name}).Inc()
}
- if len(n.Statics) > 0 {
+ if hasWhitelist && isWhitelisted && len(n.Statics) > 0 || len(n.Statics) > 0 && !hasWhitelist {
clog.Debugf("+ Processing %d statics", len(n.Statics))
- // if all else is good, process node's statics
+ // if all else is good in whitelist, process node's statics
err := ProcessStatics(n.Statics, p, clog)
if err != nil {
clog.Fatalf("Failed to process statics : %v", err)
diff --git a/pkg/parser/parsing_test.go b/pkg/parser/parsing_test.go
index 06d4f518b98f146f08b38c2d7ab87ad90ad57032..0eeb0454f58d33e8fa45d8769a165935e6feba2f 100644
--- a/pkg/parser/parsing_test.go
+++ b/pkg/parser/parsing_test.go
@@ -10,6 +10,7 @@ import (
"strings"
"testing"
+ "github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/davecgh/go-spew/spew"
log "github.com/sirupsen/logrus"
@@ -139,6 +140,10 @@ func testOneParser(pctx *UnixParserCtx, dir string, b *testing.B) error {
func prepTests() (*UnixParserCtx, error) {
var pctx *UnixParserCtx
var p UnixParser
+ err := exprhelpers.Init()
+ if err != nil {
+ log.Fatalf("exprhelpers init failed: %s", err)
+ }
//Load enrichment
datadir := "../../data/"
diff --git a/pkg/parser/stage.go b/pkg/parser/stage.go
index 95bb66a317854a26d5157dfbd7b1aa7ee12d012c..f26594a955c0459b357efe123fb4393bbcf0f2b3 100644
--- a/pkg/parser/stage.go
+++ b/pkg/parser/stage.go
@@ -43,10 +43,6 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx) ([]Node, error) {
tmpstages := make(map[string]bool)
pctx.Stages = []string{}
- err := exprhelpers.Init()
- if err != nil {
- return nil, err
- }
for _, stageFile := range stageFiles {
if !strings.HasSuffix(stageFile.Filename, ".yaml") {
log.Warningf("skip non yaml : %s", stageFile.Filename)
diff --git a/pkg/parser/tests/whitelist-base/base-grok.yaml b/pkg/parser/tests/whitelist-base/base-grok.yaml
index 2db38dc4ec6df67b20b5b03038b1d728d0d03ae6..44cbd10354605fd4459570d4544f1d3cc0dfdd10 100644
--- a/pkg/parser/tests/whitelist-base/base-grok.yaml
+++ b/pkg/parser/tests/whitelist-base/base-grok.yaml
@@ -9,3 +9,6 @@ whitelist:
- "1.2.3.0/24"
expression:
- "'supertoken1234' == evt.Enriched.test_token"
+statics:
+ - meta: statics
+ value: success
diff --git a/pkg/parser/tests/whitelist-base/test.yaml b/pkg/parser/tests/whitelist-base/test.yaml
index 471e635f95323478609f7aabc6490aa1598e6644..4524e957ed2d360302c13ad4dce08d7205356ee3 100644
--- a/pkg/parser/tests/whitelist-base/test.yaml
+++ b/pkg/parser/tests/whitelist-base/test.yaml
@@ -3,41 +3,51 @@ lines:
- Meta:
test: test1
source_ip: 8.8.8.8
+ statics: toto
- Meta:
test: test2
source_ip: 1.2.3.4
+ statics: toto
- Meta:
test: test3
source_ip: 2.2.3.4
+ statics: toto
- Meta:
test: test4
source_ip: 8.8.8.9
+ statics: toto
- Enriched:
test_token: supertoken1234
Meta:
test: test5
+ statics: toto
#these are the results we expect from the parser
results:
- Whitelisted: true
Process: true
Meta:
test: test1
+ statics: success
- Whitelisted: true
Process: true
Meta:
test: test2
+ statics: success
- Whitelisted: false
Process: true
Meta:
test: test3
+ statics: toto
- Whitelisted: false
Process: true
Meta:
test: test4
+ statics: toto
- Whitelisted: true
Process: true
Meta:
test: test5
+ statics: success
diff --git a/pkg/sqlite/commit.go b/pkg/sqlite/commit.go
new file mode 100644
index 0000000000000000000000000000000000000000..01355b71804310b7dfce4e7489b50986971efacd
--- /dev/null
+++ b/pkg/sqlite/commit.go
@@ -0,0 +1,190 @@
+package sqlite
+
+import (
+ "fmt"
+ "sync/atomic"
+ "time"
+
+ "github.com/crowdsecurity/crowdsec/pkg/types"
+ "github.com/pkg/errors"
+ log "github.com/sirupsen/logrus"
+)
+
+func (c *Context) DeleteExpired() error {
+ //Delete the expired records
+ if c.flush {
+ retx := c.Db.Where(`strftime("%s", until) < strftime("%s", "now")`).Delete(types.BanApplication{})
+ if retx.RowsAffected > 0 {
+ log.Infof("Flushed %d expired entries from Ban Application", retx.RowsAffected)
+ }
+ }
+ return nil
+}
+
+func (c *Context) Flush() error {
+ c.lock.Lock()
+ defer c.lock.Unlock()
+
+ ret := c.tx.Commit()
+
+ if ret.Error != nil {
+ c.tx = c.Db.Begin()
+ return fmt.Errorf("failed to commit records : %v", ret.Error)
+ }
+ c.tx = c.Db.Begin()
+ c.lastCommit = time.Now()
+ return nil
+}
+
+func (c *Context) CleanUpRecordsByAge() error {
+ //let's fetch all expired records that are more than XX days olds
+ sos := []types.BanApplication{}
+
+ if c.maxDurationRetention == 0 {
+ return nil
+ }
+
+ //look for soft-deleted events that are OLDER than maxDurationRetention
+ ret := c.Db.Unscoped().Table("ban_applications").Where("deleted_at is not NULL").
+ Where(fmt.Sprintf("deleted_at > date('now','-%d minutes')", int(c.maxDurationRetention.Minutes()))).
+ Order("updated_at desc").Find(&sos)
+
+ if ret.Error != nil {
+ return errors.Wrap(ret.Error, "failed to get count of old records")
+ }
+
+ //no events elligible
+ if len(sos) == 0 || ret.RowsAffected == 0 {
+ log.Debugf("no event older than %s", c.maxDurationRetention.String())
+ return nil
+ }
+ //let's do it in a single transaction
+ delTx := c.Db.Unscoped().Begin()
+ delRecords := 0
+
+ for _, record := range sos {
+ copy := record
+ delTx.Unscoped().Table("signal_occurences").Where("ID = ?", copy.SignalOccurenceID).Delete(&types.SignalOccurence{})
+ delTx.Unscoped().Table("event_sequences").Where("signal_occurence_id = ?", copy.SignalOccurenceID).Delete(&types.EventSequence{})
+ delTx.Unscoped().Table("ban_applications").Delete(©)
+ //we need to delete associations : event_sequences, signal_occurences
+ delRecords++
+ }
+ ret = delTx.Unscoped().Commit()
+ if ret.Error != nil {
+ return errors.Wrap(ret.Error, "failed to delete records")
+ }
+ log.Printf("max_records_age: deleting %d events (max age:%s)", delRecords, c.maxDurationRetention)
+ return nil
+}
+
+func (c *Context) CleanUpRecordsByCount() error {
+ var count int
+
+ if c.maxEventRetention <= 0 {
+ return nil
+ }
+
+ ret := c.Db.Unscoped().Table("ban_applications").Order("updated_at desc").Count(&count)
+
+ if ret.Error != nil {
+ return errors.Wrap(ret.Error, "failed to get bans count")
+ }
+ if count < c.maxEventRetention {
+ log.Debugf("%d < %d, don't cleanup", count, c.maxEventRetention)
+ return nil
+ }
+
+ sos := []types.BanApplication{}
+ /*get soft deleted records oldest to youngest*/
+ records := c.Db.Unscoped().Table("ban_applications").Where("deleted_at is not NULL").Where(`strftime("%s", deleted_at) < strftime("%s", "now")`).Find(&sos)
+ if records.Error != nil {
+ return errors.Wrap(records.Error, "failed to list expired bans for flush")
+ }
+
+ //let's do it in a single transaction
+ delTx := c.Db.Unscoped().Begin()
+ delRecords := 0
+ for _, ld := range sos {
+ copy := ld
+ delTx.Unscoped().Table("signal_occurences").Where("ID = ?", copy.SignalOccurenceID).Delete(&types.SignalOccurence{})
+ delTx.Unscoped().Table("event_sequences").Where("signal_occurence_id = ?", copy.SignalOccurenceID).Delete(&types.EventSequence{})
+ delTx.Unscoped().Table("ban_applications").Delete(©)
+ //we need to delete associations : event_sequences, signal_occurences
+ delRecords++
+ //let's delete as well the associated event_sequence
+ if count-delRecords <= c.maxEventRetention {
+ break
+ }
+ }
+ if len(sos) > 0 {
+ //log.Printf("Deleting %d soft-deleted results out of %d total events (%d soft-deleted)", delRecords, count, len(sos))
+ log.Printf("max_records: deleting %d events. (%d soft-deleted)", delRecords, len(sos))
+ ret = delTx.Unscoped().Commit()
+ if ret.Error != nil {
+ return errors.Wrap(ret.Error, "failed to delete records")
+ }
+ } else {
+ log.Debugf("didn't find any record to clean")
+ }
+ return nil
+}
+
+func (c *Context) StartAutoCommit() error {
+ //TBD : we shouldn't start auto-commit if we are in cli mode ?
+ c.PusherTomb.Go(func() error {
+ c.autoCommit()
+ return nil
+ })
+ return nil
+}
+
+func (c *Context) autoCommit() {
+ log.Debugf("starting autocommit")
+ ticker := time.NewTicker(200 * time.Millisecond)
+ cleanUpTicker := time.NewTicker(1 * time.Minute)
+ expireTicker := time.NewTicker(1 * time.Second)
+ if !c.flush {
+ log.Debugf("flush is disabled")
+ }
+ for {
+ select {
+ case <-c.PusherTomb.Dying():
+ //we need to shutdown
+ log.Infof("sqlite routine shutdown")
+ if err := c.Flush(); err != nil {
+ log.Errorf("error while flushing records: %s", err)
+ }
+ if ret := c.tx.Commit(); ret.Error != nil {
+ log.Errorf("failed to commit records : %v", ret.Error)
+ }
+ if err := c.tx.Close(); err != nil {
+ log.Errorf("error while closing tx : %s", err)
+ }
+ if err := c.Db.Close(); err != nil {
+ log.Errorf("error while closing db : %s", err)
+ }
+ return
+ case <-expireTicker.C:
+ if err := c.DeleteExpired(); err != nil {
+ log.Errorf("Error while deleting expired records: %s", err)
+ }
+ case <-ticker.C:
+ if atomic.LoadInt32(&c.count) != 0 &&
+ (atomic.LoadInt32(&c.count)%100 == 0 || time.Since(c.lastCommit) >= 500*time.Millisecond) {
+ if err := c.Flush(); err != nil {
+ log.Errorf("failed to flush : %s", err)
+ }
+
+ }
+ case <-cleanUpTicker.C:
+ if err := c.CleanUpRecordsByCount(); err != nil {
+ log.Errorf("error in max records cleanup : %s", err)
+ }
+ if err := c.CleanUpRecordsByAge(); err != nil {
+ log.Errorf("error in old records cleanup : %s", err)
+
+ }
+ }
+ }
+}
diff --git a/pkg/sqlite/sqlite.go b/pkg/sqlite/sqlite.go
new file mode 100644
index 0000000000000000000000000000000000000000..1527ee01f1f55141a344902c94c8fc58e14f252f
--- /dev/null
+++ b/pkg/sqlite/sqlite.go
@@ -0,0 +1,88 @@
+package sqlite
+
+import (
+ "fmt"
+ "strconv"
+ "sync"
+ "time"
+
+ "github.com/crowdsecurity/crowdsec/pkg/types"
+ "github.com/pkg/errors"
+ log "github.com/sirupsen/logrus"
+
+ "github.com/jinzhu/gorm"
+ _ "github.com/jinzhu/gorm/dialects/sqlite"
+ _ "github.com/mattn/go-sqlite3"
+ "gopkg.in/tomb.v2"
+)
+
+type Context struct {
+ Db *gorm.DB //Pointer to sqlite db
+ tx *gorm.DB //Pointer to current transaction (flushed on a regular basis)
+ lastCommit time.Time
+ flush bool
+ count int32
+ lock sync.Mutex //booboo
+ PusherTomb tomb.Tomb
+ //to manage auto cleanup : max number of records *or* oldest
+ maxEventRetention int
+ maxDurationRetention time.Duration
+}
+
+func NewSQLite(cfg map[string]string) (*Context, error) {
+ var err error
+ c := &Context{}
+
+ if v, ok := cfg["max_records"]; ok {
+ c.maxEventRetention, err = strconv.Atoi(v)
+ if err != nil {
+ log.Errorf("Ignoring invalid max_records '%s' : %s", v, err)
+ }
+ }
+ if v, ok := cfg["max_records_age"]; ok {
+ c.maxDurationRetention, err = time.ParseDuration(v)
+ if err != nil {
+ log.Errorf("Ignoring invalid duration '%s' : %s", v, err)
+ }
+ }
+ if _, ok := cfg["db_path"]; !ok {
+ return nil, fmt.Errorf("please specify a 'db_path' to SQLite db in the configuration")
+ }
+
+ if cfg["db_path"] == "" {
+ return nil, fmt.Errorf("please specify a 'db_path' to SQLite db in the configuration")
+ }
+ log.Debugf("Starting SQLite backend, path:%s", cfg["db_path"])
+
+ c.Db, err = gorm.Open("sqlite3", cfg["db_path"]+"?_busy_timeout=1000")
+ if err != nil {
+ return nil, fmt.Errorf("failed to open %s : %s", cfg["db_path"], err)
+ }
+
+ if val, ok := cfg["debug"]; ok && val == "true" {
+ log.Infof("Enabling debug for sqlite")
+ c.Db.LogMode(true)
+ }
+
+ c.flush, err = strconv.ParseBool(cfg["flush"])
+ if err != nil {
+ return nil, errors.Wrap(err, "Unable to parse 'flush' flag")
+ }
+ // Migrate the schema
+ c.Db.AutoMigrate(&types.EventSequence{}, &types.SignalOccurence{}, &types.BanApplication{})
+ c.Db.Model(&types.SignalOccurence{}).Related(&types.EventSequence{})
+ c.Db.Model(&types.SignalOccurence{}).Related(&types.BanApplication{})
+ c.tx = c.Db.Begin()
+ c.lastCommit = time.Now()
+ ret := c.tx.Commit()
+
+ if ret.Error != nil {
+ return nil, fmt.Errorf("failed to commit records : %v", ret.Error)
+
+ }
+ c.tx = c.Db.Begin()
+ if c.tx == nil {
+ return nil, fmt.Errorf("failed to begin sqlite transac : %s", err)
+ }
+ return c, nil
+}
diff --git a/pkg/types/signal_occurence.go b/pkg/types/signal_occurence.go
index 2e9f712f64532c9a9c1224656561b3a14916e97b..91f0c467a45ef03ce7184d8ff64890e03fc5f966 100644
--- a/pkg/types/signal_occurence.go
+++ b/pkg/types/signal_occurence.go
@@ -35,9 +35,9 @@ type SignalOccurence struct {
Dest_ip string `json:"dst_ip,omitempty"` //for now just the destination IP
//Policy string `json:"policy,omitempty"` //for now we forward it as well :)
//bucket info
- Capacity int `json:"capacity,omitempty"`
- Leak_speed time.Duration `json:"leak_speed,omitempty"`
-
- Reprocess bool //Reprocess, when true, will make the overflow being processed again as a fresh log would
- Labels map[string]string `gorm:"-"`
+ Capacity int `json:"capacity,omitempty"`
+ Leak_speed time.Duration `json:"leak_speed,omitempty"`
+ Whitelisted bool `gorm:"-"`
+ Reprocess bool //Reprocess, when true, will make the overflow being processed again as a fresh log would
+ Labels map[string]string `gorm:"-"`
}
diff --git a/wizard.sh b/wizard.sh
index 72cd8cd0dc9756f818b9323cad64a3ba1f61a4ea..e814ace9d6ce19e28eb6ddf92eee78bc44b55105 100755
--- a/wizard.sh
+++ b/wizard.sh
@@ -315,6 +315,8 @@ update_full() {
log_info "Backing up existing configuration"
${CSCLI_BIN_INSTALLED} backup save ${BACKUP_DIR}
+ log_info "Saving default database content"
+ cp /var/lib/crowdsec/data/crowdsec.db ${BACKUP_DIR}/crowdsec.db
log_info "Cleanup existing crowdsec configuration"
uninstall_crowdsec
log_info "Installing crowdsec"
@@ -322,6 +324,8 @@ update_full() {
log_info "Restoring configuration"
${CSCLI_BIN_INSTALLED} update
${CSCLI_BIN_INSTALLED} backup restore ${BACKUP_DIR}
+ log_info "Restoring saved database"
+ cp ${BACKUP_DIR}/crowdsec.db /var/lib/crowdsec/data/crowdsec.db
log_info "Finished, restarting"
systemctl restart crowdsec || log_err "Failed to restart crowdsec"
}