Merge branch 'master' of github.com:crowdsecurity/crowdsec

docs/index.md

@@ -27,6 +27,14 @@ Besides detecting and stopping attacks in real time based on your logs, it allow
 
 ![Architecture](assets/images/crowdsec_architecture.png)
 
+
+## Core concepts
+
+{{crowdsec.name}} relies on {{parsers.htmlname}} to normalize and enrich logs, and {{scenarios.htmlname}} to detect attacks, often bundled together in {{collections.htmlname}} to form a coherent configuration set. For example the collection [`crowdsecurity/nginx`](https://hub.crowdsec.net/author/crowdsecurity/collections/nginx) contains all the necessary parsers and scenarios to deal with nginx logs and the common attacks that can be seen on http servers.
+
+All of those are represented as YAML files, that can be found, shared and kept up-to-date thanks to the {{hub.htmlname}}, or [easily hand-crafted](/write_configurations/scenarios/) to address specific needs.
+
+
 ## Moving forward
 
 To learn more about {{crowdsec.name}} and give it a try, please see :

pkg/parser/node.go

@@ -144,7 +144,8 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
 	if n.Name != "" {
 		NodesHits.With(prometheus.Labels{"source": p.Line.Src, "name": n.Name}).Inc()
 	}
-	set := false
+	isWhitelisted := false
+	hasWhitelist := false
 	var src net.IP
 	/*overflow and log don't hold the source ip in the same field, should be changed */
 	/* perform whitelist checks for ips, cidr accordingly */
@@ -160,18 +161,22 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
 			if v.Equal(src) {
 				clog.Debugf("Event from [%s] is whitelisted by Ips !", src)
 				p.Whitelisted = true
-				set = true
+				isWhitelisted = true
+			} else {
+				clog.Debugf("whitelist: %s is not eq [%s]", src, v)
 			}
+			hasWhitelist = true
 		}
 
 		for _, v := range n.Whitelist.B_Cidrs {
 			if v.Contains(src) {
 				clog.Debugf("Event from [%s] is whitelisted by Cidrs !", src)
 				p.Whitelisted = true
-				set = true
+				isWhitelisted = true
 			} else {
 				clog.Debugf("whitelist: %s not in [%s]", src, v)
 			}
+			hasWhitelist = true
 		}
 	} else {
 		clog.Debugf("no ip in event, cidr/ip whitelists not checked")
@@ -190,13 +195,14 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
 			if out {
 				clog.Debugf("Event is whitelisted by Expr !")
 				p.Whitelisted = true
-				set = true
+				isWhitelisted = true
 			}
+			hasWhitelist = true
 		default:
 			log.Errorf("unexpected type %t (%v) while running '%s'", output, output, n.Whitelist.Exprs[eidx])
 		}
 	}
-	if set {
+	if isWhitelisted {
 		p.WhiteListReason = n.Whitelist.Reason
 		/*huglily wipe the ban order if the event is whitelisted and it's an overflow */
 		if p.Type == types.OVFLW { /*don't do this at home kids */
@@ -298,9 +304,9 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx) (bool, error) {
 	if n.Name != "" {
 		NodesHitsOk.With(prometheus.Labels{"source": p.Line.Src, "name": n.Name}).Inc()
 	}
-	if len(n.Statics) > 0 {
+	if hasWhitelist && isWhitelisted && len(n.Statics) > 0 || len(n.Statics) > 0 && !hasWhitelist {
 		clog.Debugf("+ Processing %d statics", len(n.Statics))
-		// if all else is good, process node's statics
+		// if all else is good in whitelist, process node's statics
 		err := ProcessStatics(n.Statics, p, clog)
 		if err != nil {
 			clog.Fatalf("Failed to process statics : %v", err)

pkg/parser/tests/whitelist-base/base-grok.yaml

@@ -9,3 +9,6 @@ whitelist:
     - "1.2.3.0/24"
   expression:
     - "'supertoken1234' == evt.Enriched.test_token"
+statics:
+  - meta: statics
+    value: success

pkg/parser/tests/whitelist-base/test.yaml

@@ -3,41 +3,51 @@ lines:
   - Meta:
       test: test1
       source_ip: 8.8.8.8
+      statics: toto
   - Meta:
       test: test2
       source_ip: 1.2.3.4
+      statics: toto
   - Meta:
       test: test3
       source_ip: 2.2.3.4
+      statics: toto
   - Meta:
       test: test4
       source_ip: 8.8.8.9
+      statics: toto
   - Enriched:
       test_token: supertoken1234
     Meta:
       test: test5
+      statics: toto
 #these are the results we expect from the parser
 results:
   - Whitelisted: true
     Process: true
     Meta:
       test: test1
+      statics: success
   - Whitelisted: true
     Process: true 
     Meta:
       test: test2
+      statics: success
   - Whitelisted: false
     Process: true
     Meta:
       test: test3
+      statics: toto
   - Whitelisted: false
     Process: true
     Meta:
       test: test4
+      statics: toto
   - Whitelisted: true
     Process: true
     Meta:
       test: test5
+      statics: success