add json support via expr helpers

go.mod

@@ -5,6 +5,7 @@ go 1.13
 require (
 	github.com/Microsoft/go-winio v0.4.14 // indirect
 	github.com/antonmedv/expr v1.8.2
+	github.com/buger/jsonparser v1.0.0
 	github.com/containerd/containerd v1.3.4 // indirect
 	github.com/davecgh/go-spew v1.1.1
 	github.com/denisbrodbeck/machineid v1.0.1

go.sum

@@ -18,6 +18,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/buger/jsonparser v1.0.0 h1:etJTGF5ESxjI0Ic2UaLQs2LQQpa8G9ykQScukbh4L8A=
+github.com/buger/jsonparser v1.0.0/go.mod h1:tgcrVJ81GPSF0mz+0nu1Xaz0fazGPrmmJfJtxjbHhUQ=
 github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY=

pkg/exprhelpers/exprlib.go

@@ -17,7 +17,7 @@ func Atof(x string) float64 {
 
 func GetExprEnv(ctx map[string]interface{}) map[string]interface{} {
 
-	var ExprLib = map[string]interface{}{"Atof": Atof}
+	var ExprLib = map[string]interface{}{"Atof": Atof, "JsonExtract": JsonExtract, "JsonExtractLib": JsonExtractLib}
 	for k, v := range ctx {
 		ExprLib[k] = v
 	}

pkg/exprhelpers/jsonextract.go

@@ -0,0 +1,32 @@
+package exprhelpers
+
+import (
+	"strings"
+
+	"github.com/buger/jsonparser"
+
+	log "github.com/sirupsen/logrus"
+)
+
+func JsonExtractLib(jsblob string, target ...string) string {
+	value, dataType, _, err := jsonparser.Get(
+		jsonparser.StringToBytes(jsblob),
+		target...,
+	)
+
+	if err != nil {
+		log.Errorf("jsonExtractLib : %s", err)
+		return ""
+	}
+	if dataType == jsonparser.NotExist {
+		log.Debugf("%+v doesn't exist", target)
+		return ""
+	}
+	strvalue := string(value)
+	return strvalue
+}
+
+func JsonExtract(jsblob string, target string) string {
+	fullpath := strings.Split(target, ".")
+	return JsonExtractLib(jsblob, fullpath...)
+}

pkg/parser/tests/base-json-extract/base-grok.yaml

@@ -0,0 +1,14 @@
+filter: "evt.Line.Labels.type == 'json-1'"
+debug: true
+onsuccess: next_stage
+name: tests/base-json-extract
+statics:
+  - parsed: message
+    expression: JsonExtract(evt.Line.Raw, "log")
+  - meta: other_field
+    expression: JsonExtract(evt.Line.Raw, "testfield")
+  - meta: program
+    expression: evt.Line.Labels.progrname
+
+    
+

pkg/parser/tests/base-json-extract/base-grok2.yaml

@@ -0,0 +1,14 @@
+filter: "evt.Meta.program == 'my_test_prog'"
+debug: true
+onsuccess: next_stage
+name: tests/base-grok
+pattern_syntax:
+  MYCAP: ".*"
+nodes:
+  - grok:
+      pattern: ^xxheader %{MYCAP:extracted_value} trailing stuff$
+      apply_on: message
+statics:
+  - meta: log_type
+    value: parsed_testlog
+

pkg/parser/tests/base-json-extract/parsers.yaml

@@ -0,0 +1,4 @@
+ - filename: {{.TestDirectory}}/base-grok.yaml
+   stage: s00-raw
+ - filename: {{.TestDirectory}}/base-grok2.yaml
+   stage: s01-parse

pkg/parser/tests/base-json-extract/test.yaml

@@ -0,0 +1,16 @@
+#these are the events we input into parser
+lines:
+  - Line:
+      Labels:
+        type: json-1
+        progrname: my_test_prog
+      Raw: '{"testfield": "some stuff", "log": "xxheader VALUE1 trailing stuff"}'
+results:
+  - Meta:
+      other_field: some stuff
+      program: my_test_prog
+    Parsed:
+      message: xxheader VALUE1 trailing stuff
+      extracted_value: VALUE1
+    Process: true
+