beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

use expr.Function for custom functions instead of passing them in the env (#2133)

Browse source
This commit is contained in:
blotus 2023-03-28 10:49:01 +02:00 committed by GitHub
parent 169b844212
commit 1095f6c875
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
30 changed files with 1162 additions and 464 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
cmd/crowdsec-cli/config_show.go
View file

@ -7,11 +7,12 @@ import (
"text/template" "text/template"
"github.com/antonmedv/expr" "github.com/antonmedv/expr"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra" "github.com/spf13/cobra"
"gopkg.in/yaml.v2" "gopkg.in/yaml.v2"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
) )
func showConfigKey(key string) error { func showConfigKey(key string) error {
@ -19,7 +20,10 @@ func showConfigKey(key string) error {
Config *csconfig.Config Config *csconfig.Config
} }
program, err := expr.Compile(key, expr.Env(Env{})) opts := []expr.Option{}
opts = append(opts, exprhelpers.GetExprOptions(map[string]interface{}{})...)
opts = append(opts, expr.Env(Env{}))
program, err := expr.Compile(key, opts...)
if err != nil { if err != nil {
return err return err
} }
@ -50,7 +54,6 @@ func showConfigKey(key string) error {
return nil return nil
} }
var configShowTemplate = `Global: var configShowTemplate = `Global:
{{- if .ConfigPaths }} {{- if .ConfigPaths }}
@ -172,7 +175,6 @@ Central API:
{{- end }} {{- end }}
` `
func runConfigShow(cmd *cobra.Command, args []string) error { func runConfigShow(cmd *cobra.Command, args []string) error {
flags := cmd.Flags() flags := cmd.Flags()
@ -218,7 +220,6 @@ func runConfigShow(cmd *cobra.Command, args []string) error {
return nil return nil
} }
func NewConfigShowCmd() *cobra.Command { func NewConfigShowCmd() *cobra.Command {
cmdConfigShow := &cobra.Command{ cmdConfigShow := &cobra.Command{
Use: "show", Use: "show",

2
go.mod
View file

@ -7,7 +7,7 @@ require (
github.com/AlecAivazis/survey/v2 v2.2.7 github.com/AlecAivazis/survey/v2 v2.2.7
github.com/Microsoft/go-winio v0.5.2 // indirect github.com/Microsoft/go-winio v0.5.2 // indirect
github.com/alexliesenfeld/health v0.5.1 github.com/alexliesenfeld/health v0.5.1
github.com/antonmedv/expr v1.12.2 github.com/antonmedv/expr v1.12.5
github.com/appleboy/gin-jwt/v2 v2.8.0 github.com/appleboy/gin-jwt/v2 v2.8.0
github.com/aws/aws-sdk-go v1.42.25 github.com/aws/aws-sdk-go v1.42.25
github.com/buger/jsonparser v1.1.1 github.com/buger/jsonparser v1.1.1

4
go.sum
View file

@ -90,6 +90,10 @@ github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo
github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
github.com/antonmedv/expr v1.12.2 h1:nlRcu4uHI6oSKCf6GHJTcT7hIf7dFAjgfvG0MWb7Cu0= github.com/antonmedv/expr v1.12.2 h1:nlRcu4uHI6oSKCf6GHJTcT7hIf7dFAjgfvG0MWb7Cu0=
github.com/antonmedv/expr v1.12.2/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU= github.com/antonmedv/expr v1.12.2/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU=
github.com/antonmedv/expr v1.12.4 h1:YRkeF7r0cejMS47bDYe3Jyes7L9t1AhpunC+Duq+R9k=
github.com/antonmedv/expr v1.12.4/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU=
github.com/antonmedv/expr v1.12.5 h1:Fq4okale9swwL3OeLLs9WD9H6GbgBLJyN/NUHRv+n0E=
github.com/antonmedv/expr v1.12.5/go.mod h1:FPC8iWArxls7axbVLsW+kpg1mz29A1b2M6jt+hZfDkU=
github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo= github.com/apparentlymart/go-textseg/v13 v13.0.0/go.mod h1:ZK2fH7c4NqDTLtiYLvIkEghdlcqw7yxLeM89kiTRPUo=
github.com/appleboy/gin-jwt/v2 v2.8.0 h1:Glo7cb9eBR+hj8Y7WzgfkOlqCaNLjP+RV4dNO3fpdps= github.com/appleboy/gin-jwt/v2 v2.8.0 h1:Glo7cb9eBR+hj8Y7WzgfkOlqCaNLjP+RV4dNO3fpdps=

4
pkg/acquisition/modules/wineventlog/wineventlog_test.go
View file

@ -214,8 +214,8 @@ event_ids:
} }
t.Fatalf("timeout") t.Fatalf("timeout")
case e := <-c: case e := <-c:
line, _ := exprhelpers.XMLGetNodeValue(e.Line.Raw, "/Event/EventData[1]/Data")
linesRead = append(linesRead, exprhelpers.XMLGetNodeValue(e.Line.Raw, "/Event/EventData[1]/Data")) linesRead = append(linesRead, line.(string))
if len(linesRead) == len(lines) { if len(linesRead) == len(lines) {
break READLOOP break READLOOP
} }

6
pkg/alertcontext/alertcontext.go
View file

@ -30,7 +30,7 @@ type Context struct {
func ValidateContextExpr(key string, expressions []string) error { func ValidateContextExpr(key string, expressions []string) error {
for _, expression := range expressions { for _, expression := range expressions {
_, err := expr.Compile(expression, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))) _, err := expr.Compile(expression, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
if err != nil { if err != nil {
return fmt.Errorf("compilation of '%s' failed: %v", expression, err) return fmt.Errorf("compilation of '%s' failed: %v", expression, err)
} }
@ -63,7 +63,7 @@ func NewAlertContext(contextToSend map[string][]string, valueLength int) error {
for key, values := range contextToSend { for key, values := range contextToSend {
alertContext.ContextToSendCompiled[key] = make([]*vm.Program, 0) alertContext.ContextToSendCompiled[key] = make([]*vm.Program, 0)
for _, value := range values { for _, value := range values {
valueCompiled, err := expr.Compile(value, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))) valueCompiled, err := expr.Compile(value, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
if err != nil { if err != nil {
return fmt.Errorf("compilation of '%s' context value failed: %v", value, err) return fmt.Errorf("compilation of '%s' context value failed: %v", value, err)
} }
@ -117,7 +117,7 @@ func EventToContext(events []types.Event) (models.Meta, []error) {
} }
for _, value := range values { for _, value := range values {
var val string var val string
output, err := expr.Run(value, exprhelpers.GetExprEnv(map[string]interface{}{"evt": evt})) output, err := expr.Run(value, map[string]interface{}{"evt": evt})
if err != nil { if err != nil {
errors = append(errors, fmt.Errorf("failed to get value for %s : %v", key, err)) errors = append(errors, fmt.Errorf("failed to get value for %s : %v", key, err))
continue continue

15
pkg/csprofiles/csprofiles.go
View file

@ -53,12 +53,13 @@ func NewProfile(profilesCfg []*csconfig.ProfileCfg) ([]*Runtime, error) {
return []*Runtime{}, errors.Wrapf(err, "invalid 'on_failure' for '%s' : %s", profile.Name, runtime.Cfg.OnFailure) return []*Runtime{}, errors.Wrapf(err, "invalid 'on_failure' for '%s' : %s", profile.Name, runtime.Cfg.OnFailure)
} }
for fIdx, filter := range profile.Filters { for fIdx, filter := range profile.Filters {
if runtimeFilter, err = expr.Compile(filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"Alert": &models.Alert{}}))); err != nil {
if runtimeFilter, err = expr.Compile(filter, exprhelpers.GetExprOptions(map[string]interface{}{"Alert": &models.Alert{}})...); err != nil {
return []*Runtime{}, errors.Wrapf(err, "error compiling filter of '%s'", profile.Name) return []*Runtime{}, errors.Wrapf(err, "error compiling filter of '%s'", profile.Name)
} }
runtime.RuntimeFilters[fIdx] = runtimeFilter runtime.RuntimeFilters[fIdx] = runtimeFilter
if profile.Debug != nil && *profile.Debug { if profile.Debug != nil && *profile.Debug {
if debugFilter, err = exprhelpers.NewDebugger(filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"Alert": &models.Alert{}}))); err != nil { if debugFilter, err = exprhelpers.NewDebugger(filter, exprhelpers.GetExprOptions(map[string]interface{}{"Alert": &models.Alert{}})...); err != nil {
log.Debugf("Error compiling debug filter of %s : %s", profile.Name, err) log.Debugf("Error compiling debug filter of %s : %s", profile.Name, err)
// Don't fail if we can't compile the filter - for now // Don't fail if we can't compile the filter - for now
// return errors.Wrapf(err, "Error compiling debug filter of %s", profile.Name) // return errors.Wrapf(err, "Error compiling debug filter of %s", profile.Name)
@ -69,13 +70,13 @@ func NewProfile(profilesCfg []*csconfig.ProfileCfg) ([]*Runtime, error) {
} }
if profile.DurationExpr != "" { if profile.DurationExpr != "" {
if runtimeDurationExpr, err = expr.Compile(profile.DurationExpr, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"Alert": &models.Alert{}}))); err != nil { if runtimeDurationExpr, err = expr.Compile(profile.DurationExpr, exprhelpers.GetExprOptions(map[string]interface{}{"Alert": &models.Alert{}})...); err != nil {
return []*Runtime{}, errors.Wrapf(err, "error compiling duration_expr of %s", profile.Name) return []*Runtime{}, errors.Wrapf(err, "error compiling duration_expr of %s", profile.Name)
} }
runtime.RuntimeDurationExpr = runtimeDurationExpr runtime.RuntimeDurationExpr = runtimeDurationExpr
if profile.Debug != nil && *profile.Debug { if profile.Debug != nil && *profile.Debug {
if debugDurationExpr, err = exprhelpers.NewDebugger(profile.DurationExpr, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"Alert": &models.Alert{}}))); err != nil { if debugDurationExpr, err = exprhelpers.NewDebugger(profile.DurationExpr, exprhelpers.GetExprOptions(map[string]interface{}{"Alert": &models.Alert{}})...); err != nil {
log.Debugf("Error compiling debug duration_expr of %s : %s", profile.Name, err) log.Debugf("Error compiling debug duration_expr of %s : %s", profile.Name, err)
} }
runtime.DebugDurationExpr = debugDurationExpr runtime.DebugDurationExpr = debugDurationExpr
@ -120,7 +121,7 @@ func (Profile *Runtime) GenerateDecisionFromProfile(Alert *models.Alert) ([]*mod
/*some fields are populated from the reference object : duration, scope, type*/ /*some fields are populated from the reference object : duration, scope, type*/
decision.Duration = new(string) decision.Duration = new(string)
if Profile.Cfg.DurationExpr != "" && Profile.RuntimeDurationExpr != nil { if Profile.Cfg.DurationExpr != "" && Profile.RuntimeDurationExpr != nil {
duration, err := expr.Run(Profile.RuntimeDurationExpr, exprhelpers.GetExprEnv(map[string]interface{}{"Alert": Alert})) duration, err := expr.Run(Profile.RuntimeDurationExpr, map[string]interface{}{"Alert": Alert})
if err != nil { if err != nil {
Profile.Logger.Warningf("Failed to run duration_expr : %v", err) Profile.Logger.Warningf("Failed to run duration_expr : %v", err)
*decision.Duration = *refDecision.Duration *decision.Duration = *refDecision.Duration
@ -164,7 +165,7 @@ func (Profile *Runtime) EvaluateProfile(Alert *models.Alert) ([]*models.Decision
matched := false matched := false
for eIdx, expression := range Profile.RuntimeFilters { for eIdx, expression := range Profile.RuntimeFilters {
output, err := expr.Run(expression, exprhelpers.GetExprEnv(map[string]interface{}{"Alert": Alert})) output, err := expr.Run(expression, map[string]interface{}{"Alert": Alert})
if err != nil { if err != nil {
Profile.Logger.Warningf("failed to run whitelist expr : %v", err) Profile.Logger.Warningf("failed to run whitelist expr : %v", err)
return nil, matched, errors.Wrapf(err, "while running expression %s", Profile.Cfg.Filters[eIdx]) return nil, matched, errors.Wrapf(err, "while running expression %s", Profile.Cfg.Filters[eIdx])
@ -172,7 +173,7 @@ func (Profile *Runtime) EvaluateProfile(Alert *models.Alert) ([]*models.Decision
switch out := output.(type) { switch out := output.(type) {
case bool: case bool:
if Profile.Cfg.Debug != nil && *Profile.Cfg.Debug { if Profile.Cfg.Debug != nil && *Profile.Cfg.Debug {
Profile.DebugFilters[eIdx].Run(Profile.Logger, out, exprhelpers.GetExprEnv(map[string]interface{}{"Alert": Alert})) Profile.DebugFilters[eIdx].Run(Profile.Logger, out, map[string]interface{}{"Alert": Alert})
} }
if out { if out {
matched = true matched = true

4
pkg/csprofiles/csprofiles_test.go
View file

@ -6,6 +6,7 @@ import (
"testing" "testing"
"github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/models"
"gotest.tools/v3/assert" "gotest.tools/v3/assert"
) )
@ -104,6 +105,9 @@ func TestEvaluateProfile(t *testing.T) {
profileCfg *csconfig.ProfileCfg profileCfg *csconfig.ProfileCfg
Alert *models.Alert Alert *models.Alert
} }
exprhelpers.Init(nil)
tests := []struct { tests := []struct {
name string name string
args args args args

4
pkg/exprhelpers/crowdsec_cti.go
View file

@ -73,7 +73,9 @@ func CrowdsecCTIInitCache(size int, ttl time.Duration) {
CacheExpiration = ttl CacheExpiration = ttl
} }
func CrowdsecCTI(ip string) (*cticlient.SmokeItem, error) { // func CrowdsecCTI(ip string) (*cticlient.SmokeItem, error) {
func CrowdsecCTI(params ...any) (any, error) {
ip := params[0].(string)
if !CTIApiEnabled { if !CTIApiEnabled {
ctiClient.Logger.Warningf("Crowdsec CTI API is disabled, please check your configuration") ctiClient.Logger.Warningf("Crowdsec CTI API is disabled, please check your configuration")
return &cticlient.SmokeItem{}, cticlient.ErrDisabled return &cticlient.SmokeItem{}, cticlient.ErrDisabled

11
pkg/exprhelpers/crowdsec_cti_test.go
View file

@ -157,13 +157,16 @@ func TestCache(t *testing.T) {
})) }))
item, err := CrowdsecCTI("1.2.3.4") item, err := CrowdsecCTI("1.2.3.4")
assert.Equal(t, "1.2.3.4", item.Ip) ctiResp := item.(*cticlient.SmokeItem)
assert.Equal(t, "1.2.3.4", ctiResp.Ip)
assert.Equal(t, CTIApiEnabled, true) assert.Equal(t, CTIApiEnabled, true)
assert.Equal(t, CTICache.Len(true), 1) assert.Equal(t, CTICache.Len(true), 1)
assert.Equal(t, err, nil) assert.Equal(t, err, nil)
item, err = CrowdsecCTI("1.2.3.4") item, err = CrowdsecCTI("1.2.3.4")
assert.Equal(t, "1.2.3.4", item.Ip) ctiResp = item.(*cticlient.SmokeItem)
assert.Equal(t, "1.2.3.4", ctiResp.Ip)
assert.Equal(t, CTIApiEnabled, true) assert.Equal(t, CTIApiEnabled, true)
assert.Equal(t, CTICache.Len(true), 1) assert.Equal(t, CTICache.Len(true), 1)
assert.Equal(t, err, nil) assert.Equal(t, err, nil)
@ -173,7 +176,9 @@ func TestCache(t *testing.T) {
assert.Equal(t, CTICache.Len(true), 0) assert.Equal(t, CTICache.Len(true), 0)
item, err = CrowdsecCTI("1.2.3.4") item, err = CrowdsecCTI("1.2.3.4")
assert.Equal(t, "1.2.3.4", item.Ip) ctiResp = item.(*cticlient.SmokeItem)
assert.Equal(t, "1.2.3.4", ctiResp.Ip)
assert.Equal(t, CTIApiEnabled, true) assert.Equal(t, CTIApiEnabled, true)
assert.Equal(t, CTICache.Len(true), 1) assert.Equal(t, CTICache.Len(true), 1)
assert.Equal(t, err, nil) assert.Equal(t, err, nil)

384
pkg/exprhelpers/expr_lib.go Normal file
View file

@ -0,0 +1,384 @@
package exprhelpers
import (
"time"
"github.com/crowdsecurity/crowdsec/pkg/cticlient"
)
type exprCustomFunc struct {
name string
function func(params ...any) (any, error)
signature []interface{}
}
var exprFuncs = []exprCustomFunc{
{
name: "CrowdsecCTI",
function: CrowdsecCTI,
signature: []interface{}{
new(func(string) (*cticlient.SmokeItem, error)),
},
},
{
name: "Distance",
function: Distance,
signature: []interface{}{
new(func(string, string, string, string) (float64, error)),
},
},
{
name: "GetFromStash",
function: GetFromStash,
signature: []interface{}{
new(func(string, string) (string, error)),
},
},
{
name: "Atof",
function: Atof,
signature: []interface{}{
new(func(string) float64),
},
},
{
name: "JsonExtract",
function: JsonExtract,
signature: []interface{}{
new(func(string, string) string),
},
},
{
name: "JsonExtractUnescape",
function: JsonExtractUnescape,
signature: []interface{}{
new(func(string, ...string) string),
},
},
{
name: "JsonExtractLib",
function: JsonExtractLib,
signature: []interface{}{
new(func(string, ...string) string),
},
},
{
name: "JsonExtractSlice",
function: JsonExtractSlice,
signature: []interface{}{
new(func(string, string) []interface{}),
},
},
{
name: "JsonExtractObject",
function: JsonExtractObject,
signature: []interface{}{
new(func(string, string) map[string]interface{}),
},
},
{
name: "ToJsonString",
function: ToJson,
signature: []interface{}{
new(func(interface{}) string),
},
},
{
name: "File",
function: File,
signature: []interface{}{
new(func(string) []string),
},
},
{
name: "RegexpInFile",
function: RegexpInFile,
signature: []interface{}{
new(func(string, string) bool),
},
},
{
name: "Upper",
function: Upper,
signature: []interface{}{
new(func(string) string),
},
},
{
name: "Lower",
function: Lower,
signature: []interface{}{
new(func(string) string),
},
},
{
name: "IpInRange",
function: IpInRange,
signature: []interface{}{
new(func(string, string) bool),
},
},
{
name: "TimeNow",
function: TimeNow,
signature: []interface{}{
new(func() string),
},
},
{
name: "ParseUri",
function: ParseUri,
signature: []interface{}{
new(func(string) map[string][]string),
},
},
{
name: "PathUnescape",
function: PathUnescape,
signature: []interface{}{
new(func(string) string),
},
},
{
name: "QueryUnescape",
function: QueryUnescape,
signature: []interface{}{
new(func(string) string),
},
},
{
name: "PathEscape",
function: PathEscape,
signature: []interface{}{
new(func(string) string),
},
},
{
name: "QueryEscape",
function: QueryEscape,
signature: []interface{}{
new(func(string) string),
},
},
{
name: "XMLGetAttributeValue",
function: XMLGetAttributeValue,
signature: []interface{}{
new(func(string, string, string) string),
},
},
{
name: "XMLGetNodeValue",
function: XMLGetNodeValue,
signature: []interface{}{
new(func(string, string) string),
},
},
{
name: "IpToRange",
function: IpToRange,
signature: []interface{}{
new(func(string, string) string),
},
},
{
name: "IsIPV6",
function: IsIPV6,
signature: []interface{}{
new(func(string) bool),
},
},
{
name: "IsIPV4",
function: IsIPV4,
signature: []interface{}{
new(func(string) bool),
},
},
{
name: "IsIP",
function: IsIP,
signature: []interface{}{
new(func(string) bool),
},
},
{
name: "LookupHost",
function: LookupHost,
signature: []interface{}{
new(func(string) []string),
},
},
{
name: "GetDecisionsCount",
function: GetDecisionsCount,
signature: []interface{}{
new(func(string) int),
},
},
{
name: "GetDecisionsSinceCount",
function: GetDecisionsSinceCount,
signature: []interface{}{
new(func(string, string) int),
},
},
{
name: "Sprintf",
function: Sprintf,
signature: []interface{}{
new(func(string, ...interface{}) string),
},
},
{
name: "ParseUnix",
function: ParseUnix,
signature: []interface{}{
new(func(string) string),
},
},
{
name: "SetInStash", //FIXME: signature will probably blow everything up
function: SetInStash,
signature: []interface{}{
new(func(string, string, string, *time.Duration) error),
},
},
{
name: "Fields",
function: Fields,
signature: []interface{}{
new(func(string) []string),
},
},
{
name: "Index",
function: Index,
signature: []interface{}{
new(func(string, string) int),
},
},
{
name: "IndexAny",
function: IndexAny,
signature: []interface{}{
new(func(string, string) int),
},
},
{
name: "Join",
function: Join,
signature: []interface{}{
new(func([]string, string) string),
},
},
{
name: "Split",
function: Split,
signature: []interface{}{
new(func(string, string) []string),
},
},
{
name: "SplitAfter",
function: SplitAfter,
signature: []interface{}{
new(func(string, string) []string),
},
},
{
name: "SplitAfterN",
function: SplitAfterN,
signature: []interface{}{
new(func(string, string, int) []string),
},
},
{
name: "SplitN",
function: SplitN,
signature: []interface{}{
new(func(string, string, int) []string),
},
},
{
name: "Replace",
function: Replace,
signature: []interface{}{
new(func(string, string, string, int) string),
},
},
{
name: "ReplaceAll",
function: ReplaceAll,
signature: []interface{}{
new(func(string, string, string) string),
},
},
{
name: "Trim",
function: Trim,
signature: []interface{}{
new(func(string, string) string),
},
},
{
name: "TrimLeft",
function: TrimLeft,
signature: []interface{}{
new(func(string, string) string),
},
},
{
name: "TrimRight",
function: TrimRight,
signature: []interface{}{
new(func(string, string) string),
},
},
{
name: "TrimSpace",
function: TrimSpace,
signature: []interface{}{
new(func(string) string),
},
},
{
name: "TrimPrefix",
function: TrimPrefix,
signature: []interface{}{
new(func(string, string) string),
},
},
{
name: "TrimSuffix",
function: TrimSuffix,
signature: []interface{}{
new(func(string, string) string),
},
},
{
name: "Get",
function: Get,
signature: []interface{}{
new(func([]string, int) string),
},
},
{
name: "ToString",
function: ToString,
signature: []interface{}{
new(func(interface{}) string),
},
},
{
name: "Match",
function: Match,
signature: []interface{}{
new(func(string, string) bool),
},
},
}
//go 1.20 "CutPrefix": strings.CutPrefix,
//go 1.20 "CutSuffix": strings.CutSuffix,
//"Cut": strings.Cut, -> returns more than 2 values, not supported by expr

324
pkg/exprhelpers/exprlib_test.go
View file

@ -100,17 +100,17 @@ func TestVisitor(t *testing.T) {
}) })
for _, test := range tests { for _, test := range tests {
compiledFilter, err := expr.Compile(test.filter, expr.Env(GetExprEnv(test.env))) compiledFilter, err := expr.Compile(test.filter, GetExprOptions(test.env)...)
if err != nil && test.err == nil { if err != nil && test.err == nil {
log.Fatalf("compile: %s", err) log.Fatalf("compile: %s", err)
} }
debugFilter, err := NewDebugger(test.filter, expr.Env(GetExprEnv(test.env))) debugFilter, err := NewDebugger(test.filter, GetExprOptions(test.env)...)
if err != nil && test.err == nil { if err != nil && test.err == nil {
log.Fatalf("debug: %s", err) log.Fatalf("debug: %s", err)
} }
if compiledFilter != nil { if compiledFilter != nil {
result, err := expr.Run(compiledFilter, GetExprEnv(test.env)) result, err := expr.Run(compiledFilter, test.env)
if err != nil && test.err == nil { if err != nil && test.err == nil {
log.Fatalf("run : %s", err) log.Fatalf("run : %s", err)
} }
@ -120,37 +120,49 @@ func TestVisitor(t *testing.T) {
} }
if debugFilter != nil { if debugFilter != nil {
debugFilter.Run(clog, test.result, GetExprEnv(test.env)) debugFilter.Run(clog, test.result, test.env)
} }
} }
} }
func TestMatch(t *testing.T) { func TestMatch(t *testing.T) {
err := Init(nil)
require.NoError(t, err)
tests := []struct { tests := []struct {
glob string glob string
val string val string
ret bool ret bool
expr string
}{ }{
{"foo", "foo", true}, {"foo", "foo", true, `Match(pattern, name)`},
{"foo", "bar", false}, {"foo", "bar", false, `Match(pattern, name)`},
{"foo*", "foo", true}, {"foo*", "foo", true, `Match(pattern, name)`},
{"foo*", "foobar", true}, {"foo*", "foobar", true, `Match(pattern, name)`},
{"foo*", "barfoo", false}, {"foo*", "barfoo", false, `Match(pattern, name)`},
{"foo*", "bar", false}, {"foo*", "bar", false, `Match(pattern, name)`},
{"*foo", "foo", true}, {"*foo", "foo", true, `Match(pattern, name)`},
{"*foo", "barfoo", true}, {"*foo", "barfoo", true, `Match(pattern, name)`},
{"foo*r", "foobar", true}, {"foo*r", "foobar", true, `Match(pattern, name)`},
{"foo*r", "foobazr", true}, {"foo*r", "foobazr", true, `Match(pattern, name)`},
{"foo?ar", "foobar", true}, {"foo?ar", "foobar", true, `Match(pattern, name)`},
{"foo?ar", "foobazr", false}, {"foo?ar", "foobazr", false, `Match(pattern, name)`},
{"foo?ar", "foobaz", false}, {"foo?ar", "foobaz", false, `Match(pattern, name)`},
{"*foo?ar?", "foobar", false}, {"*foo?ar?", "foobar", false, `Match(pattern, name)`},
{"*foo?ar?", "foobare", true}, {"*foo?ar?", "foobare", true, `Match(pattern, name)`},
{"*foo?ar?", "rafoobar", false}, {"*foo?ar?", "rafoobar", false, `Match(pattern, name)`},
{"*foo?ar?", "rafoobare", true}, {"*foo?ar?", "rafoobare", true, `Match(pattern, name)`},
} }
for _, test := range tests { for _, test := range tests {
ret := Match(test.glob, test.val) env := map[string]interface{}{
"pattern": test.glob,
"name": test.val,
}
vm, err := expr.Compile(test.expr, GetExprOptions(env)...)
if err != nil {
t.Fatalf("pattern:%s val:%s NOK %s", test.glob, test.val, err)
}
ret, err := expr.Run(vm, env)
assert.NoError(t, err)
if isOk := assert.Equal(t, test.ret, ret); !isOk { if isOk := assert.Equal(t, test.ret, ret); !isOk {
t.Fatalf("pattern:%s val:%s NOK %t != %t", test.glob, test.val, ret, test.ret) t.Fatalf("pattern:%s val:%s NOK %t != %t", test.glob, test.val, ret, test.ret)
} }
@ -158,19 +170,45 @@ func TestMatch(t *testing.T) {
} }
func TestDistanceHelper(t *testing.T) { func TestDistanceHelper(t *testing.T) {
err := Init(nil)
require.NoError(t, err)
//one set of coord is empty tests := []struct {
ret, err := Distance("0.0", "0.0", "12.1", "12.1") lat1 string
lon1 string
lat2 string
lon2 string
dist float64
valid bool
expr string
name string
}{
{"51.45", "1.15", "41.54", "12.27", 1389.1793118293067, true, `Distance(lat1, lon1, lat2, lon2)`, "valid"},
{"lol", "1.15", "41.54", "12.27", 0.0, false, `Distance(lat1, lon1, lat2, lon2)`, "invalid lat1"},
{"0.0", "0.0", "12.1", "12.1", 0.0, true, `Distance(lat1, lon1, lat2, lon2)`, "empty coord"},
}
for _, test := range tests {
t.Run(test.name, func(t *testing.T) {
env := map[string]interface{}{
"lat1": test.lat1,
"lon1": test.lon1,
"lat2": test.lat2,
"lon2": test.lon2,
}
vm, err := expr.Compile(test.expr, GetExprOptions(env)...)
if err != nil {
t.Fatalf("pattern:%s val:%s NOK %s", test.lat1, test.lon1, err)
}
ret, err := expr.Run(vm, env)
if test.valid {
assert.NoError(t, err) assert.NoError(t, err)
assert.Equal(t, 0.0, ret) assert.Equal(t, test.dist, ret)
//those aren't even coords } else {
ret, err = Distance("lol", "42.1", "12.1", "12.1")
assert.NotNil(t, err) assert.NotNil(t, err)
assert.Equal(t, 0.0, ret) }
//real ones })
ret, err = Distance("51.45", "1.15", "41.54", "12.27") }
assert.NoError(t, err)
assert.Equal(t, 1389.1793118293067, ret)
} }
func TestRegexpCacheBehavior(t *testing.T) { func TestRegexpCacheBehavior(t *testing.T) {
@ -185,12 +223,12 @@ func TestRegexpCacheBehavior(t *testing.T) {
err = RegexpCacheInit(filename, types.DataSource{Type: "regex", Size: types.IntPtr(1)}) err = RegexpCacheInit(filename, types.DataSource{Type: "regex", Size: types.IntPtr(1)})
require.NoError(t, err) require.NoError(t, err)
ret := RegexpInFile("crowdsec", filename) ret, _ := RegexpInFile("crowdsec", filename)
assert.False(t, ret) assert.False(t, ret.(bool))
assert.Equal(t, 1, dataFileRegexCache[filename].Len(false)) assert.Equal(t, 1, dataFileRegexCache[filename].Len(false))
ret = RegexpInFile("Crowdsec", filename) ret, _ = RegexpInFile("Crowdsec", filename)
assert.True(t, ret) assert.True(t, ret.(bool))
assert.Equal(t, 1, dataFileRegexCache[filename].Len(false)) assert.Equal(t, 1, dataFileRegexCache[filename].Len(false))
//cache with TTL //cache with TTL
@ -198,8 +236,8 @@ func TestRegexpCacheBehavior(t *testing.T) {
err = RegexpCacheInit(filename, types.DataSource{Type: "regex", Size: types.IntPtr(2), TTL: &ttl}) err = RegexpCacheInit(filename, types.DataSource{Type: "regex", Size: types.IntPtr(2), TTL: &ttl})
require.NoError(t, err) require.NoError(t, err)
ret = RegexpInFile("crowdsec", filename) ret, _ = RegexpInFile("crowdsec", filename)
assert.False(t, ret) assert.False(t, ret.(bool))
assert.Equal(t, 1, dataFileRegexCache[filename].Len(true)) assert.Equal(t, 1, dataFileRegexCache[filename].Len(true))
time.Sleep(1 * time.Second) time.Sleep(1 * time.Second)
@ -249,11 +287,11 @@ func TestRegexpInFile(t *testing.T) {
} }
for _, test := range tests { for _, test := range tests {
compiledFilter, err := expr.Compile(test.filter, expr.Env(GetExprEnv(map[string]interface{}{}))) compiledFilter, err := expr.Compile(test.filter, GetExprOptions(map[string]interface{}{})...)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
result, err := expr.Run(compiledFilter, GetExprEnv(map[string]interface{}{})) result, err := expr.Run(compiledFilter, map[string]interface{}{})
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
@ -374,11 +412,11 @@ func TestFile(t *testing.T) {
} }
for _, test := range tests { for _, test := range tests {
compiledFilter, err := expr.Compile(test.filter, expr.Env(GetExprEnv(map[string]interface{}{}))) compiledFilter, err := expr.Compile(test.filter, GetExprOptions(map[string]interface{}{})...)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
result, err := expr.Run(compiledFilter, GetExprEnv(map[string]interface{}{})) result, err := expr.Run(compiledFilter, map[string]interface{}{})
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
@ -391,6 +429,8 @@ func TestFile(t *testing.T) {
} }
func TestIpInRange(t *testing.T) { func TestIpInRange(t *testing.T) {
err := Init(nil)
assert.NoError(t, err)
tests := []struct { tests := []struct {
name string name string
env map[string]interface{} env map[string]interface{}
@ -403,7 +443,6 @@ func TestIpInRange(t *testing.T) {
env: map[string]interface{}{ env: map[string]interface{}{
"ip": "192.168.0.1", "ip": "192.168.0.1",
"ipRange": "192.168.0.0/24", "ipRange": "192.168.0.0/24",
"IpInRange": IpInRange,
}, },
code: "IpInRange(ip, ipRange)", code: "IpInRange(ip, ipRange)",
result: true, result: true,
@ -414,7 +453,6 @@ func TestIpInRange(t *testing.T) {
env: map[string]interface{}{ env: map[string]interface{}{
"ip": "192.168.0", "ip": "192.168.0",
"ipRange": "192.168.0.0/24", "ipRange": "192.168.0.0/24",
"IpInRange": IpInRange,
}, },
code: "IpInRange(ip, ipRange)", code: "IpInRange(ip, ipRange)",
result: false, result: false,
@ -425,7 +463,6 @@ func TestIpInRange(t *testing.T) {
env: map[string]interface{}{ env: map[string]interface{}{
"ip": "192.168.0.0/255", "ip": "192.168.0.0/255",
"ipRange": "192.168.0.0/24", "ipRange": "192.168.0.0/24",
"IpInRange": IpInRange,
}, },
code: "IpInRange(ip, ipRange)", code: "IpInRange(ip, ipRange)",
result: false, result: false,
@ -434,7 +471,7 @@ func TestIpInRange(t *testing.T) {
} }
for _, test := range tests { for _, test := range tests {
program, err := expr.Compile(test.code, expr.Env(test.env)) program, err := expr.Compile(test.code, GetExprOptions(test.env)...)
require.NoError(t, err) require.NoError(t, err)
output, err := expr.Run(program, test.env) output, err := expr.Run(program, test.env)
require.NoError(t, err) require.NoError(t, err)
@ -445,6 +482,8 @@ func TestIpInRange(t *testing.T) {
} }
func TestIpToRange(t *testing.T) { func TestIpToRange(t *testing.T) {
err := Init(nil)
assert.NoError(t, err)
tests := []struct { tests := []struct {
name string name string
env map[string]interface{} env map[string]interface{}
@ -457,7 +496,6 @@ func TestIpToRange(t *testing.T) {
env: map[string]interface{}{ env: map[string]interface{}{
"ip": "192.168.1.1", "ip": "192.168.1.1",
"netmask": "16", "netmask": "16",
"IpToRange": IpToRange,
}, },
code: "IpToRange(ip, netmask)", code: "IpToRange(ip, netmask)",
result: "192.168.0.0/16", result: "192.168.0.0/16",
@ -468,7 +506,6 @@ func TestIpToRange(t *testing.T) {
env: map[string]interface{}{ env: map[string]interface{}{
"ip": "2001:db8::1", "ip": "2001:db8::1",
"netmask": "/64", "netmask": "/64",
"IpToRange": IpToRange,
}, },
code: "IpToRange(ip, netmask)", code: "IpToRange(ip, netmask)",
result: "2001:db8::/64", result: "2001:db8::/64",
@ -479,7 +516,6 @@ func TestIpToRange(t *testing.T) {
env: map[string]interface{}{ env: map[string]interface{}{
"ip": "192.168.0.1", "ip": "192.168.0.1",
"netmask": "test", "netmask": "test",
"IpToRange": IpToRange,
}, },
code: "IpToRange(ip, netmask)", code: "IpToRange(ip, netmask)",
result: "", result: "",
@ -490,7 +526,6 @@ func TestIpToRange(t *testing.T) {
env: map[string]interface{}{ env: map[string]interface{}{
"ip": "a.b.c.d", "ip": "a.b.c.d",
"netmask": "24", "netmask": "24",
"IpToRange": IpToRange,
}, },
code: "IpToRange(ip, netmask)", code: "IpToRange(ip, netmask)",
result: "", result: "",
@ -501,7 +536,6 @@ func TestIpToRange(t *testing.T) {
env: map[string]interface{}{ env: map[string]interface{}{
"ip": "192.168.1.1", "ip": "192.168.1.1",
"netmask": "35", "netmask": "35",
"IpToRange": IpToRange,
}, },
code: "IpToRange(ip, netmask)", code: "IpToRange(ip, netmask)",
result: "", result: "",
@ -510,7 +544,7 @@ func TestIpToRange(t *testing.T) {
} }
for _, test := range tests { for _, test := range tests {
program, err := expr.Compile(test.code, expr.Env(test.env)) program, err := expr.Compile(test.code, GetExprOptions(test.env)...)
require.NoError(t, err) require.NoError(t, err)
output, err := expr.Run(program, test.env) output, err := expr.Run(program, test.env)
require.NoError(t, err) require.NoError(t, err)
@ -521,39 +555,72 @@ func TestIpToRange(t *testing.T) {
} }
func TestAtof(t *testing.T) { func TestAtof(t *testing.T) {
testFloat := "1.5"
expectedFloat := 1.5
if Atof(testFloat) != expectedFloat { err := Init(nil)
t.Fatalf("Atof should return 1.5 as a float") assert.NoError(t, err)
tests := []struct {
name string
env map[string]interface{}
code string
result float64
}{
{
name: "Atof() test: basic test",
env: map[string]interface{}{
"testFloat": "1.5",
},
code: "Atof(testFloat)",
result: 1.5,
},
{
name: "Atof() test: bad float",
env: map[string]interface{}{
"testFloat": "1aaa.5",
},
code: "Atof(testFloat)",
result: 0.0,
},
} }
log.Printf("test 'Atof()' : OK") for _, test := range tests {
program, err := expr.Compile(test.code, GetExprOptions(test.env)...)
//bad float require.NoError(t, err)
testFloat = "1aaa.5" output, err := expr.Run(program, test.env)
expectedFloat = 0.0 require.NoError(t, err)
require.Equal(t, test.result, output)
if Atof(testFloat) != expectedFloat {
t.Fatalf("Atof should return a negative value (error) as a float got")
} }
log.Printf("test 'Atof()' : OK")
} }
func TestUpper(t *testing.T) { func TestUpper(t *testing.T) {
testStr := "test" testStr := "test"
expectedStr := "TEST" expectedStr := "TEST"
if Upper(testStr) != expectedStr { env := map[string]interface{}{
t.Fatalf("Upper() should return test in upper case") "testStr": testStr,
} }
log.Printf("test 'Upper()' : OK") err := Init(nil)
assert.NoError(t, err)
vm, err := expr.Compile("Upper(testStr)", GetExprOptions(env)...)
assert.NoError(t, err)
out, err := expr.Run(vm, env)
assert.NoError(t, err)
v, ok := out.(string)
if !ok {
t.Fatalf("Upper() should return a string")
}
if v != expectedStr {
t.Fatalf("Upper() should return test in upper case")
}
} }
func TestTimeNow(t *testing.T) { func TestTimeNow(t *testing.T) {
ti, err := time.Parse(time.RFC3339, TimeNow()) now, _ := TimeNow()
ti, err := time.Parse(time.RFC3339, now.(string))
if err != nil { if err != nil {
t.Fatalf("Error parsing the return value of TimeNow: %s", err) t.Fatalf("Error parsing the return value of TimeNow: %s", err)
} }
@ -625,7 +692,7 @@ func TestParseUri(t *testing.T) {
} }
for _, test := range tests { for _, test := range tests {
program, err := expr.Compile(test.code, expr.Env(test.env)) program, err := expr.Compile(test.code, GetExprOptions(test.env)...)
require.NoError(t, err) require.NoError(t, err)
output, err := expr.Run(program, test.env) output, err := expr.Run(program, test.env)
require.NoError(t, err) require.NoError(t, err)
@ -665,7 +732,7 @@ func TestQueryEscape(t *testing.T) {
} }
for _, test := range tests { for _, test := range tests {
program, err := expr.Compile(test.code, expr.Env(test.env)) program, err := expr.Compile(test.code, GetExprOptions(test.env)...)
require.NoError(t, err) require.NoError(t, err)
output, err := expr.Run(program, test.env) output, err := expr.Run(program, test.env)
require.NoError(t, err) require.NoError(t, err)
@ -705,7 +772,7 @@ func TestPathEscape(t *testing.T) {
} }
for _, test := range tests { for _, test := range tests {
program, err := expr.Compile(test.code, expr.Env(test.env)) program, err := expr.Compile(test.code, GetExprOptions(test.env)...)
require.NoError(t, err) require.NoError(t, err)
output, err := expr.Run(program, test.env) output, err := expr.Run(program, test.env)
require.NoError(t, err) require.NoError(t, err)
@ -745,7 +812,7 @@ func TestPathUnescape(t *testing.T) {
} }
for _, test := range tests { for _, test := range tests {
program, err := expr.Compile(test.code, expr.Env(test.env)) program, err := expr.Compile(test.code, GetExprOptions(test.env)...)
require.NoError(t, err) require.NoError(t, err)
output, err := expr.Run(program, test.env) output, err := expr.Run(program, test.env)
require.NoError(t, err) require.NoError(t, err)
@ -785,7 +852,7 @@ func TestQueryUnescape(t *testing.T) {
} }
for _, test := range tests { for _, test := range tests {
program, err := expr.Compile(test.code, expr.Env(test.env)) program, err := expr.Compile(test.code, GetExprOptions(test.env)...)
require.NoError(t, err) require.NoError(t, err)
output, err := expr.Run(program, test.env) output, err := expr.Run(program, test.env)
require.NoError(t, err) require.NoError(t, err)
@ -825,7 +892,7 @@ func TestLower(t *testing.T) {
} }
for _, test := range tests { for _, test := range tests {
program, err := expr.Compile(test.code, expr.Env(test.env)) program, err := expr.Compile(test.code, GetExprOptions(test.env)...)
require.NoError(t, err) require.NoError(t, err)
output, err := expr.Run(program, test.env) output, err := expr.Run(program, test.env)
require.NoError(t, err) require.NoError(t, err)
@ -865,6 +932,9 @@ func TestGetDecisionsCount(t *testing.T) {
assert.Error(t, errors.Errorf("Failed to create sample decision")) assert.Error(t, errors.Errorf("Failed to create sample decision"))
} }
err = Init(dbClient)
assert.NoError(t, err)
tests := []struct { tests := []struct {
name string name string
env map[string]interface{} env map[string]interface{}
@ -885,10 +955,8 @@ func TestGetDecisionsCount(t *testing.T) {
}, },
}, },
}, },
"GetDecisionsCount": GetDecisionsCount,
"sprintf": fmt.Sprintf,
}, },
code: "sprintf('%d', GetDecisionsCount(Alert.GetValue()))", code: "Sprintf('%d', GetDecisionsCount(Alert.GetValue()))",
result: "1", result: "1",
err: "", err: "",
}, },
@ -905,19 +973,17 @@ func TestGetDecisionsCount(t *testing.T) {
}, },
}, },
}, },
"GetDecisionsCount": GetDecisionsCount,
"sprintf": fmt.Sprintf,
}, },
code: "sprintf('%d', GetDecisionsCount(Alert.GetValue()))", code: "Sprintf('%d', GetDecisionsCount(Alert.GetValue()))",
result: "0", result: "0",
err: "", err: "",
}, },
} }
for _, test := range tests { for _, test := range tests {
program, err := expr.Compile(test.code, expr.Env(GetExprEnv(test.env))) program, err := expr.Compile(test.code, GetExprOptions(test.env)...)
require.NoError(t, err) require.NoError(t, err)
output, err := expr.Run(program, GetExprEnv(test.env)) output, err := expr.Run(program, test.env)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, test.result, output) require.Equal(t, test.result, output)
log.Printf("test '%s' : OK", test.name) log.Printf("test '%s' : OK", test.name)
@ -970,6 +1036,9 @@ func TestGetDecisionsSinceCount(t *testing.T) {
assert.Error(t, errors.Errorf("Failed to create sample decision")) assert.Error(t, errors.Errorf("Failed to create sample decision"))
} }
err = Init(dbClient)
assert.NoError(t, err)
tests := []struct { tests := []struct {
name string name string
env map[string]interface{} env map[string]interface{}
@ -990,10 +1059,8 @@ func TestGetDecisionsSinceCount(t *testing.T) {
}, },
}, },
}, },
"GetDecisionsSinceCount": GetDecisionsSinceCount,
"sprintf": fmt.Sprintf,
}, },
code: "sprintf('%d', GetDecisionsSinceCount(Alert.GetValue(), '25h'))", code: "Sprintf('%d', GetDecisionsSinceCount(Alert.GetValue(), '25h'))",
result: "2", result: "2",
err: "", err: "",
}, },
@ -1010,10 +1077,8 @@ func TestGetDecisionsSinceCount(t *testing.T) {
}, },
}, },
}, },
"GetDecisionsSinceCount": GetDecisionsSinceCount,
"sprintf": fmt.Sprintf,
}, },
code: "sprintf('%d', GetDecisionsSinceCount(Alert.GetValue(), '1h'))", code: "Sprintf('%d', GetDecisionsSinceCount(Alert.GetValue(), '1h'))",
result: "1", result: "1",
err: "", err: "",
}, },
@ -1030,19 +1095,17 @@ func TestGetDecisionsSinceCount(t *testing.T) {
}, },
}, },
}, },
"GetDecisionsSinceCount": GetDecisionsSinceCount,
"sprintf": fmt.Sprintf,
}, },
code: "sprintf('%d', GetDecisionsSinceCount(Alert.GetValue(), '1h'))", code: "Sprintf('%d', GetDecisionsSinceCount(Alert.GetValue(), '1h'))",
result: "0", result: "0",
err: "", err: "",
}, },
} }
for _, test := range tests { for _, test := range tests {
program, err := expr.Compile(test.code, expr.Env(GetExprEnv(test.env))) program, err := expr.Compile(test.code, GetExprOptions(test.env)...)
require.NoError(t, err) require.NoError(t, err)
output, err := expr.Run(program, GetExprEnv(test.env)) output, err := expr.Run(program, test.env)
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, test.result, output) require.Equal(t, test.result, output)
log.Printf("test '%s' : OK", test.name) log.Printf("test '%s' : OK", test.name)
@ -1088,113 +1151,156 @@ func TestParseUnixTime(t *testing.T) {
if tc.expectedErr != "" { if tc.expectedErr != "" {
return return
} }
require.WithinDuration(t, tc.expected, output, time.Second) require.WithinDuration(t, tc.expected, output.(time.Time), time.Second)
}) })
} }
} }
func TestIsIp(t *testing.T) { func TestIsIp(t *testing.T) {
if err := Init(nil); err != nil {
log.Fatal(err)
}
tests := []struct { tests := []struct {
name string name string
method func(string) bool expr string
value string value string
expected bool expected bool
expectedBuildErr bool
}{ }{
{ {
name: "IsIPV4() test: valid IPv4", name: "IsIPV4() test: valid IPv4",
method: IsIPV4, expr: `IsIPV4(value)`,
value: "1.2.3.4", value: "1.2.3.4",
expected: true, expected: true,
}, },
{ {
name: "IsIPV6() test: valid IPv6", name: "IsIPV6() test: valid IPv6",
method: IsIPV6, expr: `IsIPV6(value)`,
value: "1.2.3.4", value: "1.2.3.4",
expected: false, expected: false,
}, },
{ {
name: "IsIPV6() test: valid IPv6", name: "IsIPV6() test: valid IPv6",
method: IsIPV6, expr: `IsIPV6(value)`,
value: "2001:0db8:85a3:0000:0000:8a2e:0370:7334", value: "2001:0db8:85a3:0000:0000:8a2e:0370:7334",
expected: true, expected: true,
}, },
{ {
name: "IsIPV4() test: valid IPv6", name: "IsIPV4() test: valid IPv6",
method: IsIPV4, expr: `IsIPV4(value)`,
value: "2001:0db8:85a3:0000:0000:8a2e:0370:7334", value: "2001:0db8:85a3:0000:0000:8a2e:0370:7334",
expected: false, expected: false,
}, },
{ {
name: "IsIP() test: invalid IP", name: "IsIP() test: invalid IP",
method: IsIP, expr: `IsIP(value)`,
value: "foo.bar", value: "foo.bar",
expected: false, expected: false,
}, },
{ {
name: "IsIP() test: valid IPv4", name: "IsIP() test: valid IPv4",
method: IsIP, expr: `IsIP(value)`,
value: "1.2.3.4", value: "1.2.3.4",
expected: true, expected: true,
}, },
{ {
name: "IsIP() test: valid IPv6", name: "IsIP() test: valid IPv6",
method: IsIP, expr: `IsIP(value)`,
value: "2001:0db8:85a3:0000:0000:8a2e:0370:7334", value: "2001:0db8:85a3:0000:0000:8a2e:0370:7334",
expected: true, expected: true,
}, },
{ {
name: "IsIPV4() test: invalid IPv4", name: "IsIPV4() test: invalid IPv4",
method: IsIPV4, expr: `IsIPV4(value)`,
value: "foo.bar", value: "foo.bar",
expected: false, expected: false,
}, },
{ {
name: "IsIPV6() test: invalid IPv6", name: "IsIPV6() test: invalid IPv6",
method: IsIPV6, expr: `IsIPV6(value)`,
value: "foo.bar", value: "foo.bar",
expected: false, expected: false,
}, },
{
name: "IsIPV4() test: invalid type",
expr: `IsIPV4(42)`,
value: "",
expected: false,
expectedBuildErr: true,
},
{
name: "IsIP() test: invalid type",
expr: `IsIP(42)`,
value: "",
expected: false,
expectedBuildErr: true,
},
{
name: "IsIPV6() test: invalid type",
expr: `IsIPV6(42)`,
value: "",
expected: false,
expectedBuildErr: true,
},
} }
for _, tc := range tests { for _, tc := range tests {
tc := tc tc := tc
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
output := tc.method(tc.value) vm, err := expr.Compile(tc.expr, GetExprOptions(map[string]interface{}{"value": tc.value})...)
require.Equal(t, tc.expected, output) if tc.expectedBuildErr {
assert.Error(t, err)
return
}
assert.NoError(t, err)
output, err := expr.Run(vm, map[string]interface{}{"value": tc.value})
assert.NoError(t, err)
assert.IsType(t, tc.expected, output)
assert.Equal(t, tc.expected, output.(bool))
}) })
} }
} }
func TestToString(t *testing.T) { func TestToString(t *testing.T) {
err := Init(nil)
require.NoError(t, err)
tests := []struct { tests := []struct {
name string name string
value interface{} value interface{}
expected string expected string
expr string
}{ }{
{ {
name: "ToString() test: valid string", name: "ToString() test: valid string",
value: "foo", value: "foo",
expected: "foo", expected: "foo",
expr: `ToString(value)`,
}, },
{ {
name: "ToString() test: valid string", name: "ToString() test: valid string",
value: interface{}("foo"), value: interface{}("foo"),
expected: "foo", expected: "foo",
expr: `ToString(value)`,
}, },
{ {
name: "ToString() test: invalid type", name: "ToString() test: invalid type",
value: 1, value: 1,
expected: "", expected: "",
expr: `ToString(value)`,
}, },
{ {
name: "ToString() test: invalid type 2", name: "ToString() test: invalid type 2",
value: interface{}(nil), value: interface{}(nil),
expected: "", expected: "",
expr: `ToString(value)`,
}, },
} }
for _, tc := range tests { for _, tc := range tests {
tc := tc tc := tc
t.Run(tc.name, func(t *testing.T) { t.Run(tc.name, func(t *testing.T) {
output := ToString(tc.value) vm, err := expr.Compile(tc.expr, GetExprOptions(map[string]interface{}{"value": tc.value})...)
assert.NoError(t, err)
output, err := expr.Run(vm, map[string]interface{}{"value": tc.value})
assert.NoError(t, err)
require.Equal(t, tc.expected, output) require.Equal(t, tc.expected, output)
}) })
} }

429
pkg/exprhelpers/exprlib.go → pkg/exprhelpers/helpers.go
View file

@ -12,6 +12,7 @@ import (
"strings" "strings"
"time" "time"
"github.com/antonmedv/expr"
"github.com/bluele/gcache" "github.com/bluele/gcache"
"github.com/c-robinson/iplib" "github.com/c-robinson/iplib"
"github.com/cespare/xxhash/v2" "github.com/cespare/xxhash/v2"
@ -43,135 +44,29 @@ var RegexpCacheMetrics = prometheus.NewGaugeVec(
var dbClient *database.Client var dbClient *database.Client
func Get(arr []string, index int) string { var exprFunctionOptions []expr.Option
if index >= len(arr) {
return ""
}
return arr[index]
}
func Atof(x string) float64 { func GetExprOptions(ctx map[string]interface{}) []expr.Option {
log.Debugf("debug atof %s", x) ret := []expr.Option{}
ret, err := strconv.ParseFloat(x, 64) ret = append(ret, exprFunctionOptions...)
if err != nil { ret = append(ret, expr.Env(ctx))
log.Warningf("Atof : can't convert float '%s' : %v", x, err)
}
return ret return ret
} }
func Upper(s string) string {
return strings.ToUpper(s)
}
func Lower(s string) string {
return strings.ToLower(s)
}
func GetExprEnv(ctx map[string]interface{}) map[string]interface{} {
var ExprLib = map[string]interface{}{
"Atof": Atof,
"JsonExtract": JsonExtract,
"JsonExtractUnescape": JsonExtractUnescape,
"JsonExtractLib": JsonExtractLib,
"JsonExtractSlice": JsonExtractSlice,
"JsonExtractObject": JsonExtractObject,
"ToJsonString": ToJson,
"File": File,
"RegexpInFile": RegexpInFile,
"Upper": Upper,
"Lower": Lower,
"IpInRange": IpInRange,
"TimeNow": TimeNow,
"ParseUri": ParseUri,
"PathUnescape": PathUnescape,
"QueryUnescape": QueryUnescape,
"PathEscape": PathEscape,
"QueryEscape": QueryEscape,
"XMLGetAttributeValue": XMLGetAttributeValue,
"XMLGetNodeValue": XMLGetNodeValue,
"IpToRange": IpToRange,
"IsIPV6": IsIPV6,
"IsIPV4": IsIPV4,
"IsIP": IsIP,
"LookupHost": LookupHost,
"GetDecisionsCount": GetDecisionsCount,
"GetDecisionsSinceCount": GetDecisionsSinceCount,
"Sprintf": fmt.Sprintf,
"CrowdsecCTI": CrowdsecCTI,
"ParseUnix": ParseUnix,
"GetFromStash": cache.GetKey,
"SetInStash": cache.SetKey,
//go 1.20 "CutPrefix": strings.CutPrefix,
//go 1.20 "CutSuffix": strings.CutSuffix,
//"Cut": strings.Cut, -> returns more than 2 values, not supported by expr
"Fields": strings.Fields,
"Index": strings.Index,
"IndexAny": strings.IndexAny,
"Join": strings.Join,
"Split": strings.Split,
"SplitAfter": strings.SplitAfter,
"SplitAfterN": strings.SplitAfterN,
"SplitN": strings.SplitN,
"Replace": strings.Replace,
"ReplaceAll": strings.ReplaceAll,
"Trim": strings.Trim,
"TrimLeft": strings.TrimLeft,
"TrimRight": strings.TrimRight,
"TrimSpace": strings.TrimSpace,
"TrimPrefix": strings.TrimPrefix,
"TrimSuffix": strings.TrimSuffix,
"Get": Get,
"String": ToString,
"Distance": Distance,
"Match": Match,
}
for k, v := range ctx {
ExprLib[k] = v
}
return ExprLib
}
func Distance(lat1 string, long1 string, lat2 string, long2 string) (float64, error) {
lat1f, err := strconv.ParseFloat(lat1, 64)
if err != nil {
log.Warningf("lat1 is not a float : %v", err)
return 0, fmt.Errorf("lat1 is not a float : %v", err)
}
long1f, err := strconv.ParseFloat(long1, 64)
if err != nil {
log.Warningf("long1 is not a float : %v", err)
return 0, fmt.Errorf("long1 is not a float : %v", err)
}
lat2f, err := strconv.ParseFloat(lat2, 64)
if err != nil {
log.Warningf("lat2 is not a float : %v", err)
return 0, fmt.Errorf("lat2 is not a float : %v", err)
}
long2f, err := strconv.ParseFloat(long2, 64)
if err != nil {
log.Warningf("long2 is not a float : %v", err)
return 0, fmt.Errorf("long2 is not a float : %v", err)
}
//either set of coordinates is 0,0, return 0 to avoid FPs
if (lat1f == 0.0 && long1f == 0.0) || (lat2f == 0.0 && long2f == 0.0) {
log.Warningf("one of the coordinates is 0,0, returning 0")
return 0, nil
}
first := haversine.Coord{Lat: lat1f, Lon: long1f}
second := haversine.Coord{Lat: lat2f, Lon: long2f}
_, km := haversine.Distance(first, second)
return km, nil
}
func Init(databaseClient *database.Client) error { func Init(databaseClient *database.Client) error {
dataFile = make(map[string][]string) dataFile = make(map[string][]string)
dataFileRegex = make(map[string][]*regexp.Regexp) dataFileRegex = make(map[string][]*regexp.Regexp)
dbClient = databaseClient dbClient = databaseClient
exprFunctionOptions = []expr.Option{}
for _, function := range exprFuncs {
exprFunctionOptions = append(exprFunctionOptions,
expr.Function(function.name,
function.function,
function.signature...,
))
}
return nil return nil
} }
@ -263,43 +158,132 @@ func FileInit(fileFolder string, filename string, fileType string) error {
return nil return nil
} }
func QueryEscape(s string) string { //Expr helpers
return url.QueryEscape(s)
// func Get(arr []string, index int) string {
func Get(params ...any) (any, error) {
arr := params[0].([]string)
index := params[1].(int)
if index >= len(arr) {
return "", nil
}
return arr[index], nil
} }
func PathEscape(s string) string { // func Atof(x string) float64 {
return url.PathEscape(s) func Atof(params ...any) (any, error) {
x := params[0].(string)
log.Debugf("debug atof %s", x)
ret, err := strconv.ParseFloat(x, 64)
if err != nil {
log.Warningf("Atof : can't convert float '%s' : %v", x, err)
}
return ret, nil
} }
func PathUnescape(s string) string { // func Upper(s string) string {
func Upper(params ...any) (any, error) {
s := params[0].(string)
return strings.ToUpper(s), nil
}
// func Lower(s string) string {
func Lower(params ...any) (any, error) {
s := params[0].(string)
return strings.ToLower(s), nil
}
// func Distance(lat1 string, long1 string, lat2 string, long2 string) (float64, error) {
func Distance(params ...any) (any, error) {
lat1 := params[0].(string)
long1 := params[1].(string)
lat2 := params[2].(string)
long2 := params[3].(string)
lat1f, err := strconv.ParseFloat(lat1, 64)
if err != nil {
log.Warningf("lat1 is not a float : %v", err)
return 0.0, fmt.Errorf("lat1 is not a float : %v", err)
}
long1f, err := strconv.ParseFloat(long1, 64)
if err != nil {
log.Warningf("long1 is not a float : %v", err)
return 0.0, fmt.Errorf("long1 is not a float : %v", err)
}
lat2f, err := strconv.ParseFloat(lat2, 64)
if err != nil {
log.Warningf("lat2 is not a float : %v", err)
return 0.0, fmt.Errorf("lat2 is not a float : %v", err)
}
long2f, err := strconv.ParseFloat(long2, 64)
if err != nil {
log.Warningf("long2 is not a float : %v", err)
return 0.0, fmt.Errorf("long2 is not a float : %v", err)
}
//either set of coordinates is 0,0, return 0 to avoid FPs
if (lat1f == 0.0 && long1f == 0.0) || (lat2f == 0.0 && long2f == 0.0) {
log.Warningf("one of the coordinates is 0,0, returning 0")
return 0.0, nil
}
first := haversine.Coord{Lat: lat1f, Lon: long1f}
second := haversine.Coord{Lat: lat2f, Lon: long2f}
_, km := haversine.Distance(first, second)
return km, nil
}
// func QueryEscape(s string) string {
func QueryEscape(params ...any) (any, error) {
s := params[0].(string)
return url.QueryEscape(s), nil
}
// func PathEscape(s string) string {
func PathEscape(params ...any) (any, error) {
s := params[0].(string)
return url.PathEscape(s), nil
}
// func PathUnescape(s string) string {
func PathUnescape(params ...any) (any, error) {
s := params[0].(string)
ret, err := url.PathUnescape(s) ret, err := url.PathUnescape(s)
if err != nil { if err != nil {
log.Debugf("unable to PathUnescape '%s': %+v", s, err) log.Debugf("unable to PathUnescape '%s': %+v", s, err)
return s return s, nil
} }
return ret return ret, nil
} }
func QueryUnescape(s string) string { // func QueryUnescape(s string) string {
func QueryUnescape(params ...any) (any, error) {
s := params[0].(string)
ret, err := url.QueryUnescape(s) ret, err := url.QueryUnescape(s)
if err != nil { if err != nil {
log.Debugf("unable to QueryUnescape '%s': %+v", s, err) log.Debugf("unable to QueryUnescape '%s': %+v", s, err)
return s return s, nil
} }
return ret return ret, nil
} }
func File(filename string) []string { // func File(filename string) []string {
func File(params ...any) (any, error) {
filename := params[0].(string)
if _, ok := dataFile[filename]; ok { if _, ok := dataFile[filename]; ok {
return dataFile[filename] return dataFile[filename], nil
} }
log.Errorf("file '%s' (type:string) not found in expr library", filename) log.Errorf("file '%s' (type:string) not found in expr library", filename)
log.Errorf("expr library : %s", spew.Sdump(dataFile)) log.Errorf("expr library : %s", spew.Sdump(dataFile))
return []string{} return []string{}, nil
} }
func RegexpInFile(data string, filename string) bool { // func RegexpInFile(data string, filename string) bool {
func RegexpInFile(params ...any) (any, error) {
data := params[0].(string)
filename := params[1].(string)
var hash uint64 var hash uint64
hasCache := false hasCache := false
@ -307,7 +291,7 @@ func RegexpInFile(data string, filename string) bool {
hasCache = true hasCache = true
hash = xxhash.Sum64String(data) hash = xxhash.Sum64String(data)
if val, err := dataFileRegexCache[filename].Get(hash); err == nil { if val, err := dataFileRegexCache[filename].Get(hash); err == nil {
return val.(bool) return val.(bool), nil
} }
} }
@ -317,7 +301,7 @@ func RegexpInFile(data string, filename string) bool {
if hasCache { if hasCache {
dataFileRegexCache[filename].Set(hash, true) dataFileRegexCache[filename].Set(hash, true)
} }
return true return true, nil
} }
} }
} else { } else {
@ -327,149 +311,177 @@ func RegexpInFile(data string, filename string) bool {
if hasCache { if hasCache {
dataFileRegexCache[filename].Set(hash, false) dataFileRegexCache[filename].Set(hash, false)
} }
return false return false, nil
} }
func IpInRange(ip string, ipRange string) bool { // func IpInRange(ip string, ipRange string) bool {
func IpInRange(params ...any) (any, error) {
var err error var err error
var ipParsed net.IP var ipParsed net.IP
var ipRangeParsed *net.IPNet var ipRangeParsed *net.IPNet
ip := params[0].(string)
ipRange := params[1].(string)
ipParsed = net.ParseIP(ip) ipParsed = net.ParseIP(ip)
if ipParsed == nil { if ipParsed == nil {
log.Debugf("'%s' is not a valid IP", ip) log.Debugf("'%s' is not a valid IP", ip)
return false return false, nil
} }
if _, ipRangeParsed, err = net.ParseCIDR(ipRange); err != nil { if _, ipRangeParsed, err = net.ParseCIDR(ipRange); err != nil {
log.Debugf("'%s' is not a valid IP Range", ipRange) log.Debugf("'%s' is not a valid IP Range", ipRange)
return false return false, nil //nolint:nilerr // This helper did not return an error before the move to expr.Function, we keep this behavior for backward compatibility
} }
if ipRangeParsed.Contains(ipParsed) { if ipRangeParsed.Contains(ipParsed) {
return true return true, nil
} }
return false return false, nil
} }
func IsIPV6(ip string) bool { // func IsIPV6(ip string) bool {
func IsIPV6(params ...any) (any, error) {
ip := params[0].(string)
ipParsed := net.ParseIP(ip) ipParsed := net.ParseIP(ip)
if ipParsed == nil { if ipParsed == nil {
log.Debugf("'%s' is not a valid IP", ip) log.Debugf("'%s' is not a valid IP", ip)
return false return false, nil
} }
// If it's a valid IP and can't be converted to IPv4 then it is an IPv6 // If it's a valid IP and can't be converted to IPv4 then it is an IPv6
return ipParsed.To4() == nil return ipParsed.To4() == nil, nil
} }
func IsIPV4(ip string) bool { // func IsIPV4(ip string) bool {
func IsIPV4(params ...any) (any, error) {
ip := params[0].(string)
ipParsed := net.ParseIP(ip) ipParsed := net.ParseIP(ip)
if ipParsed == nil { if ipParsed == nil {
log.Debugf("'%s' is not a valid IP", ip) log.Debugf("'%s' is not a valid IP", ip)
return false return false, nil
} }
return ipParsed.To4() != nil return ipParsed.To4() != nil, nil
} }
func IsIP(ip string) bool { // func IsIP(ip string) bool {
func IsIP(params ...any) (any, error) {
ip := params[0].(string)
ipParsed := net.ParseIP(ip) ipParsed := net.ParseIP(ip)
if ipParsed == nil { if ipParsed == nil {
log.Debugf("'%s' is not a valid IP", ip) log.Debugf("'%s' is not a valid IP", ip)
return false return false, nil
} }
return true return true, nil
} }
func IpToRange(ip string, cidr string) string { // func IpToRange(ip string, cidr string) string {
func IpToRange(params ...any) (any, error) {
ip := params[0].(string)
cidr := params[1].(string)
cidr = strings.TrimPrefix(cidr, "/") cidr = strings.TrimPrefix(cidr, "/")
mask, err := strconv.Atoi(cidr) mask, err := strconv.Atoi(cidr)
if err != nil { if err != nil {
log.Errorf("bad cidr '%s': %s", cidr, err) log.Errorf("bad cidr '%s': %s", cidr, err)
return "" return "", nil
} }
ipAddr := net.ParseIP(ip) ipAddr := net.ParseIP(ip)
if ipAddr == nil { if ipAddr == nil {
log.Errorf("can't parse IP address '%s'", ip) log.Errorf("can't parse IP address '%s'", ip)
return "" return "", nil
} }
ipRange := iplib.NewNet(ipAddr, mask) ipRange := iplib.NewNet(ipAddr, mask)
if ipRange.IP() == nil { if ipRange.IP() == nil {
log.Errorf("can't get cidr '%s' of '%s'", cidr, ip) log.Errorf("can't get cidr '%s' of '%s'", cidr, ip)
return "" return "", nil
} }
return ipRange.String() return ipRange.String(), nil
} }
func TimeNow() string { // func TimeNow() string {
return time.Now().UTC().Format(time.RFC3339) func TimeNow(params ...any) (any, error) {
return time.Now().UTC().Format(time.RFC3339), nil
} }
func ParseUri(uri string) map[string][]string { // func ParseUri(uri string) map[string][]string {
func ParseUri(params ...any) (any, error) {
uri := params[0].(string)
ret := make(map[string][]string) ret := make(map[string][]string)
u, err := url.Parse(uri) u, err := url.Parse(uri)
if err != nil { if err != nil {
log.Errorf("Could not parse URI: %s", err) log.Errorf("Could not parse URI: %s", err)
return ret return ret, nil
} }
parsed, err := url.ParseQuery(u.RawQuery) parsed, err := url.ParseQuery(u.RawQuery)
if err != nil { if err != nil {
log.Errorf("Could not parse query uri : %s", err) log.Errorf("Could not parse query uri : %s", err)
return ret return ret, nil
} }
for k, v := range parsed { for k, v := range parsed {
ret[k] = v ret[k] = v
} }
return ret return ret, nil
} }
func KeyExists(key string, dict map[string]interface{}) bool { // func KeyExists(key string, dict map[string]interface{}) bool {
func KeyExists(params ...any) (any, error) {
key := params[0].(string)
dict := params[1].(map[string]interface{})
_, ok := dict[key] _, ok := dict[key]
return ok return ok, nil
} }
func GetDecisionsCount(value string) int { // func GetDecisionsCount(value string) int {
func GetDecisionsCount(params ...any) (any, error) {
value := params[0].(string)
if dbClient == nil { if dbClient == nil {
log.Error("No database config to call GetDecisionsCount()") log.Error("No database config to call GetDecisionsCount()")
return 0 return 0, nil
} }
count, err := dbClient.CountDecisionsByValue(value) count, err := dbClient.CountDecisionsByValue(value)
if err != nil { if err != nil {
log.Errorf("Failed to get decisions count from value '%s'", value) log.Errorf("Failed to get decisions count from value '%s'", value)
return 0 return 0, nil //nolint:nilerr // This helper did not return an error before the move to expr.Function, we keep this behavior for backward compatibility
} }
return count return count, nil
} }
func GetDecisionsSinceCount(value string, since string) int { // func GetDecisionsSinceCount(value string, since string) int {
func GetDecisionsSinceCount(params ...any) (any, error) {
value := params[0].(string)
since := params[1].(string)
if dbClient == nil { if dbClient == nil {
log.Error("No database config to call GetDecisionsCount()") log.Error("No database config to call GetDecisionsCount()")
return 0 return 0, nil
} }
sinceDuration, err := time.ParseDuration(since) sinceDuration, err := time.ParseDuration(since)
if err != nil { if err != nil {
log.Errorf("Failed to parse since parameter '%s' : %s", since, err) log.Errorf("Failed to parse since parameter '%s' : %s", since, err)
return 0 return 0, nil
} }
sinceTime := time.Now().UTC().Add(-sinceDuration) sinceTime := time.Now().UTC().Add(-sinceDuration)
count, err := dbClient.CountDecisionsSinceByValue(value, sinceTime) count, err := dbClient.CountDecisionsSinceByValue(value, sinceTime)
if err != nil { if err != nil {
log.Errorf("Failed to get decisions count from value '%s'", value) log.Errorf("Failed to get decisions count from value '%s'", value)
return 0 return 0, nil //nolint:nilerr // This helper did not return an error before the move to expr.Function, we keep this behavior for backward compatibility
} }
return count return count, nil
} }
func LookupHost(value string) []string { // func LookupHost(value string) []string {
func LookupHost(params ...any) (any, error) {
value := params[0].(string)
addresses, err := net.LookupHost(value) addresses, err := net.LookupHost(value)
if err != nil { if err != nil {
log.Errorf("Failed to lookup host '%s' : %s", value, err) log.Errorf("Failed to lookup host '%s' : %s", value, err)
return []string{} return []string{}, nil
} }
return addresses return addresses, nil
} }
func ParseUnixTime(value string) (time.Time, error) { // func ParseUnixTime(value string) (time.Time, error) {
func ParseUnixTime(params ...any) (any, error) {
value := params[0].(string)
//Splitting string here as some unix timestamp may have milliseconds and break ParseInt //Splitting string here as some unix timestamp may have milliseconds and break ParseInt
i, err := strconv.ParseInt(strings.Split(value, ".")[0], 10, 64) i, err := strconv.ParseInt(strings.Split(value, ".")[0], 10, 64)
if err != nil || i <= 0 { if err != nil || i <= 0 {
@ -478,44 +490,75 @@ func ParseUnixTime(value string) (time.Time, error) {
return time.Unix(i, 0), nil return time.Unix(i, 0), nil
} }
func ParseUnix(value string) string { // func ParseUnix(value string) string {
func ParseUnix(params ...any) (any, error) {
value := params[0].(string)
t, err := ParseUnixTime(value) t, err := ParseUnixTime(value)
if err != nil { if err != nil {
log.Error(err) log.Error(err)
return "" return "", nil
} }
return t.Format(time.RFC3339) return t.(time.Time).Format(time.RFC3339), nil
} }
func ToString(value interface{}) string { // func ToString(value interface{}) string {
func ToString(params ...any) (any, error) {
value := params[0]
s, ok := value.(string) s, ok := value.(string)
if !ok { if !ok {
return "" return "", nil
} }
return s return s, nil
} }
func Match(pattern, name string) bool { // func GetFromStash(cacheName string, key string) (string, error) {
func GetFromStash(params ...any) (any, error) {
cacheName := params[0].(string)
key := params[1].(string)
return cache.GetKey(cacheName, key)
}
// func SetInStash(cacheName string, key string, value string, expiration *time.Duration) any {
func SetInStash(params ...any) (any, error) {
cacheName := params[0].(string)
key := params[1].(string)
value := params[2].(string)
expiration := params[3].(*time.Duration)
return cache.SetKey(cacheName, key, value, expiration), nil
}
func Sprintf(params ...any) (any, error) {
format := params[0].(string)
return fmt.Sprintf(format, params[1:]...), nil
}
// func Match(pattern, name string) bool {
func Match(params ...any) (any, error) {
var matched bool var matched bool
pattern := params[0].(string)
name := params[1].(string)
if pattern == "" { if pattern == "" {
return name == "" return name == "", nil
} }
if name == "" { if name == "" {
if pattern == "*" || pattern == "" { if pattern == "*" || pattern == "" {
return true return true, nil
} }
return false return false, nil
} }
if pattern[0] == '*' { if pattern[0] == '*' {
for i := 0; i <= len(name); i++ { for i := 0; i <= len(name); i++ {
if matched = Match(pattern[1:], name[i:]); matched { matched, _ := Match(pattern[1:], name[i:])
return matched if matched.(bool) {
return matched, nil
} }
} }
return matched return matched, nil
} }
if pattern[0] == '?' || pattern[0] == name[0] { if pattern[0] == '?' || pattern[0] == name[0] {
return Match(pattern[1:], name[1:]) return Match(pattern[1:], name[1:])
} }
return matched return matched, nil
} }

82
pkg/exprhelpers/jsonextract.go
View file

@ -11,7 +11,10 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
) )
func JsonExtractLib(jsblob string, target ...string) string { // func JsonExtractLib(jsblob string, target ...string) string {
func JsonExtractLib(params ...any) (any, error) {
jsblob := params[0].(string)
target := params[1].([]string)
value, dataType, _, err := jsonparser.Get( value, dataType, _, err := jsonparser.Get(
jsonparser.StringToBytes(jsblob), jsonparser.StringToBytes(jsblob),
target..., target...,
@ -20,45 +23,62 @@ func JsonExtractLib(jsblob string, target ...string) string {
if err != nil { if err != nil {
if errors.Is(err, jsonparser.KeyPathNotFoundError) { if errors.Is(err, jsonparser.KeyPathNotFoundError) {
log.Debugf("%+v doesn't exist", target) log.Debugf("%+v doesn't exist", target)
return "" return "", nil
} }
log.Errorf("jsonExtractLib : %+v : %s", target, err) log.Errorf("jsonExtractLib : %+v : %s", target, err)
return "" return "", nil
} }
if dataType == jsonparser.NotExist { if dataType == jsonparser.NotExist {
log.Debugf("%+v doesn't exist", target) log.Debugf("%+v doesn't exist", target)
return "" return "", nil
} }
strvalue := string(value) strvalue := string(value)
return strvalue return strvalue, nil
} }
func JsonExtractUnescape(jsblob string, target ...string) string { // func JsonExtractUnescape(jsblob string, target ...string) string {
value, err := jsonparser.GetString( func JsonExtractUnescape(params ...any) (any, error) {
var value string
var err error
jsblob := params[0].(string)
switch v := params[1].(type) {
case string:
target := v
value, err = jsonparser.GetString(
jsonparser.StringToBytes(jsblob),
target,
)
case []string:
target := v
value, err = jsonparser.GetString(
jsonparser.StringToBytes(jsblob), jsonparser.StringToBytes(jsblob),
target..., target...,
) )
}
if err != nil { if err != nil {
if errors.Is(err, jsonparser.KeyPathNotFoundError) { if errors.Is(err, jsonparser.KeyPathNotFoundError) {
log.Debugf("%+v doesn't exist", target) log.Debugf("%+v doesn't exist", params[1])
return "" return "", nil
} }
log.Errorf("JsonExtractUnescape : %+v : %s", target, err) log.Errorf("JsonExtractUnescape : %+v : %s", params[1], err)
return "" return "", nil
} }
log.Tracef("extract path %+v", target) log.Tracef("extract path %+v", params[1])
return value return value, nil
} }
func JsonExtract(jsblob string, target string) string { // func JsonExtract(jsblob string, target string) string {
func JsonExtract(params ...any) (any, error) {
jsblob := params[0].(string)
target := params[1].(string)
if !strings.HasPrefix(target, "[") { if !strings.HasPrefix(target, "[") {
target = strings.ReplaceAll(target, "[", ".[") target = strings.ReplaceAll(target, "[", ".[")
} }
fullpath := strings.Split(target, ".") fullpath := strings.Split(target, ".")
log.Tracef("extract path %+v", fullpath) log.Tracef("extract path %+v", fullpath)
return JsonExtractLib(jsblob, fullpath...) return JsonExtractLib(jsblob, fullpath)
} }
func jsonExtractType(jsblob string, target string, t jsonparser.ValueType) ([]byte, error) { func jsonExtractType(jsblob string, target string, t jsonparser.ValueType) ([]byte, error) {
@ -91,13 +111,15 @@ func jsonExtractType(jsblob string, target string, t jsonparser.ValueType) ([]by
return value, nil return value, nil
} }
func JsonExtractSlice(jsblob string, target string) []interface{} { // func JsonExtractSlice(jsblob string, target string) []interface{} {
func JsonExtractSlice(params ...any) (any, error) {
jsblob := params[0].(string)
target := params[1].(string)
value, err := jsonExtractType(jsblob, target, jsonparser.Array) value, err := jsonExtractType(jsblob, target, jsonparser.Array)
if err != nil { if err != nil {
log.Errorf("JsonExtractSlice : %s", err) log.Errorf("JsonExtractSlice : %s", err)
return nil return []interface{}(nil), nil
} }
s := make([]interface{}, 0) s := make([]interface{}, 0)
@ -105,18 +127,20 @@ func JsonExtractSlice(jsblob string, target string) []interface{} {
err = json.Unmarshal(value, &s) err = json.Unmarshal(value, &s)
if err != nil { if err != nil {
log.Errorf("JsonExtractSlice: could not convert '%s' to slice: %s", value, err) log.Errorf("JsonExtractSlice: could not convert '%s' to slice: %s", value, err)
return nil return []interface{}(nil), nil
} }
return s return s, nil
} }
func JsonExtractObject(jsblob string, target string) map[string]interface{} { // func JsonExtractObject(jsblob string, target string) map[string]interface{} {
func JsonExtractObject(params ...any) (any, error) {
jsblob := params[0].(string)
target := params[1].(string)
value, err := jsonExtractType(jsblob, target, jsonparser.Object) value, err := jsonExtractType(jsblob, target, jsonparser.Object)
if err != nil { if err != nil {
log.Errorf("JsonExtractObject: %s", err) log.Errorf("JsonExtractObject: %s", err)
return nil return map[string]interface{}(nil), nil
} }
s := make(map[string]interface{}) s := make(map[string]interface{})
@ -124,16 +148,18 @@ func JsonExtractObject(jsblob string, target string) map[string]interface{} {
err = json.Unmarshal(value, &s) err = json.Unmarshal(value, &s)
if err != nil { if err != nil {
log.Errorf("JsonExtractObject: could not convert '%s' to map[string]interface{}: %s", value, err) log.Errorf("JsonExtractObject: could not convert '%s' to map[string]interface{}: %s", value, err)
return nil return map[string]interface{}(nil), nil
} }
return s return s, nil
} }
func ToJson(obj interface{}) string { // func ToJson(obj interface{}) string {
func ToJson(params ...any) (any, error) {
obj := params[0]
b, err := json.Marshal(obj) b, err := json.Marshal(obj)
if err != nil { if err != nil {
log.Errorf("ToJson : %s", err) log.Errorf("ToJson : %s", err)
return "" return "", nil
} }
return string(b) return string(b), nil
} }

89
pkg/exprhelpers/jsonextract_test.go
View file

@ -4,6 +4,7 @@ import (
"log" "log"
"testing" "testing"
"github.com/antonmedv/expr"
"github.com/stretchr/testify/assert" "github.com/stretchr/testify/assert"
) )
@ -22,34 +23,43 @@ func TestJsonExtract(t *testing.T) {
jsonBlob string jsonBlob string
targetField string targetField string
expectResult string expectResult string
expr string
}{ }{
{ {
name: "basic json extract", name: "basic json extract",
jsonBlob: `{"test" : "1234"}`, jsonBlob: `{"test" : "1234"}`,
targetField: "test", targetField: "test",
expectResult: "1234", expectResult: "1234",
expr: "JsonExtract(blob, target)",
}, },
{ {
name: "basic json extract with non existing field", name: "basic json extract with non existing field",
jsonBlob: `{"test" : "1234"}`, jsonBlob: `{"test" : "1234"}`,
targetField: "non_existing_field", targetField: "non_existing_field",
expectResult: "", expectResult: "",
expr: "JsonExtract(blob, target)",
}, },
{ {
name: "extract subfield", name: "extract subfield",
jsonBlob: `{"test" : {"a": "b"}}`, jsonBlob: `{"test" : {"a": "b"}}`,
targetField: "test.a", targetField: "test.a",
expectResult: "b", expectResult: "b",
expr: "JsonExtract(blob, target)",
}, },
} }
for _, test := range tests { for _, test := range tests {
result := JsonExtract(test.jsonBlob, test.targetField) t.Run(test.name, func(t *testing.T) {
isOk := assert.Equal(t, test.expectResult, result) env := map[string]interface{}{
if !isOk { "blob": test.jsonBlob,
t.Fatalf("test '%s' failed", test.name) "target": test.targetField,
} }
log.Printf("test '%s' : OK", test.name) vm, err := expr.Compile(test.expr, GetExprOptions(env)...)
assert.NoError(t, err)
out, err := expr.Run(vm, env)
assert.NoError(t, err)
assert.Equal(t, test.expectResult, out)
})
} }
} }
@ -68,28 +78,36 @@ func TestJsonExtractUnescape(t *testing.T) {
jsonBlob string jsonBlob string
targetField string targetField string
expectResult string expectResult string
expr string
}{ }{
{ {
name: "basic json extract", name: "basic json extract",
jsonBlob: `{"log" : "\"GET /JBNwtQ6i.blt HTTP/1.1\" 200 13 \"-\" \"Craftbot\""}`, jsonBlob: `{"log" : "\"GET /JBNwtQ6i.blt HTTP/1.1\" 200 13 \"-\" \"Craftbot\""}`,
targetField: "log", targetField: "log",
expectResult: "\"GET /JBNwtQ6i.blt HTTP/1.1\" 200 13 \"-\" \"Craftbot\"", expectResult: "\"GET /JBNwtQ6i.blt HTTP/1.1\" 200 13 \"-\" \"Craftbot\"",
expr: "JsonExtractUnescape(blob, target)",
}, },
{ {
name: "basic json extract with non existing field", name: "basic json extract with non existing field",
jsonBlob: `{"test" : "1234"}`, jsonBlob: `{"test" : "1234"}`,
targetField: "non_existing_field", targetField: "non_existing_field",
expectResult: "", expectResult: "",
expr: "JsonExtractUnescape(blob, target)",
}, },
} }
for _, test := range tests { for _, test := range tests {
result := JsonExtractUnescape(test.jsonBlob, test.targetField) t.Run(test.name, func(t *testing.T) {
isOk := assert.Equal(t, test.expectResult, result) env := map[string]interface{}{
if !isOk { "blob": test.jsonBlob,
t.Fatalf("test '%s' failed", test.name) "target": test.targetField,
} }
log.Printf("test '%s' : OK", test.name) vm, err := expr.Compile(test.expr, GetExprOptions(env)...)
assert.NoError(t, err)
out, err := expr.Run(vm, env)
assert.NoError(t, err)
assert.Equal(t, test.expectResult, out)
})
} }
} }
@ -108,38 +126,50 @@ func TestJsonExtractSlice(t *testing.T) {
jsonBlob string jsonBlob string
targetField string targetField string
expectResult []interface{} expectResult []interface{}
expr string
}{ }{
{ {
name: "try to extract a string as a slice", name: "try to extract a string as a slice",
jsonBlob: `{"test" : "1234"}`, jsonBlob: `{"test" : "1234"}`,
targetField: "test", targetField: "test",
expectResult: nil, expectResult: nil,
expr: "JsonExtractSlice(blob, target)",
}, },
{ {
name: "basic json slice extract", name: "basic json slice extract",
jsonBlob: `{"test" : ["1234"]}`, jsonBlob: `{"test" : ["1234"]}`,
targetField: "test", targetField: "test",
expectResult: []interface{}{"1234"}, expectResult: []interface{}{"1234"},
expr: "JsonExtractSlice(blob, target)",
}, },
{ {
name: "extract with complex expression", name: "extract with complex expression",
jsonBlob: `{"test": {"foo": [{"a":"b"}]}}`, jsonBlob: `{"test": {"foo": [{"a":"b"}]}}`,
targetField: "test.foo", targetField: "test.foo",
expectResult: []interface{}{map[string]interface{}{"a": "b"}}, expectResult: []interface{}{map[string]interface{}{"a": "b"}},
expr: "JsonExtractSlice(blob, target)",
}, },
{ {
name: "extract non-existing key", name: "extract non-existing key",
jsonBlob: `{"test: "11234"}`, jsonBlob: `{"test: "11234"}`,
targetField: "foo", targetField: "foo",
expectResult: nil, expectResult: nil,
expr: "JsonExtractSlice(blob, target)",
}, },
} }
for _, test := range tests { for _, test := range tests {
test := test test := test
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
result := JsonExtractSlice(test.jsonBlob, test.targetField) env := map[string]interface{}{
assert.Equal(t, test.expectResult, result) "blob": test.jsonBlob,
"target": test.targetField,
}
vm, err := expr.Compile(test.expr, GetExprOptions(env)...)
assert.NoError(t, err)
out, err := expr.Run(vm, env)
assert.NoError(t, err)
assert.Equal(t, test.expectResult, out)
}) })
} }
} }
@ -159,61 +189,79 @@ func TestJsonExtractObject(t *testing.T) {
jsonBlob string jsonBlob string
targetField string targetField string
expectResult map[string]interface{} expectResult map[string]interface{}
expr string
}{ }{
{ {
name: "try to extract a string as an object", name: "try to extract a string as an object",
jsonBlob: `{"test" : "1234"}`, jsonBlob: `{"test" : "1234"}`,
targetField: "test", targetField: "test",
expectResult: nil, expectResult: nil,
expr: "JsonExtractObject(blob, target)",
}, },
{ {
name: "basic json object extract", name: "basic json object extract",
jsonBlob: `{"test" : {"1234": {"foo": "bar"}}}`, jsonBlob: `{"test" : {"1234": {"foo": "bar"}}}`,
targetField: "test", targetField: "test",
expectResult: map[string]interface{}{"1234": map[string]interface{}{"foo": "bar"}}, expectResult: map[string]interface{}{"1234": map[string]interface{}{"foo": "bar"}},
expr: "JsonExtractObject(blob, target)",
}, },
{ {
name: "extract with complex expression", name: "extract with complex expression",
jsonBlob: `{"test": {"foo": [{"a":"b"}]}}`, jsonBlob: `{"test": {"foo": [{"a":"b"}]}}`,
targetField: "test.foo[0]", targetField: "test.foo[0]",
expectResult: map[string]interface{}{"a": "b"}, expectResult: map[string]interface{}{"a": "b"},
expr: "JsonExtractObject(blob, target)",
}, },
} }
for _, test := range tests { for _, test := range tests {
test := test test := test
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
result := JsonExtractObject(test.jsonBlob, test.targetField) env := map[string]interface{}{
assert.Equal(t, test.expectResult, result) "blob": test.jsonBlob,
"target": test.targetField,
}
vm, err := expr.Compile(test.expr, GetExprOptions(env)...)
assert.NoError(t, err)
out, err := expr.Run(vm, env)
assert.NoError(t, err)
assert.Equal(t, test.expectResult, out)
}) })
} }
} }
func TestToJson(t *testing.T) { func TestToJson(t *testing.T) {
err := Init(nil)
assert.NoError(t, err)
tests := []struct { tests := []struct {
name string name string
obj interface{} obj interface{}
expectResult string expectResult string
expr string
}{ }{
{ {
name: "convert int", name: "convert int",
obj: 42, obj: 42,
expectResult: "42", expectResult: "42",
expr: "ToJsonString(obj)",
}, },
{ {
name: "convert slice", name: "convert slice",
obj: []string{"foo", "bar"}, obj: []string{"foo", "bar"},
expectResult: `["foo","bar"]`, expectResult: `["foo","bar"]`,
expr: "ToJsonString(obj)",
}, },
{ {
name: "convert map", name: "convert map",
obj: map[string]string{"foo": "bar"}, obj: map[string]string{"foo": "bar"},
expectResult: `{"foo":"bar"}`, expectResult: `{"foo":"bar"}`,
expr: "ToJsonString(obj)",
}, },
{ {
name: "convert struct", name: "convert struct",
obj: struct{ Foo string }{"bar"}, obj: struct{ Foo string }{"bar"},
expectResult: `{"Foo":"bar"}`, expectResult: `{"Foo":"bar"}`,
expr: "ToJsonString(obj)",
}, },
{ {
name: "convert complex struct", name: "convert complex struct",
@ -233,19 +281,26 @@ func TestToJson(t *testing.T) {
Bla: []string{"foo", "bar"}, Bla: []string{"foo", "bar"},
}, },
expectResult: `{"Foo":"bar","Bar":{"Baz":"baz"},"Bla":["foo","bar"]}`, expectResult: `{"Foo":"bar","Bar":{"Baz":"baz"},"Bla":["foo","bar"]}`,
expr: "ToJsonString(obj)",
}, },
{ {
name: "convert invalid type", name: "convert invalid type",
obj: func() {}, obj: func() {},
expectResult: "", expectResult: "",
expr: "ToJsonString(obj)",
}, },
} }
for _, test := range tests { for _, test := range tests {
test := test
t.Run(test.name, func(t *testing.T) { t.Run(test.name, func(t *testing.T) {
result := ToJson(test.obj) env := map[string]interface{}{
assert.Equal(t, test.expectResult, result) "obj": test.obj,
}
vm, err := expr.Compile(test.expr, GetExprOptions(env)...)
assert.NoError(t, err)
out, err := expr.Run(vm, env)
assert.NoError(t, err)
assert.Equal(t, test.expectResult, out)
}) })
} }
} }

69
pkg/exprhelpers/strings.go Normal file
View file

@ -0,0 +1,69 @@
package exprhelpers
import "strings"
//Wrappers for stdlib strings function exposed in expr
func Fields(params ...any) (any, error) {
return strings.Fields(params[0].(string)), nil
}
func Index(params ...any) (any, error) {
return strings.Index(params[0].(string), params[1].(string)), nil
}
func IndexAny(params ...any) (any, error) {
return strings.IndexAny(params[0].(string), params[1].(string)), nil
}
func Join(params ...any) (any, error) {
return strings.Join(params[0].([]string), params[1].(string)), nil
}
func Split(params ...any) (any, error) {
return strings.Split(params[0].(string), params[1].(string)), nil
}
func SplitAfter(params ...any) (any, error) {
return strings.SplitAfter(params[0].(string), params[1].(string)), nil
}
func SplitAfterN(params ...any) (any, error) {
return strings.SplitAfterN(params[0].(string), params[1].(string), params[2].(int)), nil
}
func SplitN(params ...any) (any, error) {
return strings.SplitN(params[0].(string), params[1].(string), params[2].(int)), nil
}
func Replace(params ...any) (any, error) {
return strings.Replace(params[0].(string), params[1].(string), params[2].(string), params[3].(int)), nil
}
func ReplaceAll(params ...any) (any, error) {
return strings.ReplaceAll(params[0].(string), params[1].(string), params[2].(string)), nil
}
func Trim(params ...any) (any, error) {
return strings.Trim(params[0].(string), params[1].(string)), nil
}
func TrimLeft(params ...any) (any, error) {
return strings.TrimLeft(params[0].(string), params[1].(string)), nil
}
func TrimPrefix(params ...any) (any, error) {
return strings.TrimPrefix(params[0].(string), params[1].(string)), nil
}
func TrimRight(params ...any) (any, error) {
return strings.TrimRight(params[0].(string), params[1].(string)), nil
}
func TrimSpace(params ...any) (any, error) {
return strings.TrimSpace(params[0].(string)), nil
}
func TrimSuffix(params ...any) (any, error) {
return strings.TrimSuffix(params[0].(string), params[1].(string)), nil
}

8
pkg/exprhelpers/visitor.go
View file

@ -81,7 +81,7 @@ func (v *visitor) Visit(node *ast.Node) {
/* /*
Build reconstruct all the variables used in a filter (to display their content later). Build reconstruct all the variables used in a filter (to display their content later).
*/ */
func (v *visitor) Build(filter string, exprEnv expr.Option) (*ExprDebugger, error) { func (v *visitor) Build(filter string, exprEnv ...expr.Option) (*ExprDebugger, error) {
var expressions []*expression var expressions []*expression
ret := &ExprDebugger{ ret := &ExprDebugger{
filter: filter, filter: filter,
@ -105,7 +105,7 @@ func (v *visitor) Build(filter string, exprEnv expr.Option) (*ExprDebugger, erro
} }
toBuild := strings.Join(variable, ".") toBuild := strings.Join(variable, ".")
v.logger.Debugf("compiling expression '%s'", toBuild) v.logger.Debugf("compiling expression '%s'", toBuild)
debugFilter, err := expr.Compile(toBuild, exprEnv) debugFilter, err := expr.Compile(toBuild, exprEnv...)
if err != nil { if err != nil {
return ret, fmt.Errorf("compilation of variable '%s' failed: %v", toBuild, err) return ret, fmt.Errorf("compilation of variable '%s' failed: %v", toBuild, err)
} }
@ -153,9 +153,9 @@ func (e *ExprDebugger) Run(logger *logrus.Entry, filterResult bool, exprEnv map[
} }
// NewDebugger is the exported function that build the debuggers expressions // NewDebugger is the exported function that build the debuggers expressions
func NewDebugger(filter string, exprEnv expr.Option) (*ExprDebugger, error) { func NewDebugger(filter string, exprEnv ...expr.Option) (*ExprDebugger, error) {
logger := log.WithField("component", "expr-debugger") logger := log.WithField("component", "expr-debugger")
visitor := &visitor{logger: logger} visitor := &visitor{logger: logger}
exprDebugger, err := visitor.Build(filter, exprEnv) exprDebugger, err := visitor.Build(filter, exprEnv...)
return exprDebugger, err return exprDebugger, err
} }

30
pkg/exprhelpers/xml.go
View file

@ -7,13 +7,16 @@ import (
var pathCache = make(map[string]etree.Path) var pathCache = make(map[string]etree.Path)
func XMLGetAttributeValue(xmlString string, path string, attributeName string) string { // func XMLGetAttributeValue(xmlString string, path string, attributeName string) string {
func XMLGetAttributeValue(params ...any) (any, error) {
xmlString := params[0].(string)
path := params[1].(string)
attributeName := params[2].(string)
if _, ok := pathCache[path]; !ok { if _, ok := pathCache[path]; !ok {
compiledPath, err := etree.CompilePath(path) compiledPath, err := etree.CompilePath(path)
if err != nil { if err != nil {
log.Errorf("Could not compile path %s: %s", path, err) log.Errorf("Could not compile path %s: %s", path, err)
return "" return "", nil
} }
pathCache[path] = compiledPath pathCache[path] = compiledPath
} }
@ -23,27 +26,30 @@ func XMLGetAttributeValue(xmlString string, path string, attributeName string) s
err := doc.ReadFromString(xmlString) err := doc.ReadFromString(xmlString)
if err != nil { if err != nil {
log.Tracef("Could not parse XML: %s", err) log.Tracef("Could not parse XML: %s", err)
return "" return "", nil
} }
elem := doc.FindElementPath(compiledPath) elem := doc.FindElementPath(compiledPath)
if elem == nil { if elem == nil {
log.Debugf("Could not find element %s", path) log.Debugf("Could not find element %s", path)
return "" return "", nil
} }
attr := elem.SelectAttr(attributeName) attr := elem.SelectAttr(attributeName)
if attr == nil { if attr == nil {
log.Debugf("Could not find attribute %s", attributeName) log.Debugf("Could not find attribute %s", attributeName)
return "" return "", nil
} }
return attr.Value return attr.Value, nil
} }
func XMLGetNodeValue(xmlString string, path string) string { // func XMLGetNodeValue(xmlString string, path string) string {
func XMLGetNodeValue(params ...any) (any, error) {
xmlString := params[0].(string)
path := params[1].(string)
if _, ok := pathCache[path]; !ok { if _, ok := pathCache[path]; !ok {
compiledPath, err := etree.CompilePath(path) compiledPath, err := etree.CompilePath(path)
if err != nil { if err != nil {
log.Errorf("Could not compile path %s: %s", path, err) log.Errorf("Could not compile path %s: %s", path, err)
return "" return "", nil
} }
pathCache[path] = compiledPath pathCache[path] = compiledPath
} }
@ -53,12 +59,12 @@ func XMLGetNodeValue(xmlString string, path string) string {
err := doc.ReadFromString(xmlString) err := doc.ReadFromString(xmlString)
if err != nil { if err != nil {
log.Tracef("Could not parse XML: %s", err) log.Tracef("Could not parse XML: %s", err)
return "" return "", nil
} }
elem := doc.FindElementPath(compiledPath) elem := doc.FindElementPath(compiledPath)
if elem == nil { if elem == nil {
log.Debugf("Could not find element %s", path) log.Debugf("Could not find element %s", path)
return "" return "", nil
} }
return elem.Text() return elem.Text(), nil
} }

4
pkg/exprhelpers/xml_test.go
View file

@ -57,7 +57,7 @@ func TestXMLGetAttributeValue(t *testing.T) {
} }
for _, test := range tests { for _, test := range tests {
result := XMLGetAttributeValue(test.xmlString, test.path, test.attribute) result, _ := XMLGetAttributeValue(test.xmlString, test.path, test.attribute)
isOk := assert.Equal(t, test.expectResult, result) isOk := assert.Equal(t, test.expectResult, result)
if !isOk { if !isOk {
t.Fatalf("test '%s' failed", test.name) t.Fatalf("test '%s' failed", test.name)
@ -104,7 +104,7 @@ func TestXMLGetNodeValue(t *testing.T) {
} }
for _, test := range tests { for _, test := range tests {
result := XMLGetNodeValue(test.xmlString, test.path) result, _ := XMLGetNodeValue(test.xmlString, test.path)
isOk := assert.Equal(t, test.expectResult, result) isOk := assert.Equal(t, test.expectResult, result)
if !isOk { if !isOk {
t.Fatalf("test '%s' failed", test.name) t.Fatalf("test '%s' failed", test.name)

4
pkg/hubtest/parser_assert.go
View file

@ -153,14 +153,14 @@ func (p *ParserAssert) RunExpression(expression string) (interface{}, error) {
env := map[string]interface{}{"results": *p.TestData} env := map[string]interface{}{"results": *p.TestData}
if runtimeFilter, err = expr.Compile(expression, expr.Env(exprhelpers.GetExprEnv(env))); err != nil { if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil {
return output, err return output, err
} }
//dump opcode in trace level //dump opcode in trace level
log.Tracef("%s", runtimeFilter.Disassemble()) log.Tracef("%s", runtimeFilter.Disassemble())
output, err = expr.Run(runtimeFilter, exprhelpers.GetExprEnv(map[string]interface{}{"results": *p.TestData})) output, err = expr.Run(runtimeFilter, map[string]interface{}{"results": *p.TestData})
if err != nil { if err != nil {
log.Warningf("running : %s", expression) log.Warningf("running : %s", expression)
log.Warningf("runtime error : %s", err) log.Warningf("runtime error : %s", err)

6
pkg/hubtest/scenario_assert.go
View file

@ -148,17 +148,17 @@ func (s *ScenarioAssert) RunExpression(expression string) (interface{}, error) {
env := map[string]interface{}{"results": *s.TestData} env := map[string]interface{}{"results": *s.TestData}
if runtimeFilter, err = expr.Compile(expression, expr.Env(exprhelpers.GetExprEnv(env))); err != nil { if runtimeFilter, err = expr.Compile(expression, exprhelpers.GetExprOptions(env)...); err != nil {
return output, err return output, err
} }
// if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(exprhelpers.GetExprEnv(env))); err != nil { // if debugFilter, err = exprhelpers.NewDebugger(assert, expr.Env(env)); err != nil {
// log.Warningf("Failed building debugher for %s : %s", assert, err) // log.Warningf("Failed building debugher for %s : %s", assert, err)
// } // }
//dump opcode in trace level //dump opcode in trace level
log.Tracef("%s", runtimeFilter.Disassemble()) log.Tracef("%s", runtimeFilter.Disassemble())
output, err = expr.Run(runtimeFilter, exprhelpers.GetExprEnv(map[string]interface{}{"results": *s.TestData})) output, err = expr.Run(runtimeFilter, map[string]interface{}{"results": *s.TestData})
if err != nil { if err != nil {
log.Warningf("running : %s", expression) log.Warningf("running : %s", expression)
log.Warningf("runtime error : %s", err) log.Warningf("runtime error : %s", err)

5
pkg/leakybucket/conditional.go
View file

@ -34,8 +34,7 @@ func (c *ConditionalOverflow) OnBucketInit(g *BucketFactory) error {
} else { } else {
conditionalExprCacheLock.Unlock() conditionalExprCacheLock.Unlock()
//release the lock during compile //release the lock during compile
compiledExpr, err = expr.Compile(g.ConditionalOverflow, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{ compiledExpr, err = expr.Compile(g.ConditionalOverflow, exprhelpers.GetExprOptions(map[string]interface{}{"queue": &Queue{}, "leaky": &Leaky{}, "evt": &types.Event{}})...)
"queue": &Queue{}, "leaky": &Leaky{}, "evt": &types.Event{}})))
if err != nil { if err != nil {
return fmt.Errorf("conditional compile error : %w", err) return fmt.Errorf("conditional compile error : %w", err)
} }
@ -52,7 +51,7 @@ func (c *ConditionalOverflow) AfterBucketPour(b *BucketFactory) func(types.Event
var condition, ok bool var condition, ok bool
if c.ConditionalFilterRuntime != nil { if c.ConditionalFilterRuntime != nil {
l.logger.Debugf("Running condition expression : %s", c.ConditionalFilter) l.logger.Debugf("Running condition expression : %s", c.ConditionalFilter)
ret, err := expr.Run(c.ConditionalFilterRuntime, exprhelpers.GetExprEnv(map[string]interface{}{"evt": &msg, "queue": l.Queue, "leaky": l})) ret, err := expr.Run(c.ConditionalFilterRuntime, map[string]interface{}{"evt": &msg, "queue": l.Queue, "leaky": l})
if err != nil { if err != nil {
l.logger.Errorf("unable to run conditional filter : %s", err) l.logger.Errorf("unable to run conditional filter : %s", err)
return &msg return &msg

10
pkg/leakybucket/manager_load.go
View file

@ -134,7 +134,7 @@ func ValidateFactory(bucketFactory *BucketFactory) error {
err error err error
) )
if bucketFactory.ScopeType.Filter != "" { if bucketFactory.ScopeType.Filter != "" {
if runTimeFilter, err = expr.Compile(bucketFactory.ScopeType.Filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))); err != nil { if runTimeFilter, err = expr.Compile(bucketFactory.ScopeType.Filter, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...); err != nil {
return fmt.Errorf("Error compiling the scope filter: %s", err) return fmt.Errorf("Error compiling the scope filter: %s", err)
} }
bucketFactory.ScopeType.RunTimeFilter = runTimeFilter bucketFactory.ScopeType.RunTimeFilter = runTimeFilter
@ -147,7 +147,7 @@ func ValidateFactory(bucketFactory *BucketFactory) error {
err error err error
) )
if bucketFactory.ScopeType.Filter != "" { if bucketFactory.ScopeType.Filter != "" {
if runTimeFilter, err = expr.Compile(bucketFactory.ScopeType.Filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))); err != nil { if runTimeFilter, err = expr.Compile(bucketFactory.ScopeType.Filter, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...); err != nil {
return fmt.Errorf("Error compiling the scope filter: %s", err) return fmt.Errorf("Error compiling the scope filter: %s", err)
} }
bucketFactory.ScopeType.RunTimeFilter = runTimeFilter bucketFactory.ScopeType.RunTimeFilter = runTimeFilter
@ -286,19 +286,19 @@ func LoadBucket(bucketFactory *BucketFactory, tomb *tomb.Tomb) error {
bucketFactory.logger.Warning("Bucket without filter, abort.") bucketFactory.logger.Warning("Bucket without filter, abort.")
return fmt.Errorf("bucket without filter directive") return fmt.Errorf("bucket without filter directive")
} }
bucketFactory.RunTimeFilter, err = expr.Compile(bucketFactory.Filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))) bucketFactory.RunTimeFilter, err = expr.Compile(bucketFactory.Filter, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
if err != nil { if err != nil {
return fmt.Errorf("invalid filter '%s' in %s : %v", bucketFactory.Filter, bucketFactory.Filename, err) return fmt.Errorf("invalid filter '%s' in %s : %v", bucketFactory.Filter, bucketFactory.Filename, err)
} }
if bucketFactory.Debug { if bucketFactory.Debug {
bucketFactory.ExprDebugger, err = exprhelpers.NewDebugger(bucketFactory.Filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))) bucketFactory.ExprDebugger, err = exprhelpers.NewDebugger(bucketFactory.Filter, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
if err != nil { if err != nil {
log.Errorf("unable to build debug filter for '%s' : %s", bucketFactory.Filter, err) log.Errorf("unable to build debug filter for '%s' : %s", bucketFactory.Filter, err)
} }
} }
if bucketFactory.GroupBy != "" { if bucketFactory.GroupBy != "" {
bucketFactory.RunTimeGroupBy, err = expr.Compile(bucketFactory.GroupBy, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))) bucketFactory.RunTimeGroupBy, err = expr.Compile(bucketFactory.GroupBy, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
if err != nil { if err != nil {
return fmt.Errorf("invalid groupby '%s' in %s : %v", bucketFactory.GroupBy, bucketFactory.Filename, err) return fmt.Errorf("invalid groupby '%s' in %s : %v", bucketFactory.GroupBy, bucketFactory.Filename, err)
} }

9
pkg/leakybucket/manager_run.go
View file

@ -13,7 +13,6 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/antonmedv/expr" "github.com/antonmedv/expr"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
) )
@ -297,8 +296,6 @@ func PourItemToHolders(parsed types.Event, holders []BucketFactory, buckets *Buc
BucketPourCache["OK"] = append(BucketPourCache["OK"], evt.(types.Event)) BucketPourCache["OK"] = append(BucketPourCache["OK"], evt.(types.Event))
} }
cachedExprEnv := exprhelpers.GetExprEnv(map[string]interface{}{"evt": &parsed})
//find the relevant holders (scenarios) //find the relevant holders (scenarios)
for idx := 0; idx < len(holders); idx++ { for idx := 0; idx < len(holders); idx++ {
//for idx, holder := range holders { //for idx, holder := range holders {
@ -306,7 +303,7 @@ func PourItemToHolders(parsed types.Event, holders []BucketFactory, buckets *Buc
//evaluate bucket's condition //evaluate bucket's condition
if holders[idx].RunTimeFilter != nil { if holders[idx].RunTimeFilter != nil {
holders[idx].logger.Tracef("event against holder %d/%d", idx, len(holders)) holders[idx].logger.Tracef("event against holder %d/%d", idx, len(holders))
output, err := expr.Run(holders[idx].RunTimeFilter, cachedExprEnv) output, err := expr.Run(holders[idx].RunTimeFilter, map[string]interface{}{"evt": &parsed})
if err != nil { if err != nil {
holders[idx].logger.Errorf("failed parsing : %v", err) holders[idx].logger.Errorf("failed parsing : %v", err)
return false, fmt.Errorf("leaky failed : %s", err) return false, fmt.Errorf("leaky failed : %s", err)
@ -318,7 +315,7 @@ func PourItemToHolders(parsed types.Event, holders []BucketFactory, buckets *Buc
} }
if holders[idx].Debug { if holders[idx].Debug {
holders[idx].ExprDebugger.Run(holders[idx].logger, condition, cachedExprEnv) holders[idx].ExprDebugger.Run(holders[idx].logger, condition, map[string]interface{}{"evt": &parsed})
} }
if !condition { if !condition {
holders[idx].logger.Debugf("Event leaving node : ko (filter mismatch)") holders[idx].logger.Debugf("Event leaving node : ko (filter mismatch)")
@ -329,7 +326,7 @@ func PourItemToHolders(parsed types.Event, holders []BucketFactory, buckets *Buc
//groupby determines the partition key for the specific bucket //groupby determines the partition key for the specific bucket
var groupby string var groupby string
if holders[idx].RunTimeGroupBy != nil { if holders[idx].RunTimeGroupBy != nil {
tmpGroupBy, err := expr.Run(holders[idx].RunTimeGroupBy, cachedExprEnv) tmpGroupBy, err := expr.Run(holders[idx].RunTimeGroupBy, map[string]interface{}{"evt": &parsed})
if err != nil { if err != nil {
holders[idx].logger.Errorf("failed groupby : %v", err) holders[idx].logger.Errorf("failed groupby : %v", err)
return false, errors.New("leaky failed :/") return false, errors.New("leaky failed :/")

8
pkg/leakybucket/overflow_filter.go
View file

@ -27,8 +27,8 @@ func NewOverflowFilter(g *BucketFactory) (*OverflowFilter, error) {
u := OverflowFilter{} u := OverflowFilter{}
u.Filter = g.OverflowFilter u.Filter = g.OverflowFilter
u.FilterRuntime, err = expr.Compile(u.Filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{
"queue": &Queue{}, "signal": &types.RuntimeAlert{}, "leaky": &Leaky{}}))) u.FilterRuntime, err = expr.Compile(u.Filter, exprhelpers.GetExprOptions(map[string]interface{}{"queue": &Queue{}, "signal": &types.RuntimeAlert{}, "leaky": &Leaky{}})...)
if err != nil { if err != nil {
g.logger.Errorf("Unable to compile filter : %v", err) g.logger.Errorf("Unable to compile filter : %v", err)
return nil, fmt.Errorf("unable to compile filter : %v", err) return nil, fmt.Errorf("unable to compile filter : %v", err)
@ -38,8 +38,8 @@ func NewOverflowFilter(g *BucketFactory) (*OverflowFilter, error) {
func (u *OverflowFilter) OnBucketOverflow(Bucket *BucketFactory) func(*Leaky, types.RuntimeAlert, *Queue) (types.RuntimeAlert, *Queue) { func (u *OverflowFilter) OnBucketOverflow(Bucket *BucketFactory) func(*Leaky, types.RuntimeAlert, *Queue) (types.RuntimeAlert, *Queue) {
return func(l *Leaky, s types.RuntimeAlert, q *Queue) (types.RuntimeAlert, *Queue) { return func(l *Leaky, s types.RuntimeAlert, q *Queue) (types.RuntimeAlert, *Queue) {
el, err := expr.Run(u.FilterRuntime, exprhelpers.GetExprEnv(map[string]interface{}{ el, err := expr.Run(u.FilterRuntime, map[string]interface{}{
"queue": q, "signal": s, "leaky": l})) "queue": q, "signal": s, "leaky": l})
if err != nil { if err != nil {
l.logger.Errorf("Failed running overflow filter: %s", err) l.logger.Errorf("Failed running overflow filter: %s", err)
return s, q return s, q

7
pkg/leakybucket/overflows.go
View file

@ -15,7 +15,6 @@ import (
log "github.com/sirupsen/logrus" log "github.com/sirupsen/logrus"
"github.com/antonmedv/expr" "github.com/antonmedv/expr"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
) )
// SourceFromEvent extracts and formats a valid models.Source object from an Event // SourceFromEvent extracts and formats a valid models.Source object from an Event
@ -52,7 +51,7 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e
*src.Value = v.Range *src.Value = v.Range
} }
if leaky.scopeType.RunTimeFilter != nil { if leaky.scopeType.RunTimeFilter != nil {
retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, exprhelpers.GetExprEnv(map[string]interface{}{"evt": &evt})) retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, map[string]interface{}{"evt": &evt})
if err != nil { if err != nil {
return srcs, errors.Wrapf(err, "while running scope filter") return srcs, errors.Wrapf(err, "while running scope filter")
} }
@ -127,7 +126,7 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e
} else if leaky.scopeType.Scope == types.Range { } else if leaky.scopeType.Scope == types.Range {
src.Value = &src.Range src.Value = &src.Range
if leaky.scopeType.RunTimeFilter != nil { if leaky.scopeType.RunTimeFilter != nil {
retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, exprhelpers.GetExprEnv(map[string]interface{}{"evt": &evt})) retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, map[string]interface{}{"evt": &evt})
if err != nil { if err != nil {
return srcs, errors.Wrapf(err, "while running scope filter") return srcs, errors.Wrapf(err, "while running scope filter")
} }
@ -144,7 +143,7 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e
if leaky.scopeType.RunTimeFilter == nil { if leaky.scopeType.RunTimeFilter == nil {
return srcs, fmt.Errorf("empty scope information") return srcs, fmt.Errorf("empty scope information")
} }
retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, exprhelpers.GetExprEnv(map[string]interface{}{"evt": &evt})) retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, map[string]interface{}{"evt": &evt})
if err != nil { if err != nil {
return srcs, errors.Wrapf(err, "while running scope filter") return srcs, errors.Wrapf(err, "while running scope filter")
} }

10
pkg/leakybucket/reset_filter.go
View file

@ -34,14 +34,14 @@ func (u *CancelOnFilter) OnBucketPour(bucketFactory *BucketFactory) func(types.E
var condition, ok bool var condition, ok bool
if u.CancelOnFilter != nil { if u.CancelOnFilter != nil {
leaky.logger.Tracef("running cancel_on filter") leaky.logger.Tracef("running cancel_on filter")
output, err := expr.Run(u.CancelOnFilter, exprhelpers.GetExprEnv(map[string]interface{}{"evt": &msg})) output, err := expr.Run(u.CancelOnFilter, map[string]interface{}{"evt": &msg})
if err != nil { if err != nil {
leaky.logger.Warningf("cancel_on error : %s", err) leaky.logger.Warningf("cancel_on error : %s", err)
return &msg return &msg
} }
//only run debugger expression if condition is false //only run debugger expression if condition is false
if u.CancelOnFilterDebug != nil { if u.CancelOnFilterDebug != nil {
u.CancelOnFilterDebug.Run(leaky.logger, condition, exprhelpers.GetExprEnv(map[string]interface{}{"evt": &msg})) u.CancelOnFilterDebug.Run(leaky.logger, condition, map[string]interface{}{"evt": &msg})
} }
if condition, ok = output.(bool); !ok { if condition, ok = output.(bool); !ok {
leaky.logger.Warningf("cancel_on, unexpected non-bool return : %T", output) leaky.logger.Warningf("cancel_on, unexpected non-bool return : %T", output)
@ -93,14 +93,16 @@ func (u *CancelOnFilter) OnBucketInit(bucketFactory *BucketFactory) error {
} else { } else {
cancelExprCacheLock.Unlock() cancelExprCacheLock.Unlock()
//release the lock during compile //release the lock during compile
compiledExpr.CancelOnFilter, err = expr.Compile(bucketFactory.CancelOnFilter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}})))
compiledExpr.CancelOnFilter, err = expr.Compile(bucketFactory.CancelOnFilter, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
if err != nil { if err != nil {
bucketFactory.logger.Errorf("reset_filter compile error : %s", err) bucketFactory.logger.Errorf("reset_filter compile error : %s", err)
return err return err
} }
u.CancelOnFilter = compiledExpr.CancelOnFilter u.CancelOnFilter = compiledExpr.CancelOnFilter
if bucketFactory.Debug { if bucketFactory.Debug {
compiledExpr.CancelOnFilterDebug, err = exprhelpers.NewDebugger(bucketFactory.CancelOnFilter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))) compiledExpr.CancelOnFilterDebug, err = exprhelpers.NewDebugger(bucketFactory.CancelOnFilter, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...,
)
if err != nil { if err != nil {
bucketFactory.logger.Errorf("reset_filter debug error : %s", err) bucketFactory.logger.Errorf("reset_filter debug error : %s", err)
return err return err

4
pkg/leakybucket/uniq.go
View file

@ -74,7 +74,7 @@ func (u *Uniq) OnBucketInit(bucketFactory *BucketFactory) error {
} else { } else {
uniqExprCacheLock.Unlock() uniqExprCacheLock.Unlock()
//release the lock during compile //release the lock during compile
compiledExpr, err = expr.Compile(bucketFactory.Distinct, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))) compiledExpr, err = expr.Compile(bucketFactory.Distinct, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
u.DistinctCompiled = compiledExpr u.DistinctCompiled = compiledExpr
uniqExprCacheLock.Lock() uniqExprCacheLock.Lock()
uniqExprCache[bucketFactory.Distinct] = *compiledExpr uniqExprCache[bucketFactory.Distinct] = *compiledExpr
@ -86,7 +86,7 @@ func (u *Uniq) OnBucketInit(bucketFactory *BucketFactory) error {
// getElement computes a string from an event and a filter // getElement computes a string from an event and a filter
func getElement(msg types.Event, cFilter *vm.Program) (string, error) { func getElement(msg types.Event, cFilter *vm.Program) (string, error) {
el, err := expr.Run(cFilter, exprhelpers.GetExprEnv(map[string]interface{}{"evt": &msg})) el, err := expr.Run(cFilter, map[string]interface{}{"evt": &msg})
if err != nil { if err != nil {
return "", err return "", err
} }

4
pkg/parser/enrich_date.go
View file

@ -84,10 +84,10 @@ func ParseDate(in string, p *types.Event, x interface{}, plog *log.Entry) (map[s
} }
timeobj, err := expr.ParseUnixTime(in) timeobj, err := expr.ParseUnixTime(in)
if err == nil { if err == nil {
ret["MarshaledTime"] = timeobj.Format(time.RFC3339) ret["MarshaledTime"] = timeobj.(time.Time).Format(time.RFC3339)
//In time machine, we take the time parsed from the event. In live mode, we keep the timestamp collected at acquisition //In time machine, we take the time parsed from the event. In live mode, we keep the timestamp collected at acquisition
if p.ExpectMode == types.TIMEMACHINE { if p.ExpectMode == types.TIMEMACHINE {
p.Time = timeobj p.Time = timeobj.(time.Time)
} }
return ret, nil return ret, nil
} }

18
pkg/parser/node.go
View file

@ -472,13 +472,13 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
//compile filter if present //compile filter if present
if n.Filter != "" { if n.Filter != "" {
n.RunTimeFilter, err = expr.Compile(n.Filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))) n.RunTimeFilter, err = expr.Compile(n.Filter, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
if err != nil { if err != nil {
return fmt.Errorf("compilation of '%s' failed: %v", n.Filter, err) return fmt.Errorf("compilation of '%s' failed: %v", n.Filter, err)
} }
if n.Debug { if n.Debug {
n.ExprDebugger, err = exprhelpers.NewDebugger(n.Filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))) n.ExprDebugger, err = exprhelpers.NewDebugger(n.Filter, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
if err != nil { if err != nil {
log.Errorf("unable to build debug filter for '%s' : %s", n.Filter, err) log.Errorf("unable to build debug filter for '%s' : %s", n.Filter, err)
} }
@ -530,7 +530,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
/*if grok source is an expression*/ /*if grok source is an expression*/
if n.Grok.ExpValue != "" { if n.Grok.ExpValue != "" {
n.Grok.RunTimeValue, err = expr.Compile(n.Grok.ExpValue, n.Grok.RunTimeValue, err = expr.Compile(n.Grok.ExpValue,
expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))) exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
if err != nil { if err != nil {
return errors.Wrap(err, "while compiling grok's expression") return errors.Wrap(err, "while compiling grok's expression")
} }
@ -542,7 +542,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
for idx := range n.Grok.Statics { for idx := range n.Grok.Statics {
if n.Grok.Statics[idx].ExpValue != "" { if n.Grok.Statics[idx].ExpValue != "" {
n.Grok.Statics[idx].RunTimeValue, err = expr.Compile(n.Grok.Statics[idx].ExpValue, n.Grok.Statics[idx].RunTimeValue, err = expr.Compile(n.Grok.Statics[idx].ExpValue,
expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))) exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
if err != nil { if err != nil {
return err return err
} }
@ -554,13 +554,13 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
/* load data capture (stash) */ /* load data capture (stash) */
for i, stash := range n.Stash { for i, stash := range n.Stash {
n.Stash[i].ValueExpression, err = expr.Compile(stash.Value, n.Stash[i].ValueExpression, err = expr.Compile(stash.Value,
expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))) exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
if err != nil { if err != nil {
return errors.Wrap(err, "while compiling stash value expression") return errors.Wrap(err, "while compiling stash value expression")
} }
n.Stash[i].KeyExpression, err = expr.Compile(stash.Key, n.Stash[i].KeyExpression, err = expr.Compile(stash.Key,
expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))) exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
if err != nil { if err != nil {
return errors.Wrap(err, "while compiling stash key expression") return errors.Wrap(err, "while compiling stash key expression")
} }
@ -607,7 +607,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
/* load statics if present */ /* load statics if present */
for idx := range n.Statics { for idx := range n.Statics {
if n.Statics[idx].ExpValue != "" { if n.Statics[idx].ExpValue != "" {
n.Statics[idx].RunTimeValue, err = expr.Compile(n.Statics[idx].ExpValue, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))) n.Statics[idx].RunTimeValue, err = expr.Compile(n.Statics[idx].ExpValue, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
if err != nil { if err != nil {
n.Logger.Errorf("Statics Compilation failed %v.", err) n.Logger.Errorf("Statics Compilation failed %v.", err)
return err return err
@ -633,11 +633,11 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
} }
for _, filter := range n.Whitelist.Exprs { for _, filter := range n.Whitelist.Exprs {
expression := &ExprWhitelist{} expression := &ExprWhitelist{}
expression.Filter, err = expr.Compile(filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))) expression.Filter, err = expr.Compile(filter, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
if err != nil { if err != nil {
n.Logger.Fatalf("Unable to compile whitelist expression '%s' : %v.", filter, err) n.Logger.Fatalf("Unable to compile whitelist expression '%s' : %v.", filter, err)
} }
expression.ExprDebugger, err = exprhelpers.NewDebugger(filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))) expression.ExprDebugger, err = exprhelpers.NewDebugger(filter, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
if err != nil { if err != nil {
log.Errorf("unable to build debug filter for '%s' : %s", filter, err) log.Errorf("unable to build debug filter for '%s' : %s", filter, err)
} }

9
pkg/parser/runtime.go
View file

@ -13,7 +13,6 @@ import (
"sync" "sync"
"time" "time"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/types" "github.com/crowdsecurity/crowdsec/pkg/types"
"strconv" "strconv"
@ -118,14 +117,12 @@ func (n *Node) ProcessStatics(statics []types.ExtraField, event *types.Event) er
var value string var value string
clog := n.Logger clog := n.Logger
cachedExprEnv := exprhelpers.GetExprEnv(map[string]interface{}{"evt": event})
for _, static := range statics { for _, static := range statics {
value = "" value = ""
if static.Value != "" { if static.Value != "" {
value = static.Value value = static.Value
} else if static.RunTimeValue != nil { } else if static.RunTimeValue != nil {
output, err := expr.Run(static.RunTimeValue, cachedExprEnv) output, err := expr.Run(static.RunTimeValue, map[string]interface{}{"evt": event})
if err != nil { if err != nil {
clog.Warningf("failed to run RunTimeValue : %v", err) clog.Warningf("failed to run RunTimeValue : %v", err)
continue continue
@ -272,8 +269,6 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error)
log.Tracef("INPUT '%s'", event.Line.Raw) log.Tracef("INPUT '%s'", event.Line.Raw)
} }
cachedExprEnv := exprhelpers.GetExprEnv(map[string]interface{}{"evt": &event})
if ParseDump { if ParseDump {
if StageParseCache == nil { if StageParseCache == nil {
StageParseMutex.Lock() StageParseMutex.Lock()
@ -321,7 +316,7 @@ func Parse(ctx UnixParserCtx, xp types.Event, nodes []Node) (types.Event, error)
if ctx.Profiling { if ctx.Profiling {
node.Profiling = true node.Profiling = true
} }
ret, err := node.process(&event, ctx, cachedExprEnv) ret, err := node.process(&event, ctx, map[string]interface{}{"evt": &event})
if err != nil { if err != nil {
clog.Errorf("Error while processing node : %v", err) clog.Errorf("Error while processing node : %v", err)
return event, err return event, err