beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

add IpToRange helpers and allows to have an expression with scope Range (#1260)

Browse source 
* add IpToRange helpers and allows to have an expression with scope Range
This commit is contained in:
AlteredCoder 2022-02-14 16:50:52 +01:00 committed by GitHub
parent 40ab8fa738
commit 5a0843852a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 217 additions and 15 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
go.mod
View file

@ -70,6 +70,7 @@ require (
github.com/ahmetalpbalkan/dlog v0.0.0-20170105205344-4fb5f8204f26 // indirect
github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/c-robinson/iplib v1.0.3 // indirect
github.com/cespare/xxhash/v2 v2.1.1 // indirect
github.com/containerd/containerd v1.4.3 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect

2
go.sum
View file

@ -90,6 +90,8 @@ github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kB
github.com/bketelsen/crypt v0.0.3-0.20200106085610-5cbc8cc4026c/go.mod h1:MKsuJmJgSg28kpZDP6UIiPt0e0Oz0kqKNGyRaWEPv84=
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/c-robinson/iplib v1.0.3 h1:NG0UF0GoEsrC1/vyfX1Lx2Ss7CySWl3KqqXh3q4DdPU=
github.com/c-robinson/iplib v1.0.3/go.mod h1:i3LuuFL1hRT5gFpBRnEydzw8R6yhGkF4szNDIbF8pgo=
github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ=
github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=

24
pkg/exprhelpers/exprlib.go
View file

@ -12,6 +12,8 @@ import (
"strings"
"time"
"github.com/c-robinson/iplib"
"github.com/davecgh/go-spew/spew"
log "github.com/sirupsen/logrus"
)
@ -53,6 +55,7 @@ func GetExprEnv(ctx map[string]interface{}) map[string]interface{} {
"QueryUnescape": QueryUnescape,
"PathEscape": PathEscape,
"QueryEscape": QueryEscape,
"IpToRange": IpToRange,
}
for k, v := range ctx {
ExprLib[k] = v
@ -175,6 +178,27 @@ func IpInRange(ip string, ipRange string) bool {
return false
}
func IpToRange(ip string, cidr string) string {
cidr = strings.TrimPrefix(cidr, "/")
mask, err := strconv.Atoi(cidr)
if err != nil {
log.Errorf("bad cidr '%s': %s", cidr, err)
return ""
}
ipAddr := net.ParseIP(ip)
if ipAddr == nil {
log.Errorf("can't parse IP address '%s'", ip)
return ""
}
ipRange := iplib.NewNet(ipAddr, mask)
if ipRange.IP() == nil {
log.Errorf("can't get cidr '%s' of '%s'", cidr, ip)
return ""
}
return ipRange.String()
}
func TimeNow() string {
return time.Now().UTC().Format(time.RFC3339)
}

76
pkg/exprhelpers/exprlib_test.go
View file

@ -342,6 +342,82 @@ func TestIpInRange(t *testing.T) {
}
func TestIpToRange(t *testing.T) {
tests := []struct {
name string
env map[string]interface{}
code string
result string
err string
}{
{
name: "IpToRange() test: IPv4",
env: map[string]interface{}{
"ip": "192.168.1.1",
"netmask": "16",
"IpToRange": IpToRange,
},
code: "IpToRange(ip, netmask)",
result: "192.168.0.0/16",
err: "",
},
{
name: "IpToRange() test: IPv6",
env: map[string]interface{}{
"ip": "2001:db8::1",
"netmask": "/64",
"IpToRange": IpToRange,
},
code: "IpToRange(ip, netmask)",
result: "2001:db8::/64",
err: "",
},
{
name: "IpToRange() test: malformed netmask",
env: map[string]interface{}{
"ip": "192.168.0.1",
"netmask": "test",
"IpToRange": IpToRange,
},
code: "IpToRange(ip, netmask)",
result: "",
err: "",
},
{
name: "IpToRange() test: malformed IP",
env: map[string]interface{}{
"ip": "a.b.c.d",
"netmask": "24",
"IpToRange": IpToRange,
},
code: "IpToRange(ip, netmask)",
result: "",
err: "",
},
{
name: "IpToRange() test: too high netmask",
env: map[string]interface{}{
"ip": "192.168.1.1",
"netmask": "35",
"IpToRange": IpToRange,
},
code: "IpToRange(ip, netmask)",
result: "",
err: "",
},
}
for _, test := range tests {
program, err := expr.Compile(test.code, expr.Env(test.env))
require.NoError(t, err)
output, err := expr.Run(program, test.env)
require.NoError(t, err)
require.Equal(t, test.result, output)
log.Printf("test '%s' : OK", test.name)
}
}
func TestAtof(t *testing.T) {
testFloat := "1.5"
expectedFloat := 1.5

13
pkg/leakybucket/manager_load.go
View file

@ -114,17 +114,30 @@ func ValidateFactory(bucketFactory *BucketFactory) error {
bucketFactory.ScopeType.Scope = types.Ip
case types.Ip:
case types.Range:
var (
runTimeFilter *vm.Program
err error
)
if bucketFactory.ScopeType.Filter != "" {
if runTimeFilter, err = expr.Compile(bucketFactory.ScopeType.Filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))); err != nil {
return fmt.Errorf("Error compiling the scope filter: %s", err)
}
bucketFactory.ScopeType.RunTimeFilter = runTimeFilter
}
default:
//Compile the scope filter
var (
runTimeFilter *vm.Program
err error
)
if bucketFactory.ScopeType.Filter != "" {
if runTimeFilter, err = expr.Compile(bucketFactory.ScopeType.Filter, expr.Env(exprhelpers.GetExprEnv(map[string]interface{}{"evt": &types.Event{}}))); err != nil {
return fmt.Errorf("Error compiling the scope filter: %s", err)
}
bucketFactory.ScopeType.RunTimeFilter = runTimeFilter
}
}
return nil
}

32
pkg/leakybucket/overflows.go
View file

@ -20,7 +20,6 @@ import (
//SourceFromEvent extracts and formats a valid models.Source object from an Event
func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, error) {
srcs := make(map[string]models.Source)
/*if it's already an overflow, we have properly formatted sources.
we can just twitch them to reflect the requested scope*/
if evt.Type == types.OVFLW {
@ -37,7 +36,6 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e
if leaky.scopeType.Scope == types.Range {
/*the original bucket was target IPs, check that we do have range*/
if *v.Scope == types.Ip {
if v.Range != "" {
src := models.Source{}
src.AsName = v.AsName
src.AsNumber = v.AsNumber
@ -47,8 +45,23 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e
src.Range = v.Range
src.Value = new(string)
src.Scope = new(string)
*src.Value = v.Range
*src.Scope = leaky.scopeType.Scope
*src.Value = ""
if v.Range != "" {
*src.Value = v.Range
}
if leaky.scopeType.RunTimeFilter != nil {
retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, exprhelpers.GetExprEnv(map[string]interface{}{"evt": &evt}))
if err != nil {
return srcs, errors.Wrapf(err, "while running scope filter")
}
value, ok := retValue.(string)
if !ok {
value = ""
}
src.Value = &value
}
if *src.Value != "" {
srcs[*src.Value] = src
} else {
log.Warningf("bucket %s requires scope Range, but none was provided. It seems that the %s wasn't enriched to include its range.", leaky.Name, *v.Value)
@ -112,6 +125,18 @@ func SourceFromEvent(evt types.Event, leaky *Leaky) (map[string]models.Source, e
src.Value = &src.IP
} else if leaky.scopeType.Scope == types.Range {
src.Value = &src.Range
if leaky.scopeType.RunTimeFilter != nil {
retValue, err := expr.Run(leaky.scopeType.RunTimeFilter, exprhelpers.GetExprEnv(map[string]interface{}{"evt": &evt}))
if err != nil {
return srcs, errors.Wrapf(err, "while running scope filter")
}
value, ok := retValue.(string)
if !ok {
value = ""
}
src.Value = &value
}
}
srcs[*src.Value] = src
default:
@ -209,7 +234,6 @@ func alertFormatSource(leaky *Leaky, queue *Queue) (map[string]models.Source, st
//NewAlert will generate a RuntimeAlert and its APIAlert(s) from a bucket that overflowed
func NewAlert(leaky *Leaky, queue *Queue) (types.RuntimeAlert, error) {
var runtimeAlert types.RuntimeAlert
leaky.logger.Tracef("Overflow (start: %s, end: %s)", leaky.First_ts, leaky.Ovflw_ts)

14
pkg/leakybucket/tests/leaky-scope-range-expression/bucket.yaml Normal file
View file

@ -0,0 +1,14 @@
type: leaky
debug: true
name: test/leaky-scope-range-expression
description: "Leaky with scope range-expression"
filter: "evt.Line.Labels.type =='testlog'"
leakspeed: "10s"
capacity: 1
groupby: evt.Meta.source_ip
labels:
type: overflow_1
scope:
type: Range
expression: IpToRange(evt.Meta.source_ip, "/16")

1
pkg/leakybucket/tests/leaky-scope-range-expression/scenarios.yaml Normal file
View file

@ -0,0 +1 @@
- filename: {{.TestDirectory}}/bucket.yaml

47
pkg/leakybucket/tests/leaky-scope-range-expression/test.json Normal file
View file

@ -0,0 +1,47 @@
{
"lines": [
{
"Line": {
"Labels": {
"type": "testlog"
},
"Raw": "xxheader VALUE1 trailing stuff"
},
"MarshaledTime": "2020-01-01T10:00:00+00:00",
"Meta": {
"source_ip": "192.168.1.1"
}
},
{
"Line": {
"Labels": {
"type": "testlog"
},
"Raw": "xxheader VALUE2 trailing stuff"
},
"MarshaledTime": "2020-01-01T10:00:05+00:00",
"Meta": {
"source_ip": "192.168.1.1"
}
}
],
"results": [
{
"Alert": {
"sources": {
"192.168.0.0/16": {
"scope": "Range",
"value": "192.168.0.0/16",
"ip": "192.168.1.1"
}
},
"Alert" : {
"scenario": "test/leaky-scope-range-expression",
"events_count": 2
}
}
}
]
}