Compare commits
10 commits
master
...
appsec-pro
Author | SHA1 | Date | |
---|---|---|---|
|
5f0044d276 | ||
|
bdc62d3715 | ||
|
927f2e4c48 | ||
|
170e5e8dd8 | ||
|
127969d325 | ||
|
81976c6982 | ||
|
b370925967 | ||
|
7447b8bf04 | ||
|
f6038feabe | ||
|
c71cb4bcda |
17 changed files with 275 additions and 118 deletions
|
@ -19,6 +19,7 @@ import (
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/appsec"
|
"github.com/crowdsecurity/crowdsec/pkg/appsec"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
||||||
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
|
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/parser"
|
"github.com/crowdsecurity/crowdsec/pkg/parser"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
|
@ -32,6 +33,13 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, []
|
||||||
return nil, nil, fmt.Errorf("while loading context: %w", err)
|
return nil, nil, fmt.Errorf("while loading context: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
err = exprhelpers.GeoIPInit(hub.GetDataDir())
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
//GeoIP databases are not mandatory, do not make crowdsec fail if they are not present
|
||||||
|
log.Warnf("unable to initialize GeoIP: %s", err)
|
||||||
|
}
|
||||||
|
|
||||||
// Start loading configs
|
// Start loading configs
|
||||||
csParsers := parser.NewParsers(hub)
|
csParsers := parser.NewParsers(hub)
|
||||||
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
|
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
|
||||||
|
|
|
@ -177,6 +177,9 @@ func ShutdownCrowdsecRoutines() error {
|
||||||
// He's dead, Jim.
|
// He's dead, Jim.
|
||||||
crowdsecTomb.Kill(nil)
|
crowdsecTomb.Kill(nil)
|
||||||
|
|
||||||
|
// close the potential geoips reader we have to avoid leaking ressources on reload
|
||||||
|
exprhelpers.GeoIPClose()
|
||||||
|
|
||||||
return reterr
|
return reterr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -3,18 +3,32 @@ package appsecacquisition
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
|
"net"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/crowdsecurity/coraza/v3/collection"
|
"github.com/crowdsecurity/coraza/v3/collection"
|
||||||
"github.com/crowdsecurity/coraza/v3/types/variables"
|
"github.com/crowdsecurity/coraza/v3/types/variables"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/appsec"
|
"github.com/crowdsecurity/crowdsec/pkg/appsec"
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/models"
|
"github.com/crowdsecurity/crowdsec/pkg/models"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
"github.com/crowdsecurity/go-cs-lib/ptr"
|
"github.com/crowdsecurity/go-cs-lib/ptr"
|
||||||
|
"github.com/oschwald/geoip2-golang"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
func appendMeta(meta models.Meta, key string, value string) models.Meta {
|
||||||
|
if value == "" {
|
||||||
|
return meta
|
||||||
|
}
|
||||||
|
meta = append(meta, &models.MetaItems0{
|
||||||
|
Key: key,
|
||||||
|
Value: value,
|
||||||
|
})
|
||||||
|
return meta
|
||||||
|
}
|
||||||
|
|
||||||
func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) {
|
func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) {
|
||||||
//if the request didnd't trigger inband rules, we don't want to generate an event to LAPI/CAPI
|
//if the request didnd't trigger inband rules, we don't want to generate an event to LAPI/CAPI
|
||||||
if !inEvt.Appsec.HasInBandMatches {
|
if !inEvt.Appsec.HasInBandMatches {
|
||||||
|
@ -29,12 +43,40 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) {
|
||||||
Scope: ptr.Of(types.Ip),
|
Scope: ptr.Of(types.Ip),
|
||||||
}
|
}
|
||||||
|
|
||||||
|
asndata, err := exprhelpers.GeoIPASNEnrich(inEvt.Parsed["source_ip"])
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Unable to enrich ip '%s'", inEvt.Parsed["source_ip"])
|
||||||
|
} else if asndata != nil {
|
||||||
|
record := asndata.(*geoip2.ASN)
|
||||||
|
source.AsName = record.AutonomousSystemOrganization
|
||||||
|
source.AsNumber = fmt.Sprintf("%d", record.AutonomousSystemNumber)
|
||||||
|
}
|
||||||
|
|
||||||
|
cityData, err := exprhelpers.GeoIPEnrich(inEvt.Parsed["source_ip"])
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Unable to enrich ip '%s'", inEvt.Parsed["source_ip"])
|
||||||
|
} else if cityData != nil {
|
||||||
|
record := cityData.(*geoip2.City)
|
||||||
|
source.Cn = record.Country.IsoCode
|
||||||
|
source.Latitude = float32(record.Location.Latitude)
|
||||||
|
source.Longitude = float32(record.Location.Longitude)
|
||||||
|
}
|
||||||
|
|
||||||
|
rangeData, err := exprhelpers.GeoIPRangeEnrich(inEvt.Parsed["source_ip"])
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Unable to enrich ip '%s'", inEvt.Parsed["source_ip"])
|
||||||
|
} else if rangeData != nil {
|
||||||
|
record := rangeData.(*net.IPNet)
|
||||||
|
source.Range = record.String()
|
||||||
|
}
|
||||||
|
|
||||||
evt.Overflow.Sources = make(map[string]models.Source)
|
evt.Overflow.Sources = make(map[string]models.Source)
|
||||||
evt.Overflow.Sources["ip"] = source
|
evt.Overflow.Sources["ip"] = source
|
||||||
|
|
||||||
alert := models.Alert{}
|
alert := models.Alert{}
|
||||||
alert.Capacity = ptr.Of(int32(1))
|
alert.Capacity = ptr.Of(int32(1))
|
||||||
alert.Events = make([]*models.Event, 0)
|
alert.Events = make([]*models.Event, len(evt.Appsec.GetRuleIDs()))
|
||||||
alert.Meta = make(models.Meta, 0)
|
alert.Meta = make(models.Meta, 0)
|
||||||
for _, key := range []string{"target_uri", "method"} {
|
for _, key := range []string{"target_uri", "method"} {
|
||||||
|
|
||||||
|
@ -64,7 +106,38 @@ func AppsecEventGeneration(inEvt types.Event) (*types.Event, error) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
alert.EventsCount = ptr.Of(int32(1))
|
now := ptr.Of(time.Now().UTC().Format(time.RFC3339))
|
||||||
|
|
||||||
|
for _, matched_rule := range inEvt.Appsec.MatchedRules {
|
||||||
|
evtRule := models.Event{}
|
||||||
|
|
||||||
|
evtRule.Timestamp = now
|
||||||
|
|
||||||
|
evtRule.Meta = make(models.Meta, 0)
|
||||||
|
|
||||||
|
for _, key := range []string{"id", "name", "method", "uri", "matched_zones", "msg"} {
|
||||||
|
|
||||||
|
switch value := matched_rule[key].(type) {
|
||||||
|
case string:
|
||||||
|
evtRule.Meta = appendMeta(evtRule.Meta, key, value)
|
||||||
|
case int:
|
||||||
|
evtRule.Meta = appendMeta(evtRule.Meta, key, fmt.Sprintf("%d", value))
|
||||||
|
case []string:
|
||||||
|
for _, v := range value {
|
||||||
|
evtRule.Meta = appendMeta(evtRule.Meta, key, v)
|
||||||
|
}
|
||||||
|
case []int:
|
||||||
|
for _, v := range value {
|
||||||
|
evtRule.Meta = appendMeta(evtRule.Meta, key, fmt.Sprintf("%d", v))
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
evtRule.Meta = appendMeta(evtRule.Meta, key, fmt.Sprintf("%v", value))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
alert.Events = append(alert.Events, &evtRule)
|
||||||
|
}
|
||||||
|
|
||||||
|
alert.EventsCount = ptr.Of(int32(len(evt.Appsec.MatchedRules)))
|
||||||
alert.Leakspeed = ptr.Of("")
|
alert.Leakspeed = ptr.Of("")
|
||||||
alert.Scenario = ptr.Of(inEvt.Appsec.MatchedRules.GetName())
|
alert.Scenario = ptr.Of(inEvt.Appsec.MatchedRules.GetName())
|
||||||
alert.ScenarioHash = ptr.Of(inEvt.Appsec.MatchedRules.GetHash())
|
alert.ScenarioHash = ptr.Of(inEvt.Appsec.MatchedRules.GetHash())
|
||||||
|
@ -200,7 +273,7 @@ func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *appsec.ParsedR
|
||||||
})
|
})
|
||||||
|
|
||||||
for _, rule := range req.Tx.MatchedRules() {
|
for _, rule := range req.Tx.MatchedRules() {
|
||||||
if rule.Message() == "" || rule.DisruptiveAction() == "pass" || rule.DisruptiveAction() == "allow" {
|
if rule.Message() == "" {
|
||||||
r.logger.Tracef("discarding rule %d (action: %s)", rule.Rule().ID(), rule.DisruptiveAction())
|
r.logger.Tracef("discarding rule %d (action: %s)", rule.Rule().ID(), rule.DisruptiveAction())
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,9 +1,11 @@
|
||||||
package exprhelpers
|
package exprhelpers
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"net"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/cticlient"
|
"github.com/crowdsecurity/crowdsec/pkg/cticlient"
|
||||||
|
"github.com/oschwald/geoip2-golang"
|
||||||
)
|
)
|
||||||
|
|
||||||
type exprCustomFunc struct {
|
type exprCustomFunc struct {
|
||||||
|
@ -455,6 +457,27 @@ var exprFuncs = []exprCustomFunc{
|
||||||
new(func(string) bool),
|
new(func(string) bool),
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
{
|
||||||
|
name: "GeoIPEnrich",
|
||||||
|
function: GeoIPEnrich,
|
||||||
|
signature: []interface{}{
|
||||||
|
new(func(string) *geoip2.City),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "GeoIPASNEnrich",
|
||||||
|
function: GeoIPASNEnrich,
|
||||||
|
signature: []interface{}{
|
||||||
|
new(func(string) *geoip2.ASN),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "GeoIPRangeEnrich",
|
||||||
|
function: GeoIPRangeEnrich,
|
||||||
|
signature: []interface{}{
|
||||||
|
new(func(string) *net.IPNet),
|
||||||
|
},
|
||||||
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
//go 1.20 "CutPrefix": strings.CutPrefix,
|
//go 1.20 "CutPrefix": strings.CutPrefix,
|
||||||
|
|
63
pkg/exprhelpers/geoip.go
Normal file
63
pkg/exprhelpers/geoip.go
Normal file
|
@ -0,0 +1,63 @@
|
||||||
|
package exprhelpers
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net"
|
||||||
|
)
|
||||||
|
|
||||||
|
func GeoIPEnrich(params ...any) (any, error) {
|
||||||
|
if geoIPCityReader == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ip := params[0].(string)
|
||||||
|
|
||||||
|
parsedIP := net.ParseIP(ip)
|
||||||
|
|
||||||
|
city, err := geoIPCityReader.City(parsedIP)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return city, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func GeoIPASNEnrich(params ...any) (any, error) {
|
||||||
|
if geoIPASNReader == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ip := params[0].(string)
|
||||||
|
|
||||||
|
parsedIP := net.ParseIP(ip)
|
||||||
|
asn, err := geoIPASNReader.ASN(parsedIP)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return asn, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func GeoIPRangeEnrich(params ...any) (any, error) {
|
||||||
|
if geoIPRangeReader == nil {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ip := params[0].(string)
|
||||||
|
|
||||||
|
var dummy interface{}
|
||||||
|
|
||||||
|
parsedIP := net.ParseIP(ip)
|
||||||
|
rangeIP, ok, err := geoIPRangeReader.LookupNetwork(parsedIP, &dummy)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if !ok {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return rangeIP, nil
|
||||||
|
}
|
|
@ -20,6 +20,8 @@ import (
|
||||||
"github.com/c-robinson/iplib"
|
"github.com/c-robinson/iplib"
|
||||||
"github.com/cespare/xxhash/v2"
|
"github.com/cespare/xxhash/v2"
|
||||||
"github.com/davecgh/go-spew/spew"
|
"github.com/davecgh/go-spew/spew"
|
||||||
|
"github.com/oschwald/geoip2-golang"
|
||||||
|
"github.com/oschwald/maxminddb-golang"
|
||||||
"github.com/prometheus/client_golang/prometheus"
|
"github.com/prometheus/client_golang/prometheus"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
"github.com/umahmood/haversine"
|
"github.com/umahmood/haversine"
|
||||||
|
@ -55,6 +57,10 @@ var exprFunctionOptions []expr.Option
|
||||||
|
|
||||||
var keyValuePattern = regexp.MustCompile(`(?P<key>[^=\s]+)=(?:"(?P<quoted_value>[^"\\]*(?:\\.[^"\\]*)*)"|(?P<value>[^=\s]+)|\s*)`)
|
var keyValuePattern = regexp.MustCompile(`(?P<key>[^=\s]+)=(?:"(?P<quoted_value>[^"\\]*(?:\\.[^"\\]*)*)"|(?P<value>[^=\s]+)|\s*)`)
|
||||||
|
|
||||||
|
var geoIPCityReader *geoip2.Reader
|
||||||
|
var geoIPASNReader *geoip2.Reader
|
||||||
|
var geoIPRangeReader *maxminddb.Reader
|
||||||
|
|
||||||
func GetExprOptions(ctx map[string]interface{}) []expr.Option {
|
func GetExprOptions(ctx map[string]interface{}) []expr.Option {
|
||||||
if len(exprFunctionOptions) == 0 {
|
if len(exprFunctionOptions) == 0 {
|
||||||
exprFunctionOptions = []expr.Option{}
|
exprFunctionOptions = []expr.Option{}
|
||||||
|
@ -72,6 +78,42 @@ func GetExprOptions(ctx map[string]interface{}) []expr.Option {
|
||||||
return ret
|
return ret
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func GeoIPInit(datadir string) error {
|
||||||
|
var err error
|
||||||
|
|
||||||
|
geoIPCityReader, err = geoip2.Open(filepath.Join(datadir, "GeoLite2-City.mmdb"))
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("unable to open GeoLite2-City.mmdb : %s", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
geoIPASNReader, err = geoip2.Open(filepath.Join(datadir, "GeoLite2-ASN.mmdb"))
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("unable to open GeoLite2-ASN.mmdb : %s", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
geoIPRangeReader, err = maxminddb.Open(filepath.Join(datadir, "GeoLite2-ASN.mmdb"))
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("unable to open GeoLite2-ASN.mmdb : %s", err)
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func GeoIPClose() {
|
||||||
|
if geoIPCityReader != nil {
|
||||||
|
geoIPCityReader.Close()
|
||||||
|
}
|
||||||
|
if geoIPASNReader != nil {
|
||||||
|
geoIPASNReader.Close()
|
||||||
|
}
|
||||||
|
if geoIPRangeReader != nil {
|
||||||
|
geoIPRangeReader.Close()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func Init(databaseClient *database.Client) error {
|
func Init(databaseClient *database.Client) error {
|
||||||
dataFile = make(map[string][]string)
|
dataFile = make(map[string][]string)
|
||||||
dataFileRegex = make(map[string][]*regexp.Regexp)
|
dataFileRegex = make(map[string][]*regexp.Regexp)
|
||||||
|
|
|
@ -7,7 +7,7 @@ import (
|
||||||
)
|
)
|
||||||
|
|
||||||
/* should be part of a package shared with enrich/geoip.go */
|
/* should be part of a package shared with enrich/geoip.go */
|
||||||
type EnrichFunc func(string, *types.Event, interface{}, *log.Entry) (map[string]string, error)
|
type EnrichFunc func(string, *types.Event, *log.Entry) (map[string]string, error)
|
||||||
type InitFunc func(map[string]string) (interface{}, error)
|
type InitFunc func(map[string]string) (interface{}, error)
|
||||||
|
|
||||||
type EnricherCtx struct {
|
type EnricherCtx struct {
|
||||||
|
@ -16,59 +16,42 @@ type EnricherCtx struct {
|
||||||
|
|
||||||
type Enricher struct {
|
type Enricher struct {
|
||||||
Name string
|
Name string
|
||||||
InitFunc InitFunc
|
|
||||||
EnrichFunc EnrichFunc
|
EnrichFunc EnrichFunc
|
||||||
Ctx interface{}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
/* mimic plugin loading */
|
/* mimic plugin loading */
|
||||||
func Loadplugin(path string) (EnricherCtx, error) {
|
func Loadplugin() (EnricherCtx, error) {
|
||||||
enricherCtx := EnricherCtx{}
|
enricherCtx := EnricherCtx{}
|
||||||
enricherCtx.Registered = make(map[string]*Enricher)
|
enricherCtx.Registered = make(map[string]*Enricher)
|
||||||
|
|
||||||
enricherConfig := map[string]string{"datadir": path}
|
|
||||||
|
|
||||||
EnrichersList := []*Enricher{
|
EnrichersList := []*Enricher{
|
||||||
{
|
{
|
||||||
Name: "GeoIpCity",
|
Name: "GeoIpCity",
|
||||||
InitFunc: GeoIPCityInit,
|
|
||||||
EnrichFunc: GeoIpCity,
|
EnrichFunc: GeoIpCity,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: "GeoIpASN",
|
Name: "GeoIpASN",
|
||||||
InitFunc: GeoIPASNInit,
|
|
||||||
EnrichFunc: GeoIpASN,
|
EnrichFunc: GeoIpASN,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: "IpToRange",
|
Name: "IpToRange",
|
||||||
InitFunc: IpToRangeInit,
|
|
||||||
EnrichFunc: IpToRange,
|
EnrichFunc: IpToRange,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: "reverse_dns",
|
Name: "reverse_dns",
|
||||||
InitFunc: reverseDNSInit,
|
|
||||||
EnrichFunc: reverse_dns,
|
EnrichFunc: reverse_dns,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: "ParseDate",
|
Name: "ParseDate",
|
||||||
InitFunc: parseDateInit,
|
|
||||||
EnrichFunc: ParseDate,
|
EnrichFunc: ParseDate,
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
Name: "UnmarshalJSON",
|
Name: "UnmarshalJSON",
|
||||||
InitFunc: unmarshalInit,
|
|
||||||
EnrichFunc: unmarshalJSON,
|
EnrichFunc: unmarshalJSON,
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, enricher := range EnrichersList {
|
for _, enricher := range EnrichersList {
|
||||||
log.Debugf("Initiating enricher '%s'", enricher.Name)
|
|
||||||
pluginCtx, err := enricher.InitFunc(enricherConfig)
|
|
||||||
if err != nil {
|
|
||||||
log.Errorf("unable to register plugin '%s': %v", enricher.Name, err)
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
enricher.Ctx = pluginCtx
|
|
||||||
log.Infof("Successfully registered enricher '%s'", enricher.Name)
|
log.Infof("Successfully registered enricher '%s'", enricher.Name)
|
||||||
enricherCtx.Registered[enricher.Name] = enricher
|
enricherCtx.Registered[enricher.Name] = enricher
|
||||||
}
|
}
|
||||||
|
|
|
@ -56,7 +56,7 @@ func GenDateParse(date string) (string, time.Time) {
|
||||||
return "", time.Time{}
|
return "", time.Time{}
|
||||||
}
|
}
|
||||||
|
|
||||||
func ParseDate(in string, p *types.Event, x interface{}, plog *log.Entry) (map[string]string, error) {
|
func ParseDate(in string, p *types.Event, plog *log.Entry) (map[string]string, error) {
|
||||||
|
|
||||||
var ret = make(map[string]string)
|
var ret = make(map[string]string)
|
||||||
var strDate string
|
var strDate string
|
||||||
|
@ -105,7 +105,3 @@ func ParseDate(in string, p *types.Event, x interface{}, plog *log.Entry) (map[s
|
||||||
|
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func parseDateInit(cfg map[string]string) (interface{}, error) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -48,7 +48,7 @@ func TestDateParse(t *testing.T) {
|
||||||
for _, tt := range tests {
|
for _, tt := range tests {
|
||||||
tt := tt
|
tt := tt
|
||||||
t.Run(tt.name, func(t *testing.T) {
|
t.Run(tt.name, func(t *testing.T) {
|
||||||
strTime, err := ParseDate(tt.evt.StrTime, &tt.evt, nil, logger)
|
strTime, err := ParseDate(tt.evt.StrTime, &tt.evt, logger)
|
||||||
cstest.RequireErrorContains(t, err, tt.expectedErr)
|
cstest.RequireErrorContains(t, err, tt.expectedErr)
|
||||||
if tt.expectedErr != "" {
|
if tt.expectedErr != "" {
|
||||||
return
|
return
|
||||||
|
|
|
@ -11,7 +11,7 @@ import (
|
||||||
/* All plugins must export a list of function pointers for exported symbols */
|
/* All plugins must export a list of function pointers for exported symbols */
|
||||||
//var ExportedFuncs = []string{"reverse_dns"}
|
//var ExportedFuncs = []string{"reverse_dns"}
|
||||||
|
|
||||||
func reverse_dns(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
|
func reverse_dns(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
|
||||||
ret := make(map[string]string)
|
ret := make(map[string]string)
|
||||||
if field == "" {
|
if field == "" {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
|
@ -25,7 +25,3 @@ func reverse_dns(field string, p *types.Event, ctx interface{}, plog *log.Entry)
|
||||||
ret["reverse_dns"] = rets[0]
|
ret["reverse_dns"] = rets[0]
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func reverseDNSInit(cfg map[string]string) (interface{}, error) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -6,53 +6,53 @@ import (
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
"github.com/oschwald/geoip2-golang"
|
"github.com/oschwald/geoip2-golang"
|
||||||
"github.com/oschwald/maxminddb-golang"
|
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
func IpToRange(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
|
func IpToRange(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
|
||||||
var dummy interface{}
|
|
||||||
ret := make(map[string]string)
|
|
||||||
|
|
||||||
if field == "" {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
ip := net.ParseIP(field)
|
|
||||||
if ip == nil {
|
|
||||||
plog.Infof("Can't parse ip %s, no range enrich", field)
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
net, ok, err := ctx.(*maxminddb.Reader).LookupNetwork(ip, &dummy)
|
|
||||||
if err != nil {
|
|
||||||
plog.Errorf("Failed to fetch network for %s : %v", ip.String(), err)
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
if !ok {
|
|
||||||
plog.Debugf("Unable to find range of %s", ip.String())
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
ret["SourceRange"] = net.String()
|
|
||||||
return ret, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func GeoIpASN(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
|
|
||||||
ret := make(map[string]string)
|
|
||||||
if field == "" {
|
if field == "" {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
ip := net.ParseIP(field)
|
r, err := exprhelpers.GeoIPRangeEnrich(field)
|
||||||
if ip == nil {
|
|
||||||
plog.Infof("Can't parse ip %s, no ASN enrich", ip)
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
record, err := ctx.(*geoip2.Reader).ASN(ip)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
plog.Errorf("Unable to enrich ip '%s'", field)
|
plog.Errorf("Unable to enrich ip '%s'", field)
|
||||||
return nil, nil //nolint:nilerr
|
return nil, nil //nolint:nilerr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if r == nil {
|
||||||
|
plog.Warnf("No range found for ip '%s'", field)
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
record := r.(*net.IPNet)
|
||||||
|
|
||||||
|
ret := make(map[string]string)
|
||||||
|
ret["SourceRange"] = record.String()
|
||||||
|
|
||||||
|
return ret, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func GeoIpASN(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
|
||||||
|
if field == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
r, err := exprhelpers.GeoIPASNEnrich(field)
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
plog.Errorf("Unable to enrich ip '%s'", field)
|
||||||
|
return nil, nil //nolint:nilerr
|
||||||
|
}
|
||||||
|
|
||||||
|
record := r.(*geoip2.ASN)
|
||||||
|
|
||||||
|
ret := make(map[string]string)
|
||||||
|
|
||||||
ret["ASNNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber)
|
ret["ASNNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber)
|
||||||
ret["ASNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber)
|
ret["ASNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber)
|
||||||
ret["ASNOrg"] = record.AutonomousSystemOrganization
|
ret["ASNOrg"] = record.AutonomousSystemOrganization
|
||||||
|
@ -62,21 +62,21 @@ func GeoIpASN(field string, p *types.Event, ctx interface{}, plog *log.Entry) (m
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func GeoIpCity(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
|
func GeoIpCity(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
|
||||||
ret := make(map[string]string)
|
|
||||||
if field == "" {
|
if field == "" {
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
ip := net.ParseIP(field)
|
|
||||||
if ip == nil {
|
r, err := exprhelpers.GeoIPEnrich(field)
|
||||||
plog.Infof("Can't parse ip %s, no City enrich", ip)
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
record, err := ctx.(*geoip2.Reader).City(ip)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
plog.Debugf("Unable to enrich ip '%s'", ip)
|
plog.Errorf("Unable to enrich ip '%s'", field)
|
||||||
return nil, nil //nolint:nilerr
|
return nil, nil //nolint:nilerr
|
||||||
}
|
}
|
||||||
|
|
||||||
|
record := r.(*geoip2.City)
|
||||||
|
ret := make(map[string]string)
|
||||||
|
|
||||||
if record.Country.IsoCode != "" {
|
if record.Country.IsoCode != "" {
|
||||||
ret["IsoCode"] = record.Country.IsoCode
|
ret["IsoCode"] = record.Country.IsoCode
|
||||||
ret["IsInEU"] = strconv.FormatBool(record.Country.IsInEuropeanUnion)
|
ret["IsInEU"] = strconv.FormatBool(record.Country.IsInEuropeanUnion)
|
||||||
|
@ -88,7 +88,7 @@ func GeoIpCity(field string, p *types.Event, ctx interface{}, plog *log.Entry) (
|
||||||
ret["IsInEU"] = strconv.FormatBool(record.RepresentedCountry.IsInEuropeanUnion)
|
ret["IsInEU"] = strconv.FormatBool(record.RepresentedCountry.IsInEuropeanUnion)
|
||||||
} else {
|
} else {
|
||||||
ret["IsoCode"] = ""
|
ret["IsoCode"] = ""
|
||||||
ret["IsInEU"] = strconv.FormatBool(false)
|
ret["IsInEU"] = "false"
|
||||||
}
|
}
|
||||||
|
|
||||||
ret["Latitude"] = fmt.Sprintf("%f", record.Location.Latitude)
|
ret["Latitude"] = fmt.Sprintf("%f", record.Location.Latitude)
|
||||||
|
@ -98,33 +98,3 @@ func GeoIpCity(field string, p *types.Event, ctx interface{}, plog *log.Entry) (
|
||||||
|
|
||||||
return ret, nil
|
return ret, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func GeoIPCityInit(cfg map[string]string) (interface{}, error) {
|
|
||||||
dbCityReader, err := geoip2.Open(cfg["datadir"] + "/GeoLite2-City.mmdb")
|
|
||||||
if err != nil {
|
|
||||||
log.Debugf("couldn't open geoip : %v", err)
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return dbCityReader, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func GeoIPASNInit(cfg map[string]string) (interface{}, error) {
|
|
||||||
dbASReader, err := geoip2.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
|
|
||||||
if err != nil {
|
|
||||||
log.Debugf("couldn't open geoip : %v", err)
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return dbASReader, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
func IpToRangeInit(cfg map[string]string) (interface{}, error) {
|
|
||||||
ipToRangeReader, err := maxminddb.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
|
|
||||||
if err != nil {
|
|
||||||
log.Debugf("couldn't open geoip : %v", err)
|
|
||||||
return nil, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return ipToRangeReader, nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -8,7 +8,7 @@ import (
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/types"
|
"github.com/crowdsecurity/crowdsec/pkg/types"
|
||||||
)
|
)
|
||||||
|
|
||||||
func unmarshalJSON(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
|
func unmarshalJSON(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
|
||||||
err := json.Unmarshal([]byte(p.Line.Raw), &p.Unmarshaled)
|
err := json.Unmarshal([]byte(p.Line.Raw), &p.Unmarshaled)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
plog.Errorf("could not unmarshal JSON: %s", err)
|
plog.Errorf("could not unmarshal JSON: %s", err)
|
||||||
|
@ -17,7 +17,3 @@ func unmarshalJSON(field string, p *types.Event, ctx interface{}, plog *log.Entr
|
||||||
plog.Tracef("unmarshaled JSON: %+v", p.Unmarshaled)
|
plog.Tracef("unmarshaled JSON: %+v", p.Unmarshaled)
|
||||||
return nil, nil
|
return nil, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func unmarshalInit(cfg map[string]string) (interface{}, error) {
|
|
||||||
return nil, nil
|
|
||||||
}
|
|
||||||
|
|
|
@ -64,7 +64,7 @@ type Node struct {
|
||||||
Data []*types.DataSource `yaml:"data,omitempty"`
|
Data []*types.DataSource `yaml:"data,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Node) validate(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
func (n *Node) validate(ectx EnricherCtx) error {
|
||||||
|
|
||||||
//stage is being set automagically
|
//stage is being set automagically
|
||||||
if n.Stage == "" {
|
if n.Stage == "" {
|
||||||
|
@ -563,7 +563,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
||||||
return fmt.Errorf("Node is empty")
|
return fmt.Errorf("Node is empty")
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := n.validate(pctx, ectx); err != nil {
|
if err := n.validate(ectx); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -56,7 +56,7 @@ func TestParserConfigs(t *testing.T) {
|
||||||
t.Fatalf("Compile: (%d/%d) expected error", idx+1, len(CfgTests))
|
t.Fatalf("Compile: (%d/%d) expected error", idx+1, len(CfgTests))
|
||||||
}
|
}
|
||||||
|
|
||||||
err = CfgTests[idx].NodeCfg.validate(pctx, EnricherCtx{})
|
err = CfgTests[idx].NodeCfg.validate(EnricherCtx{})
|
||||||
if CfgTests[idx].Valid == true && err != nil {
|
if CfgTests[idx].Valid == true && err != nil {
|
||||||
t.Fatalf("Valid: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err)
|
t.Fatalf("Valid: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -152,7 +152,11 @@ func prepTests() (*UnixParserCtx, EnricherCtx, error) {
|
||||||
|
|
||||||
//Load enrichment
|
//Load enrichment
|
||||||
datadir := "./test_data/"
|
datadir := "./test_data/"
|
||||||
ectx, err = Loadplugin(datadir)
|
err = exprhelpers.GeoIPInit(datadir)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("unable to initialize GeoIP: %s", err)
|
||||||
|
}
|
||||||
|
ectx, err = Loadplugin()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("failed to load plugin geoip : %v", err)
|
log.Fatalf("failed to load plugin geoip : %v", err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -155,7 +155,7 @@ func (n *Node) ProcessStatics(statics []ExtraField, event *types.Event) error {
|
||||||
/*still way too hackish, but : inject all the results in enriched, and */
|
/*still way too hackish, but : inject all the results in enriched, and */
|
||||||
if enricherPlugin, ok := n.EnrichFunctions.Registered[static.Method]; ok {
|
if enricherPlugin, ok := n.EnrichFunctions.Registered[static.Method]; ok {
|
||||||
clog.Tracef("Found method '%s'", static.Method)
|
clog.Tracef("Found method '%s'", static.Method)
|
||||||
ret, err := enricherPlugin.EnrichFunc(value, event, enricherPlugin.Ctx, n.Logger.WithField("method", static.Method))
|
ret, err := enricherPlugin.EnrichFunc(value, event, n.Logger.WithField("method", static.Method))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
clog.Errorf("method '%s' returned an error : %v", static.Method, err)
|
clog.Errorf("method '%s' returned an error : %v", static.Method, err)
|
||||||
}
|
}
|
||||||
|
|
|
@ -117,7 +117,7 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
|
||||||
*/
|
*/
|
||||||
log.Infof("Loading enrich plugins")
|
log.Infof("Loading enrich plugins")
|
||||||
|
|
||||||
parsers.EnricherCtx, err = Loadplugin(cConfig.ConfigPaths.DataDir)
|
parsers.EnricherCtx, err = Loadplugin()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return parsers, fmt.Errorf("failed to load enrich plugin : %v", err)
|
return parsers, fmt.Errorf("failed to load enrich plugin : %v", err)
|
||||||
}
|
}
|
||||||
|
|
Loading…
Reference in a new issue