beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

add geoip enrich expr helpers

Browse source
This commit is contained in:
Sebastien Blot 2024-04-15 22:06:59 +02:00
parent 81976c6982
commit 127969d325
No known key found for this signature in database
GPG key ID: DFC2902F40449F6A
15 changed files with 184 additions and 115 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

8
cmd/crowdsec/crowdsec.go
View file

@ -19,6 +19,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/appsec"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
@ -32,6 +33,13 @@ func initCrowdsec(cConfig *csconfig.Config, hub *cwhub.Hub) (*parser.Parsers, []
return nil, nil, fmt.Errorf("while loading context: %w", err)
}
err = exprhelpers.GeoIPInit(hub.GetDataDir())
if err != nil {
//GeoIP databases are not mandatory, do not make crowdsec fail if they are not present
log.Warnf("unable to initialize GeoIP: %s", err)
}
// Start loading configs
csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {

23
pkg/exprhelpers/expr_lib.go
View file

@ -1,9 +1,11 @@
package exprhelpers
import (
"net"
"time"
"github.com/crowdsecurity/crowdsec/pkg/cticlient"
"github.com/oschwald/geoip2-golang"
)
type exprCustomFunc struct {
@ -455,6 +457,27 @@ var exprFuncs = []exprCustomFunc{
new(func(string) bool),
},
},
{
name: "GeoIPEnrich",
function: GeoIPEnrich,
signature: []interface{}{
new(func(string) *geoip2.City),
},
},
{
name: "GeoIPASNEnrich",
function: GeoIPASNEnrich,
signature: []interface{}{
new(func(string) *geoip2.ASN),
},
},
{
name: "GeoIPRangeEnrich",
function: GeoIPRangeEnrich,
signature: []interface{}{
new(func(string) *net.IPNet),
},
},
}
//go 1.20 "CutPrefix": strings.CutPrefix,

63
pkg/exprhelpers/geoip.go Normal file
View file

@ -0,0 +1,63 @@
package exprhelpers
import (
"net"
)
func GeoIPEnrich(params ...any) (any, error) {
if geoIPCityReader == nil {
return nil, nil
}
ip := params[0].(string)
parsedIP := net.ParseIP(ip)
city, err := geoIPCityReader.City(parsedIP)
if err != nil {
return nil, err
}
return city, nil
}
func GeoIPASNEnrich(params ...any) (any, error) {
if geoIPASNReader == nil {
return nil, nil
}
ip := params[0].(string)
parsedIP := net.ParseIP(ip)
asn, err := geoIPASNReader.ASN(parsedIP)
if err != nil {
return nil, err
}
return asn, nil
}
func GeoIPRangeEnrich(params ...any) (any, error) {
if geoIPRangeReader == nil {
return nil, nil
}
ip := params[0].(string)
var dummy interface{}
parsedIP := net.ParseIP(ip)
rangeIP, ok, err := geoIPRangeReader.LookupNetwork(parsedIP, &dummy)
if err != nil {
return nil, err
}
if !ok {
return nil, nil
}
return rangeIP, nil
}

30
pkg/exprhelpers/helpers.go
View file

@ -20,6 +20,8 @@ import (
"github.com/c-robinson/iplib"
"github.com/cespare/xxhash/v2"
"github.com/davecgh/go-spew/spew"
"github.com/oschwald/geoip2-golang"
"github.com/oschwald/maxminddb-golang"
"github.com/prometheus/client_golang/prometheus"
log "github.com/sirupsen/logrus"
"github.com/umahmood/haversine"
@ -55,6 +57,10 @@ var exprFunctionOptions []expr.Option
var keyValuePattern = regexp.MustCompile(`(?P<key>[^=\s]+)=(?:"(?P<quoted_value>[^"\\]*(?:\\.[^"\\]*)*)"|(?P<value>[^=\s]+)|\s*)`)
var geoIPCityReader *geoip2.Reader
var geoIPASNReader *geoip2.Reader
var geoIPRangeReader *maxminddb.Reader
func GetExprOptions(ctx map[string]interface{}) []expr.Option {
if len(exprFunctionOptions) == 0 {
exprFunctionOptions = []expr.Option{}
@ -72,6 +78,30 @@ func GetExprOptions(ctx map[string]interface{}) []expr.Option {
return ret
}
func GeoIPInit(datadir string) error {
var err error
geoIPCityReader, err = geoip2.Open(filepath.Join(datadir, "GeoLite2-City.mmdb"))
if err != nil {
log.Errorf("unable to open GeoLite2-City.mmdb : %s", err)
return err
}
geoIPASNReader, err = geoip2.Open(filepath.Join(datadir, "GeoLite2-ASN.mmdb"))
if err != nil {
log.Errorf("unable to open GeoLite2-ASN.mmdb : %s", err)
return err
}
geoIPRangeReader, err = maxminddb.Open(filepath.Join(datadir, "GeoLite2-ASN.mmdb"))
if err != nil {
log.Errorf("unable to open GeoLite2-ASN.mmdb : %s", err)
return err
}
return nil
}
func Init(databaseClient *database.Client) error {
dataFile = make(map[string][]string)
dataFileRegex = make(map[string][]*regexp.Regexp)

21
pkg/parser/enrich.go
View file

@ -7,7 +7,7 @@ import (
)
/* should be part of a package shared with enrich/geoip.go */
type EnrichFunc func(string, *types.Event, interface{}, *log.Entry) (map[string]string, error)
type EnrichFunc func(string, *types.Event, *log.Entry) (map[string]string, error)
type InitFunc func(map[string]string) (interface{}, error)
type EnricherCtx struct {
@ -16,59 +16,42 @@ type EnricherCtx struct {
type Enricher struct {
Name string
InitFunc InitFunc
EnrichFunc EnrichFunc
Ctx interface{}
}
/* mimic plugin loading */
func Loadplugin(path string) (EnricherCtx, error) {
func Loadplugin() (EnricherCtx, error) {
enricherCtx := EnricherCtx{}
enricherCtx.Registered = make(map[string]*Enricher)
enricherConfig := map[string]string{"datadir": path}
EnrichersList := []*Enricher{
{
Name: "GeoIpCity",
InitFunc: GeoIPCityInit,
EnrichFunc: GeoIpCity,
},
{
Name: "GeoIpASN",
InitFunc: GeoIPASNInit,
EnrichFunc: GeoIpASN,
},
{
Name: "IpToRange",
InitFunc: IpToRangeInit,
EnrichFunc: IpToRange,
},
{
Name: "reverse_dns",
InitFunc: reverseDNSInit,
EnrichFunc: reverse_dns,
},
{
Name: "ParseDate",
InitFunc: parseDateInit,
EnrichFunc: ParseDate,
},
{
Name: "UnmarshalJSON",
InitFunc: unmarshalInit,
EnrichFunc: unmarshalJSON,
},
}
for _, enricher := range EnrichersList {
log.Debugf("Initiating enricher '%s'", enricher.Name)
pluginCtx, err := enricher.InitFunc(enricherConfig)
if err != nil {
log.Errorf("unable to register plugin '%s': %v", enricher.Name, err)
continue
}
enricher.Ctx = pluginCtx
log.Infof("Successfully registered enricher '%s'", enricher.Name)
enricherCtx.Registered[enricher.Name] = enricher
}

6
pkg/parser/enrich_date.go
View file

@ -56,7 +56,7 @@ func GenDateParse(date string) (string, time.Time) {
return "", time.Time{}
}
func ParseDate(in string, p *types.Event, x interface{}, plog *log.Entry) (map[string]string, error) {
func ParseDate(in string, p *types.Event, plog *log.Entry) (map[string]string, error) {
var ret = make(map[string]string)
var strDate string
@ -105,7 +105,3 @@ func ParseDate(in string, p *types.Event, x interface{}, plog *log.Entry) (map[s
return ret, nil
}
func parseDateInit(cfg map[string]string) (interface{}, error) {
return nil, nil
}

2
pkg/parser/enrich_date_test.go
View file

@ -48,7 +48,7 @@ func TestDateParse(t *testing.T) {
for _, tt := range tests {
tt := tt
t.Run(tt.name, func(t *testing.T) {
strTime, err := ParseDate(tt.evt.StrTime, &tt.evt, nil, logger)
strTime, err := ParseDate(tt.evt.StrTime, &tt.evt, logger)
cstest.RequireErrorContains(t, err, tt.expectedErr)
if tt.expectedErr != "" {
return

6
pkg/parser/enrich_dns.go
View file

@ -11,7 +11,7 @@ import (
/* All plugins must export a list of function pointers for exported symbols */
//var ExportedFuncs = []string{"reverse_dns"}
func reverse_dns(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
func reverse_dns(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
ret := make(map[string]string)
if field == "" {
return nil, nil
@ -25,7 +25,3 @@ func reverse_dns(field string, p *types.Event, ctx interface{}, plog *log.Entry)
ret["reverse_dns"] = rets[0]
return ret, nil
}
func reverseDNSInit(cfg map[string]string) (interface{}, error) {
return nil, nil
}

118
pkg/parser/enrich_geoip.go
View file

@ -6,53 +6,53 @@ import (
"strconv"
"github.com/oschwald/geoip2-golang"
"github.com/oschwald/maxminddb-golang"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
func IpToRange(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
var dummy interface{}
ret := make(map[string]string)
if field == "" {
return nil, nil
}
ip := net.ParseIP(field)
if ip == nil {
plog.Infof("Can't parse ip %s, no range enrich", field)
return nil, nil
}
net, ok, err := ctx.(*maxminddb.Reader).LookupNetwork(ip, &dummy)
if err != nil {
plog.Errorf("Failed to fetch network for %s : %v", ip.String(), err)
return nil, nil
}
if !ok {
plog.Debugf("Unable to find range of %s", ip.String())
return nil, nil
}
ret["SourceRange"] = net.String()
return ret, nil
}
func GeoIpASN(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
ret := make(map[string]string)
func IpToRange(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
if field == "" {
return nil, nil
}
ip := net.ParseIP(field)
if ip == nil {
plog.Infof("Can't parse ip %s, no ASN enrich", ip)
return nil, nil
}
record, err := ctx.(*geoip2.Reader).ASN(ip)
r, err := exprhelpers.GeoIPRangeEnrich(field)
if err != nil {
plog.Errorf("Unable to enrich ip '%s'", field)
return nil, nil //nolint:nilerr
}
if r == nil {
plog.Warnf("No range found for ip '%s'", field)
return nil, nil //nolint:nilerr
}
record := r.(*net.IPNet)
ret := make(map[string]string)
ret["SourceRange"] = record.String()
return ret, nil
}
func GeoIpASN(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
if field == "" {
return nil, nil
}
r, err := exprhelpers.GeoIPASNEnrich(field)
if err != nil {
plog.Errorf("Unable to enrich ip '%s'", field)
return nil, nil //nolint:nilerr
}
record := r.(*geoip2.ASN)
ret := make(map[string]string)
ret["ASNNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber)
ret["ASNumber"] = fmt.Sprintf("%d", record.AutonomousSystemNumber)
ret["ASNOrg"] = record.AutonomousSystemOrganization
@ -62,21 +62,21 @@ func GeoIpASN(field string, p *types.Event, ctx interface{}, plog *log.Entry) (m
return ret, nil
}
func GeoIpCity(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
ret := make(map[string]string)
func GeoIpCity(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
if field == "" {
return nil, nil
}
ip := net.ParseIP(field)
if ip == nil {
plog.Infof("Can't parse ip %s, no City enrich", ip)
return nil, nil
}
record, err := ctx.(*geoip2.Reader).City(ip)
r, err := exprhelpers.GeoIPEnrich(field)
if err != nil {
plog.Debugf("Unable to enrich ip '%s'", ip)
plog.Errorf("Unable to enrich ip '%s'", field)
return nil, nil //nolint:nilerr
}
record := r.(*geoip2.City)
ret := make(map[string]string)
if record.Country.IsoCode != "" {
ret["IsoCode"] = record.Country.IsoCode
ret["IsInEU"] = strconv.FormatBool(record.Country.IsInEuropeanUnion)
@ -88,7 +88,7 @@ func GeoIpCity(field string, p *types.Event, ctx interface{}, plog *log.Entry) (
ret["IsInEU"] = strconv.FormatBool(record.RepresentedCountry.IsInEuropeanUnion)
} else {
ret["IsoCode"] = ""
ret["IsInEU"] = strconv.FormatBool(false)
ret["IsInEU"] = "false"
}
ret["Latitude"] = fmt.Sprintf("%f", record.Location.Latitude)
@ -98,33 +98,3 @@ func GeoIpCity(field string, p *types.Event, ctx interface{}, plog *log.Entry) (
return ret, nil
}
func GeoIPCityInit(cfg map[string]string) (interface{}, error) {
dbCityReader, err := geoip2.Open(cfg["datadir"] + "/GeoLite2-City.mmdb")
if err != nil {
log.Debugf("couldn't open geoip : %v", err)
return nil, err
}
return dbCityReader, nil
}
func GeoIPASNInit(cfg map[string]string) (interface{}, error) {
dbASReader, err := geoip2.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
if err != nil {
log.Debugf("couldn't open geoip : %v", err)
return nil, err
}
return dbASReader, nil
}
func IpToRangeInit(cfg map[string]string) (interface{}, error) {
ipToRangeReader, err := maxminddb.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
if err != nil {
log.Debugf("couldn't open geoip : %v", err)
return nil, err
}
return ipToRangeReader, nil
}

6
pkg/parser/enrich_unmarshal.go
View file

@ -8,7 +8,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/types"
)
func unmarshalJSON(field string, p *types.Event, ctx interface{}, plog *log.Entry) (map[string]string, error) {
func unmarshalJSON(field string, p *types.Event, plog *log.Entry) (map[string]string, error) {
err := json.Unmarshal([]byte(p.Line.Raw), &p.Unmarshaled)
if err != nil {
plog.Errorf("could not unmarshal JSON: %s", err)
@ -17,7 +17,3 @@ func unmarshalJSON(field string, p *types.Event, ctx interface{}, plog *log.Entr
plog.Tracef("unmarshaled JSON: %+v", p.Unmarshaled)
return nil, nil
}
func unmarshalInit(cfg map[string]string) (interface{}, error) {
return nil, nil
}

4
pkg/parser/node.go
View file

@ -64,7 +64,7 @@ type Node struct {
Data []*types.DataSource `yaml:"data,omitempty"`
}
func (n *Node) validate(pctx *UnixParserCtx, ectx EnricherCtx) error {
func (n *Node) validate(ectx EnricherCtx) error {
//stage is being set automagically
if n.Stage == "" {
@ -563,7 +563,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
return fmt.Errorf("Node is empty")
}
if err := n.validate(pctx, ectx); err != nil {
if err := n.validate(ectx); err != nil {
return err
}

2
pkg/parser/node_test.go
View file

@ -56,7 +56,7 @@ func TestParserConfigs(t *testing.T) {
t.Fatalf("Compile: (%d/%d) expected error", idx+1, len(CfgTests))
}
err = CfgTests[idx].NodeCfg.validate(pctx, EnricherCtx{})
err = CfgTests[idx].NodeCfg.validate(EnricherCtx{})
if CfgTests[idx].Valid == true && err != nil {
t.Fatalf("Valid: (%d/%d) expected valid, got : %s", idx+1, len(CfgTests), err)
}

6
pkg/parser/parsing_test.go
View file

@ -152,7 +152,11 @@ func prepTests() (*UnixParserCtx, EnricherCtx, error) {
//Load enrichment
datadir := "./test_data/"
ectx, err = Loadplugin(datadir)
err = exprhelpers.GeoIPInit(datadir)
if err != nil {
log.Fatalf("unable to initialize GeoIP: %s", err)
}
ectx, err = Loadplugin()
if err != nil {
log.Fatalf("failed to load plugin geoip : %v", err)
}

2
pkg/parser/runtime.go
View file

@ -155,7 +155,7 @@ func (n *Node) ProcessStatics(statics []ExtraField, event *types.Event) error {
/*still way too hackish, but : inject all the results in enriched, and */
if enricherPlugin, ok := n.EnrichFunctions.Registered[static.Method]; ok {
clog.Tracef("Found method '%s'", static.Method)
ret, err := enricherPlugin.EnrichFunc(value, event, enricherPlugin.Ctx, n.Logger.WithField("method", static.Method))
ret, err := enricherPlugin.EnrichFunc(value, event, n.Logger.WithField("method", static.Method))
if err != nil {
clog.Errorf("method '%s' returned an error : %v", static.Method, err)
}

2
pkg/parser/unix_parser.go
View file

@ -117,7 +117,7 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
*/
log.Infof("Loading enrich plugins")
parsers.EnricherCtx, err = Loadplugin(cConfig.ConfigPaths.DataDir)
parsers.EnricherCtx, err = Loadplugin()
if err != nil {
return parsers, fmt.Errorf("failed to load enrich plugin : %v", err)
}