Compare commits
1 commit
master
...
go-1.22-li
Author | SHA1 | Date | |
---|---|---|---|
|
1f5298f0f1 |
27 changed files with 142 additions and 258 deletions
|
@ -3,7 +3,7 @@
|
||||||
linters-settings:
|
linters-settings:
|
||||||
cyclop:
|
cyclop:
|
||||||
# lower this after refactoring
|
# lower this after refactoring
|
||||||
max-complexity: 48
|
max-complexity: 53
|
||||||
|
|
||||||
gci:
|
gci:
|
||||||
sections:
|
sections:
|
||||||
|
@ -22,7 +22,7 @@ linters-settings:
|
||||||
|
|
||||||
gocyclo:
|
gocyclo:
|
||||||
# lower this after refactoring
|
# lower this after refactoring
|
||||||
min-complexity: 48
|
min-complexity: 49
|
||||||
|
|
||||||
funlen:
|
funlen:
|
||||||
# Checks the number of lines in a function.
|
# Checks the number of lines in a function.
|
||||||
|
@ -82,6 +82,11 @@ linters-settings:
|
||||||
- "!**/pkg/apiserver/controllers/v1/errors.go"
|
- "!**/pkg/apiserver/controllers/v1/errors.go"
|
||||||
yaml:
|
yaml:
|
||||||
files:
|
files:
|
||||||
|
- "!**/cmd/notification-dummy/main.go"
|
||||||
|
- "!**/cmd/notification-email/main.go"
|
||||||
|
- "!**/cmd/notification-http/main.go"
|
||||||
|
- "!**/cmd/notification-slack/main.go"
|
||||||
|
- "!**/cmd/notification-splunk/main.go"
|
||||||
- "!**/pkg/acquisition/acquisition.go"
|
- "!**/pkg/acquisition/acquisition.go"
|
||||||
- "!**/pkg/acquisition/acquisition_test.go"
|
- "!**/pkg/acquisition/acquisition_test.go"
|
||||||
- "!**/pkg/acquisition/modules/appsec/appsec.go"
|
- "!**/pkg/acquisition/modules/appsec/appsec.go"
|
||||||
|
|
|
@ -31,7 +31,7 @@ func (cli *cliDecisions) decisionsToTable(alerts *models.GetAlertsResponse, prin
|
||||||
spamLimit := make(map[string]bool)
|
spamLimit := make(map[string]bool)
|
||||||
skipped := 0
|
skipped := 0
|
||||||
|
|
||||||
for aIdx := 0; aIdx < len(*alerts); aIdx++ {
|
for aIdx := range len(*alerts) {
|
||||||
alertItem := (*alerts)[aIdx]
|
alertItem := (*alerts)[aIdx]
|
||||||
newDecisions := make([]*models.Decision, 0)
|
newDecisions := make([]*models.Decision, 0)
|
||||||
|
|
||||||
|
|
|
@ -41,7 +41,7 @@ func generatePassword(length int) string {
|
||||||
|
|
||||||
buf := make([]byte, length)
|
buf := make([]byte, length)
|
||||||
|
|
||||||
for i := 0; i < length; i++ {
|
for i := range length {
|
||||||
rInt, err := saferand.Int(saferand.Reader, big.NewInt(int64(charsetLength)))
|
rInt, err := saferand.Int(saferand.Reader, big.NewInt(int64(charsetLength)))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Fatalf("failed getting data from prng for password generation : %s", err)
|
log.Fatalf("failed getting data from prng for password generation : %s", err)
|
||||||
|
|
|
@ -319,7 +319,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
|
||||||
`,
|
`,
|
||||||
Args: cobra.NoArgs,
|
Args: cobra.NoArgs,
|
||||||
DisableAutoGenTag: true,
|
DisableAutoGenTag: true,
|
||||||
RunE: func(_ *cobra.Command, _ []string) error {
|
Run: func(_ *cobra.Command, _ []string) {
|
||||||
var err error
|
var err error
|
||||||
var skipHub, skipDB, skipCAPI, skipLAPI, skipAgent bool
|
var skipHub, skipDB, skipCAPI, skipLAPI, skipAgent bool
|
||||||
infos := map[string][]byte{
|
infos := map[string][]byte{
|
||||||
|
@ -473,19 +473,15 @@ cscli support dump -f /tmp/crowdsec-support.zip
|
||||||
|
|
||||||
err = zipWriter.Close()
|
err = zipWriter.Close()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not finalize zip file: %s", err)
|
log.Fatalf("could not finalize zip file: %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if outFile == "-" {
|
|
||||||
_, err = os.Stdout.Write(w.Bytes())
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
err = os.WriteFile(outFile, w.Bytes(), 0o600)
|
err = os.WriteFile(outFile, w.Bytes(), 0o600)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("could not write zip file to %s: %s", outFile, err)
|
log.Fatalf("could not write zip file to %s: %s", outFile, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Infof("Written zip file to %s", outFile)
|
log.Infof("Written zip file to %s", outFile)
|
||||||
return nil
|
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
@ -65,7 +65,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.H
|
||||||
parsersTomb.Go(func() error {
|
parsersTomb.Go(func() error {
|
||||||
parserWg.Add(1)
|
parserWg.Add(1)
|
||||||
|
|
||||||
for i := 0; i < cConfig.Crowdsec.ParserRoutinesCount; i++ {
|
for range cConfig.Crowdsec.ParserRoutinesCount {
|
||||||
parsersTomb.Go(func() error {
|
parsersTomb.Go(func() error {
|
||||||
defer trace.CatchPanic("crowdsec/runParse")
|
defer trace.CatchPanic("crowdsec/runParse")
|
||||||
|
|
||||||
|
@ -97,7 +97,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.H
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := 0; i < cConfig.Crowdsec.BucketsRoutinesCount; i++ {
|
for range cConfig.Crowdsec.BucketsRoutinesCount {
|
||||||
bucketsTomb.Go(func() error {
|
bucketsTomb.Go(func() error {
|
||||||
defer trace.CatchPanic("crowdsec/runPour")
|
defer trace.CatchPanic("crowdsec/runPour")
|
||||||
|
|
||||||
|
@ -128,7 +128,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers, hub *cwhub.H
|
||||||
outputsTomb.Go(func() error {
|
outputsTomb.Go(func() error {
|
||||||
outputWg.Add(1)
|
outputWg.Add(1)
|
||||||
|
|
||||||
for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ {
|
for range cConfig.Crowdsec.OutputRoutinesCount {
|
||||||
outputsTomb.Go(func() error {
|
outputsTomb.Go(func() error {
|
||||||
defer trace.CatchPanic("crowdsec/runOutput")
|
defer trace.CatchPanic("crowdsec/runOutput")
|
||||||
|
|
||||||
|
|
|
@ -5,11 +5,10 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
||||||
"github.com/hashicorp/go-hclog"
|
"github.com/hashicorp/go-hclog"
|
||||||
plugin "github.com/hashicorp/go-plugin"
|
plugin "github.com/hashicorp/go-plugin"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v2"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type PluginConfig struct {
|
type PluginConfig struct {
|
||||||
|
@ -33,7 +32,6 @@ func (s *DummyPlugin) Notify(ctx context.Context, notification *protobufs.Notifi
|
||||||
if _, ok := s.PluginConfigByName[notification.Name]; !ok {
|
if _, ok := s.PluginConfigByName[notification.Name]; !ok {
|
||||||
return nil, fmt.Errorf("invalid plugin config name %s", notification.Name)
|
return nil, fmt.Errorf("invalid plugin config name %s", notification.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg := s.PluginConfigByName[notification.Name]
|
cfg := s.PluginConfigByName[notification.Name]
|
||||||
|
|
||||||
if cfg.LogLevel != nil && *cfg.LogLevel != "" {
|
if cfg.LogLevel != nil && *cfg.LogLevel != "" {
|
||||||
|
@ -44,22 +42,19 @@ func (s *DummyPlugin) Notify(ctx context.Context, notification *protobufs.Notifi
|
||||||
logger.Debug(notification.Text)
|
logger.Debug(notification.Text)
|
||||||
|
|
||||||
if cfg.OutputFile != nil && *cfg.OutputFile != "" {
|
if cfg.OutputFile != nil && *cfg.OutputFile != "" {
|
||||||
f, err := os.OpenFile(*cfg.OutputFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
|
f, err := os.OpenFile(*cfg.OutputFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Error(fmt.Sprintf("Cannot open notification file: %s", err))
|
logger.Error(fmt.Sprintf("Cannot open notification file: %s", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, err := f.WriteString(notification.Text + "\n"); err != nil {
|
if _, err := f.WriteString(notification.Text + "\n"); err != nil {
|
||||||
f.Close()
|
f.Close()
|
||||||
logger.Error(fmt.Sprintf("Cannot write notification to file: %s", err))
|
logger.Error(fmt.Sprintf("Cannot write notification to file: %s", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
err = f.Close()
|
err = f.Close()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Error(fmt.Sprintf("Cannot close notification file: %s", err))
|
logger.Error(fmt.Sprintf("Cannot close notification file: %s", err))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
fmt.Println(notification.Text)
|
fmt.Println(notification.Text)
|
||||||
|
|
||||||
return &protobufs.Empty{}, nil
|
return &protobufs.Empty{}, nil
|
||||||
|
@ -69,12 +64,11 @@ func (s *DummyPlugin) Configure(ctx context.Context, config *protobufs.Config) (
|
||||||
d := PluginConfig{}
|
d := PluginConfig{}
|
||||||
err := yaml.Unmarshal(config.Config, &d)
|
err := yaml.Unmarshal(config.Config, &d)
|
||||||
s.PluginConfigByName[d.Name] = d
|
s.PluginConfigByName[d.Name] = d
|
||||||
|
|
||||||
return &protobufs.Empty{}, err
|
return &protobufs.Empty{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
handshake := plugin.HandshakeConfig{
|
var handshake = plugin.HandshakeConfig{
|
||||||
ProtocolVersion: 1,
|
ProtocolVersion: 1,
|
||||||
MagicCookieKey: "CROWDSEC_PLUGIN_KEY",
|
MagicCookieKey: "CROWDSEC_PLUGIN_KEY",
|
||||||
MagicCookieValue: os.Getenv("CROWDSEC_PLUGIN_KEY"),
|
MagicCookieValue: os.Getenv("CROWDSEC_PLUGIN_KEY"),
|
||||||
|
|
|
@ -2,17 +2,15 @@ package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"errors"
|
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
||||||
"github.com/hashicorp/go-hclog"
|
"github.com/hashicorp/go-hclog"
|
||||||
plugin "github.com/hashicorp/go-plugin"
|
plugin "github.com/hashicorp/go-plugin"
|
||||||
mail "github.com/xhit/go-simple-mail/v2"
|
mail "github.com/xhit/go-simple-mail/v2"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v2"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
var baseLogger hclog.Logger = hclog.New(&hclog.LoggerOptions{
|
var baseLogger hclog.Logger = hclog.New(&hclog.LoggerOptions{
|
||||||
|
@ -74,20 +72,19 @@ func (n *EmailPlugin) Configure(ctx context.Context, config *protobufs.Config) (
|
||||||
}
|
}
|
||||||
|
|
||||||
if d.Name == "" {
|
if d.Name == "" {
|
||||||
return nil, errors.New("name is required")
|
return nil, fmt.Errorf("name is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
if d.SMTPHost == "" {
|
if d.SMTPHost == "" {
|
||||||
return nil, errors.New("SMTP host is not set")
|
return nil, fmt.Errorf("SMTP host is not set")
|
||||||
}
|
}
|
||||||
|
|
||||||
if d.ReceiverEmails == nil || len(d.ReceiverEmails) == 0 {
|
if d.ReceiverEmails == nil || len(d.ReceiverEmails) == 0 {
|
||||||
return nil, errors.New("receiver emails are not set")
|
return nil, fmt.Errorf("receiver emails are not set")
|
||||||
}
|
}
|
||||||
|
|
||||||
n.ConfigByName[d.Name] = d
|
n.ConfigByName[d.Name] = d
|
||||||
baseLogger.Debug(fmt.Sprintf("Email plugin '%s' use SMTP host '%s:%d'", d.Name, d.SMTPHost, d.SMTPPort))
|
baseLogger.Debug(fmt.Sprintf("Email plugin '%s' use SMTP host '%s:%d'", d.Name, d.SMTPHost, d.SMTPPort))
|
||||||
|
|
||||||
return &protobufs.Empty{}, nil
|
return &protobufs.Empty{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -95,7 +92,6 @@ func (n *EmailPlugin) Notify(ctx context.Context, notification *protobufs.Notifi
|
||||||
if _, ok := n.ConfigByName[notification.Name]; !ok {
|
if _, ok := n.ConfigByName[notification.Name]; !ok {
|
||||||
return nil, fmt.Errorf("invalid plugin config name %s", notification.Name)
|
return nil, fmt.Errorf("invalid plugin config name %s", notification.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg := n.ConfigByName[notification.Name]
|
cfg := n.ConfigByName[notification.Name]
|
||||||
|
|
||||||
logger := baseLogger.Named(cfg.Name)
|
logger := baseLogger.Named(cfg.Name)
|
||||||
|
@ -121,7 +117,6 @@ func (n *EmailPlugin) Notify(ctx context.Context, notification *protobufs.Notifi
|
||||||
server.ConnectTimeout, err = time.ParseDuration(cfg.ConnectTimeout)
|
server.ConnectTimeout, err = time.ParseDuration(cfg.ConnectTimeout)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Warn(fmt.Sprintf("invalid connect timeout '%s', using default '10s'", cfg.ConnectTimeout))
|
logger.Warn(fmt.Sprintf("invalid connect timeout '%s', using default '10s'", cfg.ConnectTimeout))
|
||||||
|
|
||||||
server.ConnectTimeout = 10 * time.Second
|
server.ConnectTimeout = 10 * time.Second
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -130,18 +125,15 @@ func (n *EmailPlugin) Notify(ctx context.Context, notification *protobufs.Notifi
|
||||||
server.SendTimeout, err = time.ParseDuration(cfg.SendTimeout)
|
server.SendTimeout, err = time.ParseDuration(cfg.SendTimeout)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Warn(fmt.Sprintf("invalid send timeout '%s', using default '10s'", cfg.SendTimeout))
|
logger.Warn(fmt.Sprintf("invalid send timeout '%s', using default '10s'", cfg.SendTimeout))
|
||||||
|
|
||||||
server.SendTimeout = 10 * time.Second
|
server.SendTimeout = 10 * time.Second
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Debug("making smtp connection")
|
logger.Debug("making smtp connection")
|
||||||
|
|
||||||
smtpClient, err := server.Connect()
|
smtpClient, err := server.Connect()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return &protobufs.Empty{}, err
|
return &protobufs.Empty{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Debug("smtp connection done")
|
logger.Debug("smtp connection done")
|
||||||
|
|
||||||
email := mail.NewMSG()
|
email := mail.NewMSG()
|
||||||
|
@ -154,14 +146,12 @@ func (n *EmailPlugin) Notify(ctx context.Context, notification *protobufs.Notifi
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return &protobufs.Empty{}, err
|
return &protobufs.Empty{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Info(fmt.Sprintf("sent email to %v", cfg.ReceiverEmails))
|
logger.Info(fmt.Sprintf("sent email to %v", cfg.ReceiverEmails))
|
||||||
|
|
||||||
return &protobufs.Empty{}, nil
|
return &protobufs.Empty{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
handshake := plugin.HandshakeConfig{
|
var handshake = plugin.HandshakeConfig{
|
||||||
ProtocolVersion: 1,
|
ProtocolVersion: 1,
|
||||||
MagicCookieKey: "CROWDSEC_PLUGIN_KEY",
|
MagicCookieKey: "CROWDSEC_PLUGIN_KEY",
|
||||||
MagicCookieValue: os.Getenv("CROWDSEC_PLUGIN_KEY"),
|
MagicCookieValue: os.Getenv("CROWDSEC_PLUGIN_KEY"),
|
||||||
|
|
|
@ -12,11 +12,10 @@ import (
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
||||||
"github.com/hashicorp/go-hclog"
|
"github.com/hashicorp/go-hclog"
|
||||||
plugin "github.com/hashicorp/go-plugin"
|
plugin "github.com/hashicorp/go-plugin"
|
||||||
"gopkg.in/yaml.v3"
|
"gopkg.in/yaml.v2"
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type PluginConfig struct {
|
type PluginConfig struct {
|
||||||
|
@ -91,23 +90,18 @@ func getTLSClient(c *PluginConfig) error {
|
||||||
|
|
||||||
tlsConfig.Certificates = []tls.Certificate{cert}
|
tlsConfig.Certificates = []tls.Certificate{cert}
|
||||||
}
|
}
|
||||||
|
|
||||||
transport := &http.Transport{
|
transport := &http.Transport{
|
||||||
TLSClientConfig: tlsConfig,
|
TLSClientConfig: tlsConfig,
|
||||||
}
|
}
|
||||||
|
|
||||||
if c.UnixSocket != "" {
|
if c.UnixSocket != "" {
|
||||||
logger.Info(fmt.Sprintf("Using socket '%s'", c.UnixSocket))
|
logger.Info(fmt.Sprintf("Using socket '%s'", c.UnixSocket))
|
||||||
|
|
||||||
transport.DialContext = func(_ context.Context, _, _ string) (net.Conn, error) {
|
transport.DialContext = func(_ context.Context, _, _ string) (net.Conn, error) {
|
||||||
return net.Dial("unix", strings.TrimSuffix(c.UnixSocket, "/"))
|
return net.Dial("unix", strings.TrimSuffix(c.UnixSocket, "/"))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
c.Client = &http.Client{
|
c.Client = &http.Client{
|
||||||
Transport: transport,
|
Transport: transport,
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -115,7 +109,6 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
|
||||||
if _, ok := s.PluginConfigByName[notification.Name]; !ok {
|
if _, ok := s.PluginConfigByName[notification.Name]; !ok {
|
||||||
return nil, fmt.Errorf("invalid plugin config name %s", notification.Name)
|
return nil, fmt.Errorf("invalid plugin config name %s", notification.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg := s.PluginConfigByName[notification.Name]
|
cfg := s.PluginConfigByName[notification.Name]
|
||||||
|
|
||||||
if cfg.LogLevel != nil && *cfg.LogLevel != "" {
|
if cfg.LogLevel != nil && *cfg.LogLevel != "" {
|
||||||
|
@ -128,14 +121,11 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
for headerName, headerValue := range cfg.Headers {
|
for headerName, headerValue := range cfg.Headers {
|
||||||
logger.Debug(fmt.Sprintf("adding header %s: %s", headerName, headerValue))
|
logger.Debug(fmt.Sprintf("adding header %s: %s", headerName, headerValue))
|
||||||
request.Header.Add(headerName, headerValue)
|
request.Header.Add(headerName, headerValue)
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Debug(fmt.Sprintf("making HTTP %s call to %s with body %s", cfg.Method, cfg.URL, notification.Text))
|
logger.Debug(fmt.Sprintf("making HTTP %s call to %s with body %s", cfg.Method, cfg.URL, notification.Text))
|
||||||
|
|
||||||
resp, err := cfg.Client.Do(request.WithContext(ctx))
|
resp, err := cfg.Client.Do(request.WithContext(ctx))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logger.Error(fmt.Sprintf("Failed to make HTTP request : %s", err))
|
logger.Error(fmt.Sprintf("Failed to make HTTP request : %s", err))
|
||||||
|
@ -145,7 +135,7 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
|
||||||
|
|
||||||
respData, err := io.ReadAll(resp.Body)
|
respData, err := io.ReadAll(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to read response body got error %w", err)
|
return nil, fmt.Errorf("failed to read response body got error %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Debug(fmt.Sprintf("got response %s", string(respData)))
|
logger.Debug(fmt.Sprintf("got response %s", string(respData)))
|
||||||
|
@ -153,7 +143,6 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
|
||||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||||
logger.Warn(fmt.Sprintf("HTTP server returned non 200 status code: %d", resp.StatusCode))
|
logger.Warn(fmt.Sprintf("HTTP server returned non 200 status code: %d", resp.StatusCode))
|
||||||
logger.Debug(fmt.Sprintf("HTTP server returned body: %s", string(respData)))
|
logger.Debug(fmt.Sprintf("HTTP server returned body: %s", string(respData)))
|
||||||
|
|
||||||
return &protobufs.Empty{}, nil
|
return &protobufs.Empty{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -162,25 +151,21 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
|
||||||
|
|
||||||
func (s *HTTPPlugin) Configure(ctx context.Context, config *protobufs.Config) (*protobufs.Empty, error) {
|
func (s *HTTPPlugin) Configure(ctx context.Context, config *protobufs.Config) (*protobufs.Empty, error) {
|
||||||
d := PluginConfig{}
|
d := PluginConfig{}
|
||||||
|
|
||||||
err := yaml.Unmarshal(config.Config, &d)
|
err := yaml.Unmarshal(config.Config, &d)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
err = getTLSClient(&d)
|
err = getTLSClient(&d)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
s.PluginConfigByName[d.Name] = d
|
s.PluginConfigByName[d.Name] = d
|
||||||
logger.Debug(fmt.Sprintf("HTTP plugin '%s' use URL '%s'", d.Name, d.URL))
|
logger.Debug(fmt.Sprintf("HTTP plugin '%s' use URL '%s'", d.Name, d.URL))
|
||||||
|
|
||||||
return &protobufs.Empty{}, err
|
return &protobufs.Empty{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
handshake := plugin.HandshakeConfig{
|
var handshake = plugin.HandshakeConfig{
|
||||||
ProtocolVersion: 1,
|
ProtocolVersion: 1,
|
||||||
MagicCookieKey: "CROWDSEC_PLUGIN_KEY",
|
MagicCookieKey: "CROWDSEC_PLUGIN_KEY",
|
||||||
MagicCookieValue: os.Getenv("CROWDSEC_PLUGIN_KEY"),
|
MagicCookieValue: os.Getenv("CROWDSEC_PLUGIN_KEY"),
|
||||||
|
|
|
@ -5,12 +5,12 @@ import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
||||||
"github.com/hashicorp/go-hclog"
|
"github.com/hashicorp/go-hclog"
|
||||||
plugin "github.com/hashicorp/go-plugin"
|
plugin "github.com/hashicorp/go-plugin"
|
||||||
"github.com/slack-go/slack"
|
|
||||||
"gopkg.in/yaml.v3"
|
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
"github.com/slack-go/slack"
|
||||||
|
"gopkg.in/yaml.v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
type PluginConfig struct {
|
type PluginConfig struct {
|
||||||
|
@ -33,16 +33,13 @@ func (n *Notify) Notify(ctx context.Context, notification *protobufs.Notificatio
|
||||||
if _, ok := n.ConfigByName[notification.Name]; !ok {
|
if _, ok := n.ConfigByName[notification.Name]; !ok {
|
||||||
return nil, fmt.Errorf("invalid plugin config name %s", notification.Name)
|
return nil, fmt.Errorf("invalid plugin config name %s", notification.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg := n.ConfigByName[notification.Name]
|
cfg := n.ConfigByName[notification.Name]
|
||||||
|
|
||||||
if cfg.LogLevel != nil && *cfg.LogLevel != "" {
|
if cfg.LogLevel != nil && *cfg.LogLevel != "" {
|
||||||
logger.SetLevel(hclog.LevelFromString(*cfg.LogLevel))
|
logger.SetLevel(hclog.LevelFromString(*cfg.LogLevel))
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Info(fmt.Sprintf("found notify signal for %s config", notification.Name))
|
logger.Info(fmt.Sprintf("found notify signal for %s config", notification.Name))
|
||||||
logger.Debug(fmt.Sprintf("posting to %s webhook, message %s", cfg.Webhook, notification.Text))
|
logger.Debug(fmt.Sprintf("posting to %s webhook, message %s", cfg.Webhook, notification.Text))
|
||||||
|
|
||||||
err := slack.PostWebhookContext(ctx, n.ConfigByName[notification.Name].Webhook, &slack.WebhookMessage{
|
err := slack.PostWebhookContext(ctx, n.ConfigByName[notification.Name].Webhook, &slack.WebhookMessage{
|
||||||
Text: notification.Text,
|
Text: notification.Text,
|
||||||
})
|
})
|
||||||
|
@ -55,19 +52,16 @@ func (n *Notify) Notify(ctx context.Context, notification *protobufs.Notificatio
|
||||||
|
|
||||||
func (n *Notify) Configure(ctx context.Context, config *protobufs.Config) (*protobufs.Empty, error) {
|
func (n *Notify) Configure(ctx context.Context, config *protobufs.Config) (*protobufs.Empty, error) {
|
||||||
d := PluginConfig{}
|
d := PluginConfig{}
|
||||||
|
|
||||||
if err := yaml.Unmarshal(config.Config, &d); err != nil {
|
if err := yaml.Unmarshal(config.Config, &d); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|
||||||
n.ConfigByName[d.Name] = d
|
n.ConfigByName[d.Name] = d
|
||||||
logger.Debug(fmt.Sprintf("Slack plugin '%s' use URL '%s'", d.Name, d.Webhook))
|
logger.Debug(fmt.Sprintf("Slack plugin '%s' use URL '%s'", d.Name, d.Webhook))
|
||||||
|
|
||||||
return &protobufs.Empty{}, nil
|
return &protobufs.Empty{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
handshake := plugin.HandshakeConfig{
|
var handshake = plugin.HandshakeConfig{
|
||||||
ProtocolVersion: 1,
|
ProtocolVersion: 1,
|
||||||
MagicCookieKey: "CROWDSEC_PLUGIN_KEY",
|
MagicCookieKey: "CROWDSEC_PLUGIN_KEY",
|
||||||
MagicCookieValue: os.Getenv("CROWDSEC_PLUGIN_KEY"),
|
MagicCookieValue: os.Getenv("CROWDSEC_PLUGIN_KEY"),
|
||||||
|
|
|
@ -10,11 +10,11 @@ import (
|
||||||
"os"
|
"os"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
|
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
||||||
"github.com/hashicorp/go-hclog"
|
"github.com/hashicorp/go-hclog"
|
||||||
plugin "github.com/hashicorp/go-plugin"
|
plugin "github.com/hashicorp/go-plugin"
|
||||||
"gopkg.in/yaml.v3"
|
|
||||||
|
|
||||||
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
"gopkg.in/yaml.v2"
|
||||||
)
|
)
|
||||||
|
|
||||||
var logger hclog.Logger = hclog.New(&hclog.LoggerOptions{
|
var logger hclog.Logger = hclog.New(&hclog.LoggerOptions{
|
||||||
|
@ -44,7 +44,6 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio
|
||||||
if _, ok := s.PluginConfigByName[notification.Name]; !ok {
|
if _, ok := s.PluginConfigByName[notification.Name]; !ok {
|
||||||
return &protobufs.Empty{}, fmt.Errorf("splunk invalid config name %s", notification.Name)
|
return &protobufs.Empty{}, fmt.Errorf("splunk invalid config name %s", notification.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
cfg := s.PluginConfigByName[notification.Name]
|
cfg := s.PluginConfigByName[notification.Name]
|
||||||
|
|
||||||
if cfg.LogLevel != nil && *cfg.LogLevel != "" {
|
if cfg.LogLevel != nil && *cfg.LogLevel != "" {
|
||||||
|
@ -54,7 +53,6 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio
|
||||||
logger.Info(fmt.Sprintf("received notify signal for %s config", notification.Name))
|
logger.Info(fmt.Sprintf("received notify signal for %s config", notification.Name))
|
||||||
|
|
||||||
p := Payload{Event: notification.Text}
|
p := Payload{Event: notification.Text}
|
||||||
|
|
||||||
data, err := json.Marshal(p)
|
data, err := json.Marshal(p)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return &protobufs.Empty{}, err
|
return &protobufs.Empty{}, err
|
||||||
|
@ -67,7 +65,6 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio
|
||||||
|
|
||||||
req.Header.Add("Authorization", fmt.Sprintf("Splunk %s", cfg.Token))
|
req.Header.Add("Authorization", fmt.Sprintf("Splunk %s", cfg.Token))
|
||||||
logger.Debug(fmt.Sprintf("posting event %s to %s", string(data), req.URL))
|
logger.Debug(fmt.Sprintf("posting event %s to %s", string(data), req.URL))
|
||||||
|
|
||||||
resp, err := s.Client.Do(req.WithContext(ctx))
|
resp, err := s.Client.Do(req.WithContext(ctx))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return &protobufs.Empty{}, err
|
return &protobufs.Empty{}, err
|
||||||
|
@ -76,19 +73,15 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio
|
||||||
if resp.StatusCode != http.StatusOK {
|
if resp.StatusCode != http.StatusOK {
|
||||||
content, err := io.ReadAll(resp.Body)
|
content, err := io.ReadAll(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return &protobufs.Empty{}, fmt.Errorf("got non 200 response and failed to read error %w", err)
|
return &protobufs.Empty{}, fmt.Errorf("got non 200 response and failed to read error %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
return &protobufs.Empty{}, fmt.Errorf("got non 200 response %s", string(content))
|
return &protobufs.Empty{}, fmt.Errorf("got non 200 response %s", string(content))
|
||||||
}
|
}
|
||||||
|
|
||||||
respData, err := io.ReadAll(resp.Body)
|
respData, err := io.ReadAll(resp.Body)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return &protobufs.Empty{}, fmt.Errorf("failed to read response body got error %w", err)
|
return &protobufs.Empty{}, fmt.Errorf("failed to read response body got error %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
logger.Debug(fmt.Sprintf("got response %s", string(respData)))
|
logger.Debug(fmt.Sprintf("got response %s", string(respData)))
|
||||||
|
|
||||||
return &protobufs.Empty{}, nil
|
return &protobufs.Empty{}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -97,12 +90,11 @@ func (s *Splunk) Configure(ctx context.Context, config *protobufs.Config) (*prot
|
||||||
err := yaml.Unmarshal(config.Config, &d)
|
err := yaml.Unmarshal(config.Config, &d)
|
||||||
s.PluginConfigByName[d.Name] = d
|
s.PluginConfigByName[d.Name] = d
|
||||||
logger.Debug(fmt.Sprintf("Splunk plugin '%s' use URL '%s'", d.Name, d.URL))
|
logger.Debug(fmt.Sprintf("Splunk plugin '%s' use URL '%s'", d.Name, d.URL))
|
||||||
|
|
||||||
return &protobufs.Empty{}, err
|
return &protobufs.Empty{}, err
|
||||||
}
|
}
|
||||||
|
|
||||||
func main() {
|
func main() {
|
||||||
handshake := plugin.HandshakeConfig{
|
var handshake = plugin.HandshakeConfig{
|
||||||
ProtocolVersion: 1,
|
ProtocolVersion: 1,
|
||||||
MagicCookieKey: "CROWDSEC_PLUGIN_KEY",
|
MagicCookieKey: "CROWDSEC_PLUGIN_KEY",
|
||||||
MagicCookieValue: os.Getenv("CROWDSEC_PLUGIN_KEY"),
|
MagicCookieValue: os.Getenv("CROWDSEC_PLUGIN_KEY"),
|
||||||
|
|
|
@ -269,7 +269,7 @@ func LoadAcquisitionFromFile(config *csconfig.CrowdsecServiceCfg, prom *csconfig
|
||||||
|
|
||||||
func GetMetrics(sources []DataSource, aggregated bool) error {
|
func GetMetrics(sources []DataSource, aggregated bool) error {
|
||||||
var metrics []prometheus.Collector
|
var metrics []prometheus.Collector
|
||||||
for i := 0; i < len(sources); i++ {
|
for i := range len(sources) {
|
||||||
if aggregated {
|
if aggregated {
|
||||||
metrics = sources[i].GetMetrics()
|
metrics = sources[i].GetMetrics()
|
||||||
} else {
|
} else {
|
||||||
|
@ -343,7 +343,7 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := 0; i < len(sources); i++ {
|
for i := range len(sources) {
|
||||||
subsrc := sources[i] //ensure its a copy
|
subsrc := sources[i] //ensure its a copy
|
||||||
log.Debugf("starting one source %d/%d ->> %T", i, len(sources), subsrc)
|
log.Debugf("starting one source %d/%d ->> %T", i, len(sources), subsrc)
|
||||||
|
|
||||||
|
|
|
@ -322,7 +322,7 @@ func (f *MockCat) UnmarshalConfig(cfg []byte) error { return nil }
|
||||||
func (f *MockCat) GetName() string { return "mock_cat" }
|
func (f *MockCat) GetName() string { return "mock_cat" }
|
||||||
func (f *MockCat) GetMode() string { return "cat" }
|
func (f *MockCat) GetMode() string { return "cat" }
|
||||||
func (f *MockCat) OneShotAcquisition(out chan types.Event, tomb *tomb.Tomb) error {
|
func (f *MockCat) OneShotAcquisition(out chan types.Event, tomb *tomb.Tomb) error {
|
||||||
for i := 0; i < 10; i++ {
|
for range 10 {
|
||||||
evt := types.Event{}
|
evt := types.Event{}
|
||||||
evt.Line.Src = "test"
|
evt.Line.Src = "test"
|
||||||
out <- evt
|
out <- evt
|
||||||
|
@ -369,7 +369,7 @@ func (f *MockTail) OneShotAcquisition(out chan types.Event, tomb *tomb.Tomb) err
|
||||||
return fmt.Errorf("can't run in cat mode")
|
return fmt.Errorf("can't run in cat mode")
|
||||||
}
|
}
|
||||||
func (f *MockTail) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
|
func (f *MockTail) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
|
||||||
for i := 0; i < 10; i++ {
|
for range 10 {
|
||||||
evt := types.Event{}
|
evt := types.Event{}
|
||||||
evt.Line.Src = "test"
|
evt.Line.Src = "test"
|
||||||
out <- evt
|
out <- evt
|
||||||
|
@ -452,7 +452,7 @@ type MockTailError struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
func (f *MockTailError) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
|
func (f *MockTailError) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error {
|
||||||
for i := 0; i < 10; i++ {
|
for range 10 {
|
||||||
evt := types.Event{}
|
evt := types.Event{}
|
||||||
evt.Line.Src = "test"
|
evt.Line.Src = "test"
|
||||||
out <- evt
|
out <- evt
|
||||||
|
|
|
@ -202,7 +202,7 @@ func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry, MetricsLe
|
||||||
|
|
||||||
w.AppsecRunners = make([]AppsecRunner, w.config.Routines)
|
w.AppsecRunners = make([]AppsecRunner, w.config.Routines)
|
||||||
|
|
||||||
for nbRoutine := 0; nbRoutine < w.config.Routines; nbRoutine++ {
|
for nbRoutine := range w.config.Routines {
|
||||||
appsecRunnerUUID := uuid.New().String()
|
appsecRunnerUUID := uuid.New().String()
|
||||||
//we copy AppsecRutime for each runner
|
//we copy AppsecRutime for each runner
|
||||||
wrt := *w.AppsecRuntime
|
wrt := *w.AppsecRuntime
|
||||||
|
|
|
@ -413,7 +413,7 @@ force_inotify: true`, testPattern),
|
||||||
fd, err := os.Create("test_files/stream.log")
|
fd, err := os.Create("test_files/stream.log")
|
||||||
require.NoError(t, err, "could not create test file")
|
require.NoError(t, err, "could not create test file")
|
||||||
|
|
||||||
for i := 0; i < 5; i++ {
|
for i := range 5 {
|
||||||
_, err = fmt.Fprintf(fd, "%d\n", i)
|
_, err = fmt.Fprintf(fd, "%d\n", i)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("could not write test file : %s", err)
|
t.Fatalf("could not write test file : %s", err)
|
||||||
|
|
|
@ -208,7 +208,7 @@ func (k *KinesisSource) decodeFromSubscription(record []byte) ([]CloudwatchSubsc
|
||||||
|
|
||||||
func (k *KinesisSource) WaitForConsumerDeregistration(consumerName string, streamARN string) error {
|
func (k *KinesisSource) WaitForConsumerDeregistration(consumerName string, streamARN string) error {
|
||||||
maxTries := k.Config.MaxRetries
|
maxTries := k.Config.MaxRetries
|
||||||
for i := 0; i < maxTries; i++ {
|
for i := range maxTries {
|
||||||
_, err := k.kClient.DescribeStreamConsumer(&kinesis.DescribeStreamConsumerInput{
|
_, err := k.kClient.DescribeStreamConsumer(&kinesis.DescribeStreamConsumerInput{
|
||||||
ConsumerName: aws.String(consumerName),
|
ConsumerName: aws.String(consumerName),
|
||||||
StreamARN: aws.String(streamARN),
|
StreamARN: aws.String(streamARN),
|
||||||
|
@ -249,7 +249,7 @@ func (k *KinesisSource) DeregisterConsumer() error {
|
||||||
|
|
||||||
func (k *KinesisSource) WaitForConsumerRegistration(consumerARN string) error {
|
func (k *KinesisSource) WaitForConsumerRegistration(consumerARN string) error {
|
||||||
maxTries := k.Config.MaxRetries
|
maxTries := k.Config.MaxRetries
|
||||||
for i := 0; i < maxTries; i++ {
|
for i := range maxTries {
|
||||||
describeOutput, err := k.kClient.DescribeStreamConsumer(&kinesis.DescribeStreamConsumerInput{
|
describeOutput, err := k.kClient.DescribeStreamConsumer(&kinesis.DescribeStreamConsumerInput{
|
||||||
ConsumerARN: aws.String(consumerARN),
|
ConsumerARN: aws.String(consumerARN),
|
||||||
})
|
})
|
||||||
|
|
|
@ -71,7 +71,7 @@ func WriteToStream(streamName string, count int, shards int, sub bool) {
|
||||||
}
|
}
|
||||||
sess := session.Must(session.NewSession())
|
sess := session.Must(session.NewSession())
|
||||||
kinesisClient := kinesis.New(sess, aws.NewConfig().WithEndpoint(endpoint).WithRegion("us-east-1"))
|
kinesisClient := kinesis.New(sess, aws.NewConfig().WithEndpoint(endpoint).WithRegion("us-east-1"))
|
||||||
for i := 0; i < count; i++ {
|
for i := range count {
|
||||||
partition := "partition"
|
partition := "partition"
|
||||||
if shards != 1 {
|
if shards != 1 {
|
||||||
partition = fmt.Sprintf("partition-%d", i%shards)
|
partition = fmt.Sprintf("partition-%d", i%shards)
|
||||||
|
@ -186,7 +186,7 @@ stream_name: stream-1-shard`,
|
||||||
//Allow the datasource to start listening to the stream
|
//Allow the datasource to start listening to the stream
|
||||||
time.Sleep(4 * time.Second)
|
time.Sleep(4 * time.Second)
|
||||||
WriteToStream(f.Config.StreamName, test.count, test.shards, false)
|
WriteToStream(f.Config.StreamName, test.count, test.shards, false)
|
||||||
for i := 0; i < test.count; i++ {
|
for i := range test.count {
|
||||||
e := <-out
|
e := <-out
|
||||||
assert.Equal(t, fmt.Sprintf("%d", i), e.Line.Raw)
|
assert.Equal(t, fmt.Sprintf("%d", i), e.Line.Raw)
|
||||||
}
|
}
|
||||||
|
@ -233,7 +233,7 @@ stream_name: stream-2-shards`,
|
||||||
time.Sleep(4 * time.Second)
|
time.Sleep(4 * time.Second)
|
||||||
WriteToStream(f.Config.StreamName, test.count, test.shards, false)
|
WriteToStream(f.Config.StreamName, test.count, test.shards, false)
|
||||||
c := 0
|
c := 0
|
||||||
for i := 0; i < test.count; i++ {
|
for range test.count {
|
||||||
<-out
|
<-out
|
||||||
c += 1
|
c += 1
|
||||||
}
|
}
|
||||||
|
@ -281,7 +281,7 @@ from_subscription: true`,
|
||||||
//Allow the datasource to start listening to the stream
|
//Allow the datasource to start listening to the stream
|
||||||
time.Sleep(4 * time.Second)
|
time.Sleep(4 * time.Second)
|
||||||
WriteToStream(f.Config.StreamName, test.count, test.shards, true)
|
WriteToStream(f.Config.StreamName, test.count, test.shards, true)
|
||||||
for i := 0; i < test.count; i++ {
|
for i := range test.count {
|
||||||
e := <-out
|
e := <-out
|
||||||
assert.Equal(t, fmt.Sprintf("%d", i), e.Line.Raw)
|
assert.Equal(t, fmt.Sprintf("%d", i), e.Line.Raw)
|
||||||
}
|
}
|
||||||
|
|
|
@ -276,7 +276,7 @@ func feedLoki(logger *log.Entry, n int, title string) error {
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
for i := 0; i < n; i++ {
|
for i := range n {
|
||||||
streams.Streams[0].Values[i] = LogValue{
|
streams.Streams[0].Values[i] = LogValue{
|
||||||
Time: time.Now(),
|
Time: time.Now(),
|
||||||
Line: fmt.Sprintf("Log line #%d %v", i, title),
|
Line: fmt.Sprintf("Log line #%d %v", i, title),
|
||||||
|
|
|
@ -34,7 +34,7 @@ func isValidHostname(s string) bool {
|
||||||
last := byte('.')
|
last := byte('.')
|
||||||
nonNumeric := false // true once we've seen a letter or hyphen
|
nonNumeric := false // true once we've seen a letter or hyphen
|
||||||
partlen := 0
|
partlen := 0
|
||||||
for i := 0; i < len(s); i++ {
|
for i := range len(s) {
|
||||||
c := s[i]
|
c := s[i]
|
||||||
switch {
|
switch {
|
||||||
default:
|
default:
|
||||||
|
|
|
@ -41,7 +41,7 @@ func (r retryRoundTripper) RoundTrip(req *http.Request) (*http.Response, error)
|
||||||
maxAttempts = 1
|
maxAttempts = 1
|
||||||
}
|
}
|
||||||
|
|
||||||
for i := 0; i < maxAttempts; i++ {
|
for i := range maxAttempts {
|
||||||
if i > 0 {
|
if i > 0 {
|
||||||
if r.withBackOff {
|
if r.withBackOff {
|
||||||
//nolint:gosec
|
//nolint:gosec
|
||||||
|
|
|
@ -1076,7 +1076,7 @@ func TestAPICPush(t *testing.T) {
|
||||||
expectedCalls: 2,
|
expectedCalls: 2,
|
||||||
alerts: func() []*models.Alert {
|
alerts: func() []*models.Alert {
|
||||||
alerts := make([]*models.Alert, 100)
|
alerts := make([]*models.Alert, 100)
|
||||||
for i := 0; i < 100; i++ {
|
for i := range 100 {
|
||||||
alerts[i] = &models.Alert{
|
alerts[i] = &models.Alert{
|
||||||
Scenario: ptr.Of("crowdsec/test"),
|
Scenario: ptr.Of("crowdsec/test"),
|
||||||
ScenarioHash: ptr.Of("certified"),
|
ScenarioHash: ptr.Of("certified"),
|
||||||
|
|
|
@ -109,7 +109,7 @@ func FormatAlerts(result []*ent.Alert) models.AddAlertsRequest {
|
||||||
func (c *Controller) sendAlertToPluginChannel(alert *models.Alert, profileID uint) {
|
func (c *Controller) sendAlertToPluginChannel(alert *models.Alert, profileID uint) {
|
||||||
if c.PluginChannel != nil {
|
if c.PluginChannel != nil {
|
||||||
RETRY:
|
RETRY:
|
||||||
for try := 0; try < 3; try++ {
|
for try := range 3 {
|
||||||
select {
|
select {
|
||||||
case c.PluginChannel <- csplugin.ProfileAlert{ProfileID: profileID, Alert: alert}:
|
case c.PluginChannel <- csplugin.ProfileAlert{ProfileID: profileID, Alert: alert}:
|
||||||
log.Debugf("alert sent to Plugin channel")
|
log.Debugf("alert sent to Plugin channel")
|
||||||
|
|
|
@ -34,7 +34,7 @@ func resetWatcherAlertCounter(pw *PluginWatcher) {
|
||||||
}
|
}
|
||||||
|
|
||||||
func insertNAlertsToPlugin(pw *PluginWatcher, n int, pluginName string) {
|
func insertNAlertsToPlugin(pw *PluginWatcher, n int, pluginName string) {
|
||||||
for i := 0; i < n; i++ {
|
for range n {
|
||||||
pw.Inserts <- pluginName
|
pw.Inserts <- pluginName
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -346,7 +346,7 @@ func (erp ExprRuntimeDebug) ipDebug(ip int, vm *vm.VM, program *vm.Program, part
|
||||||
}
|
}
|
||||||
|
|
||||||
func (erp ExprRuntimeDebug) ipSeek(ip int) []string {
|
func (erp ExprRuntimeDebug) ipSeek(ip int) []string {
|
||||||
for i := 0; i < len(erp.Lines); i++ {
|
for i := range len(erp.Lines) {
|
||||||
parts := strings.Split(erp.Lines[i], "\t")
|
parts := strings.Split(erp.Lines[i], "\t")
|
||||||
if parts[0] == strconv.Itoa(ip) {
|
if parts[0] == strconv.Itoa(ip) {
|
||||||
return parts
|
return parts
|
||||||
|
|
|
@ -216,7 +216,7 @@ func flatten(args []interface{}, v reflect.Value) []interface{} {
|
||||||
}
|
}
|
||||||
|
|
||||||
if v.Kind() == reflect.Array || v.Kind() == reflect.Slice {
|
if v.Kind() == reflect.Array || v.Kind() == reflect.Slice {
|
||||||
for i := 0; i < v.Len(); i++ {
|
for i := range v.Len() {
|
||||||
args = flatten(args, v.Index(i))
|
args = flatten(args, v.Index(i))
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
|
|
|
@ -298,7 +298,7 @@ func PourItemToHolders(parsed types.Event, holders []BucketFactory, buckets *Buc
|
||||||
BucketPourCache["OK"] = append(BucketPourCache["OK"], evt.(types.Event))
|
BucketPourCache["OK"] = append(BucketPourCache["OK"], evt.(types.Event))
|
||||||
}
|
}
|
||||||
//find the relevant holders (scenarios)
|
//find the relevant holders (scenarios)
|
||||||
for idx := 0; idx < len(holders); idx++ {
|
for idx := range len(holders) {
|
||||||
//for idx, holder := range holders {
|
//for idx, holder := range holders {
|
||||||
|
|
||||||
//evaluate bucket's condition
|
//evaluate bucket's condition
|
||||||
|
|
|
@ -22,70 +22,69 @@ import (
|
||||||
|
|
||||||
type Node struct {
|
type Node struct {
|
||||||
FormatVersion string `yaml:"format"`
|
FormatVersion string `yaml:"format"`
|
||||||
// Enable config + runtime debug of node via config o/
|
//Enable config + runtime debug of node via config o/
|
||||||
Debug bool `yaml:"debug,omitempty"`
|
Debug bool `yaml:"debug,omitempty"`
|
||||||
// If enabled, the node (and its child) will report their own statistics
|
//If enabled, the node (and its child) will report their own statistics
|
||||||
Profiling bool `yaml:"profiling,omitempty"`
|
Profiling bool `yaml:"profiling,omitempty"`
|
||||||
// Name, author, description and reference(s) for parser pattern
|
//Name, author, description and reference(s) for parser pattern
|
||||||
Name string `yaml:"name,omitempty"`
|
Name string `yaml:"name,omitempty"`
|
||||||
Author string `yaml:"author,omitempty"`
|
Author string `yaml:"author,omitempty"`
|
||||||
Description string `yaml:"description,omitempty"`
|
Description string `yaml:"description,omitempty"`
|
||||||
References []string `yaml:"references,omitempty"`
|
References []string `yaml:"references,omitempty"`
|
||||||
// if debug is present in the node, keep its specific Logger in runtime structure
|
//if debug is present in the node, keep its specific Logger in runtime structure
|
||||||
Logger *log.Entry `yaml:"-"`
|
Logger *log.Entry `yaml:"-"`
|
||||||
// This is mostly a hack to make writing less repetitive.
|
//This is mostly a hack to make writing less repetitive.
|
||||||
// relying on stage, we know which field to parse, and we
|
//relying on stage, we know which field to parse, and we
|
||||||
// can also promote log to next stage on success
|
//can also promote log to next stage on success
|
||||||
Stage string `yaml:"stage,omitempty"`
|
Stage string `yaml:"stage,omitempty"`
|
||||||
// OnSuccess allows to tag a node to be able to move log to next stage on success
|
//OnSuccess allows to tag a node to be able to move log to next stage on success
|
||||||
OnSuccess string `yaml:"onsuccess,omitempty"`
|
OnSuccess string `yaml:"onsuccess,omitempty"`
|
||||||
rn string // this is only for us in debug, a random generated name for each node
|
rn string //this is only for us in debug, a random generated name for each node
|
||||||
// Filter is executed at runtime (with current log line as context)
|
//Filter is executed at runtime (with current log line as context)
|
||||||
// and must succeed or node is exited
|
//and must succeed or node is exited
|
||||||
Filter string `yaml:"filter,omitempty"`
|
Filter string `yaml:"filter,omitempty"`
|
||||||
RunTimeFilter *vm.Program `yaml:"-" json:"-"` // the actual compiled filter
|
RunTimeFilter *vm.Program `yaml:"-" json:"-"` //the actual compiled filter
|
||||||
// If node has leafs, execute all of them until one asks for a 'break'
|
//If node has leafs, execute all of them until one asks for a 'break'
|
||||||
LeavesNodes []Node `yaml:"nodes,omitempty"`
|
LeavesNodes []Node `yaml:"nodes,omitempty"`
|
||||||
// Flag used to describe when to 'break' or return an 'error'
|
//Flag used to describe when to 'break' or return an 'error'
|
||||||
EnrichFunctions EnricherCtx
|
EnrichFunctions EnricherCtx
|
||||||
|
|
||||||
/* If the node is actually a leaf, it can have : grok, enrich, statics */
|
/* If the node is actually a leaf, it can have : grok, enrich, statics */
|
||||||
// pattern_syntax are named grok patterns that are re-utilized over several grok patterns
|
//pattern_syntax are named grok patterns that are re-utilized over several grok patterns
|
||||||
SubGroks yaml.MapSlice `yaml:"pattern_syntax,omitempty"`
|
SubGroks yaml.MapSlice `yaml:"pattern_syntax,omitempty"`
|
||||||
|
|
||||||
// Holds a grok pattern
|
//Holds a grok pattern
|
||||||
Grok GrokPattern `yaml:"grok,omitempty"`
|
Grok GrokPattern `yaml:"grok,omitempty"`
|
||||||
// Statics can be present in any type of node and is executed last
|
//Statics can be present in any type of node and is executed last
|
||||||
Statics []ExtraField `yaml:"statics,omitempty"`
|
Statics []ExtraField `yaml:"statics,omitempty"`
|
||||||
// Stash allows to capture data from the log line and store it in an accessible cache
|
//Stash allows to capture data from the log line and store it in an accessible cache
|
||||||
Stash []DataCapture `yaml:"stash,omitempty"`
|
Stash []DataCapture `yaml:"stash,omitempty"`
|
||||||
// Whitelists
|
//Whitelists
|
||||||
Whitelist Whitelist `yaml:"whitelist,omitempty"`
|
Whitelist Whitelist `yaml:"whitelist,omitempty"`
|
||||||
Data []*types.DataSource `yaml:"data,omitempty"`
|
Data []*types.DataSource `yaml:"data,omitempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Node) validate(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
func (n *Node) validate(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
||||||
// stage is being set automagically
|
|
||||||
|
//stage is being set automagically
|
||||||
if n.Stage == "" {
|
if n.Stage == "" {
|
||||||
return errors.New("stage needs to be an existing stage")
|
return fmt.Errorf("stage needs to be an existing stage")
|
||||||
}
|
}
|
||||||
|
|
||||||
/* "" behaves like continue */
|
/* "" behaves like continue */
|
||||||
if n.OnSuccess != "continue" && n.OnSuccess != "next_stage" && n.OnSuccess != "" {
|
if n.OnSuccess != "continue" && n.OnSuccess != "next_stage" && n.OnSuccess != "" {
|
||||||
return fmt.Errorf("onsuccess '%s' not continue,next_stage", n.OnSuccess)
|
return fmt.Errorf("onsuccess '%s' not continue,next_stage", n.OnSuccess)
|
||||||
}
|
}
|
||||||
|
|
||||||
if n.Filter != "" && n.RunTimeFilter == nil {
|
if n.Filter != "" && n.RunTimeFilter == nil {
|
||||||
return fmt.Errorf("non-empty filter '%s' was not compiled", n.Filter)
|
return fmt.Errorf("non-empty filter '%s' was not compiled", n.Filter)
|
||||||
}
|
}
|
||||||
|
|
||||||
if n.Grok.RunTimeRegexp != nil || n.Grok.TargetField != "" {
|
if n.Grok.RunTimeRegexp != nil || n.Grok.TargetField != "" {
|
||||||
if n.Grok.TargetField == "" && n.Grok.ExpValue == "" {
|
if n.Grok.TargetField == "" && n.Grok.ExpValue == "" {
|
||||||
return errors.New("grok requires 'expression' or 'apply_on'")
|
return fmt.Errorf("grok requires 'expression' or 'apply_on'")
|
||||||
}
|
}
|
||||||
|
|
||||||
if n.Grok.RegexpName == "" && n.Grok.RegexpValue == "" {
|
if n.Grok.RegexpName == "" && n.Grok.RegexpValue == "" {
|
||||||
return errors.New("grok needs 'pattern' or 'name'")
|
return fmt.Errorf("grok needs 'pattern' or 'name'")
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -94,7 +93,6 @@ func (n *Node) validate(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
||||||
if static.ExpValue == "" {
|
if static.ExpValue == "" {
|
||||||
return fmt.Errorf("static %d : when method is set, expression must be present", idx)
|
return fmt.Errorf("static %d : when method is set, expression must be present", idx)
|
||||||
}
|
}
|
||||||
|
|
||||||
if _, ok := ectx.Registered[static.Method]; !ok {
|
if _, ok := ectx.Registered[static.Method]; !ok {
|
||||||
log.Warningf("the method '%s' doesn't exist or the plugin has not been initialized", static.Method)
|
log.Warningf("the method '%s' doesn't exist or the plugin has not been initialized", static.Method)
|
||||||
}
|
}
|
||||||
|
@ -102,7 +100,6 @@ func (n *Node) validate(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
||||||
if static.Meta == "" && static.Parsed == "" && static.TargetByName == "" {
|
if static.Meta == "" && static.Parsed == "" && static.TargetByName == "" {
|
||||||
return fmt.Errorf("static %d : at least one of meta/event/target must be set", idx)
|
return fmt.Errorf("static %d : at least one of meta/event/target must be set", idx)
|
||||||
}
|
}
|
||||||
|
|
||||||
if static.Value == "" && static.RunTimeValue == nil {
|
if static.Value == "" && static.RunTimeValue == nil {
|
||||||
return fmt.Errorf("static %d value or expression must be set", idx)
|
return fmt.Errorf("static %d value or expression must be set", idx)
|
||||||
}
|
}
|
||||||
|
@ -113,76 +110,72 @@ func (n *Node) validate(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
||||||
if stash.Name == "" {
|
if stash.Name == "" {
|
||||||
return fmt.Errorf("stash %d : name must be set", idx)
|
return fmt.Errorf("stash %d : name must be set", idx)
|
||||||
}
|
}
|
||||||
|
|
||||||
if stash.Value == "" {
|
if stash.Value == "" {
|
||||||
return fmt.Errorf("stash %s : value expression must be set", stash.Name)
|
return fmt.Errorf("stash %s : value expression must be set", stash.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
if stash.Key == "" {
|
if stash.Key == "" {
|
||||||
return fmt.Errorf("stash %s : key expression must be set", stash.Name)
|
return fmt.Errorf("stash %s : key expression must be set", stash.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
if stash.TTL == "" {
|
if stash.TTL == "" {
|
||||||
return fmt.Errorf("stash %s : ttl must be set", stash.Name)
|
return fmt.Errorf("stash %s : ttl must be set", stash.Name)
|
||||||
}
|
}
|
||||||
|
|
||||||
if stash.Strategy == "" {
|
if stash.Strategy == "" {
|
||||||
stash.Strategy = "LRU"
|
stash.Strategy = "LRU"
|
||||||
}
|
}
|
||||||
// should be configurable
|
//should be configurable
|
||||||
if stash.MaxMapSize == 0 {
|
if stash.MaxMapSize == 0 {
|
||||||
stash.MaxMapSize = 100
|
stash.MaxMapSize = 100
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Node) processFilter(cachedExprEnv map[string]interface{}) (bool, error) {
|
func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[string]interface{}) (bool, error) {
|
||||||
|
var NodeState bool
|
||||||
|
var NodeHasOKGrok bool
|
||||||
clog := n.Logger
|
clog := n.Logger
|
||||||
if n.RunTimeFilter == nil {
|
|
||||||
clog.Tracef("Node has not filter, enter")
|
|
||||||
return true, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
// Evaluate node's filter
|
cachedExprEnv := expressionEnv
|
||||||
output, err := exprhelpers.Run(n.RunTimeFilter, cachedExprEnv, clog, n.Debug)
|
|
||||||
if err != nil {
|
|
||||||
clog.Warningf("failed to run filter : %v", err)
|
|
||||||
clog.Debugf("Event leaving node : ko")
|
|
||||||
|
|
||||||
return false, nil
|
clog.Tracef("Event entering node")
|
||||||
}
|
if n.RunTimeFilter != nil {
|
||||||
|
//Evaluate node's filter
|
||||||
switch out := output.(type) {
|
output, err := exprhelpers.Run(n.RunTimeFilter, cachedExprEnv, clog, n.Debug)
|
||||||
case bool:
|
if err != nil {
|
||||||
if !out {
|
clog.Warningf("failed to run filter : %v", err)
|
||||||
clog.Debugf("Event leaving node : ko (failed filter)")
|
clog.Debugf("Event leaving node : ko")
|
||||||
return false, nil
|
return false, nil
|
||||||
}
|
}
|
||||||
default:
|
|
||||||
clog.Warningf("Expr '%s' returned non-bool, abort : %T", n.Filter, output)
|
|
||||||
clog.Debugf("Event leaving node : ko")
|
|
||||||
|
|
||||||
return false, nil
|
switch out := output.(type) {
|
||||||
|
case bool:
|
||||||
|
if !out {
|
||||||
|
clog.Debugf("Event leaving node : ko (failed filter)")
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
clog.Warningf("Expr '%s' returned non-bool, abort : %T", n.Filter, output)
|
||||||
|
clog.Debugf("Event leaving node : ko")
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
NodeState = true
|
||||||
|
} else {
|
||||||
|
clog.Tracef("Node has not filter, enter")
|
||||||
|
NodeState = true
|
||||||
}
|
}
|
||||||
|
|
||||||
return true, nil
|
if n.Name != "" {
|
||||||
}
|
NodesHits.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc()
|
||||||
|
}
|
||||||
func (n *Node) processWhitelist(cachedExprEnv map[string]interface{}, p *types.Event) (bool, error) {
|
exprErr := error(nil)
|
||||||
var exprErr error
|
|
||||||
|
|
||||||
isWhitelisted := n.CheckIPsWL(p)
|
isWhitelisted := n.CheckIPsWL(p)
|
||||||
if !isWhitelisted {
|
if !isWhitelisted {
|
||||||
isWhitelisted, exprErr = n.CheckExprWL(cachedExprEnv, p)
|
isWhitelisted, exprErr = n.CheckExprWL(cachedExprEnv, p)
|
||||||
}
|
}
|
||||||
|
|
||||||
if exprErr != nil {
|
if exprErr != nil {
|
||||||
// Previous code returned nil if there was an error, so we keep this behavior
|
// Previous code returned nil if there was an error, so we keep this behavior
|
||||||
return false, nil //nolint:nilerr
|
return false, nil //nolint:nilerr
|
||||||
}
|
}
|
||||||
|
|
||||||
if isWhitelisted && !p.Whitelisted {
|
if isWhitelisted && !p.Whitelisted {
|
||||||
p.Whitelisted = true
|
p.Whitelisted = true
|
||||||
p.WhitelistReason = n.Whitelist.Reason
|
p.WhitelistReason = n.Whitelist.Reason
|
||||||
|
@ -192,51 +185,18 @@ func (n *Node) processWhitelist(cachedExprEnv map[string]interface{}, p *types.E
|
||||||
for k := range p.Overflow.Sources {
|
for k := range p.Overflow.Sources {
|
||||||
ips = append(ips, k)
|
ips = append(ips, k)
|
||||||
}
|
}
|
||||||
|
clog.Infof("Ban for %s whitelisted, reason [%s]", strings.Join(ips, ","), n.Whitelist.Reason)
|
||||||
n.Logger.Infof("Ban for %s whitelisted, reason [%s]", strings.Join(ips, ","), n.Whitelist.Reason)
|
|
||||||
|
|
||||||
p.Overflow.Whitelisted = true
|
p.Overflow.Whitelisted = true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return isWhitelisted, nil
|
//Process grok if present, should be exclusive with nodes :)
|
||||||
}
|
|
||||||
|
|
||||||
func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[string]interface{}) (bool, error) {
|
|
||||||
var NodeHasOKGrok bool
|
|
||||||
|
|
||||||
clog := n.Logger
|
|
||||||
|
|
||||||
cachedExprEnv := expressionEnv
|
|
||||||
|
|
||||||
clog.Tracef("Event entering node")
|
|
||||||
|
|
||||||
NodeState, err := n.processFilter(cachedExprEnv)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if !NodeState {
|
|
||||||
return false, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
if n.Name != "" {
|
|
||||||
NodesHits.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc()
|
|
||||||
}
|
|
||||||
|
|
||||||
isWhitelisted, err := n.processWhitelist(cachedExprEnv, p)
|
|
||||||
if err != nil {
|
|
||||||
return false, err
|
|
||||||
}
|
|
||||||
|
|
||||||
// Process grok if present, should be exclusive with nodes :)
|
|
||||||
gstr := ""
|
gstr := ""
|
||||||
|
|
||||||
if n.Grok.RunTimeRegexp != nil {
|
if n.Grok.RunTimeRegexp != nil {
|
||||||
clog.Tracef("Processing grok pattern : %s : %p", n.Grok.RegexpName, n.Grok.RunTimeRegexp)
|
clog.Tracef("Processing grok pattern : %s : %p", n.Grok.RegexpName, n.Grok.RunTimeRegexp)
|
||||||
// for unparsed, parsed etc. set sensible defaults to reduce user hassle
|
//for unparsed, parsed etc. set sensible defaults to reduce user hassle
|
||||||
if n.Grok.TargetField != "" {
|
if n.Grok.TargetField != "" {
|
||||||
// it's a hack to avoid using real reflect
|
//it's a hack to avoid using real reflect
|
||||||
if n.Grok.TargetField == "Line.Raw" {
|
if n.Grok.TargetField == "Line.Raw" {
|
||||||
gstr = p.Line.Raw
|
gstr = p.Line.Raw
|
||||||
} else if val, ok := p.Parsed[n.Grok.TargetField]; ok {
|
} else if val, ok := p.Parsed[n.Grok.TargetField]; ok {
|
||||||
|
@ -251,7 +211,6 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
||||||
clog.Warningf("failed to run RunTimeValue : %v", err)
|
clog.Warningf("failed to run RunTimeValue : %v", err)
|
||||||
NodeState = false
|
NodeState = false
|
||||||
}
|
}
|
||||||
|
|
||||||
switch out := output.(type) {
|
switch out := output.(type) {
|
||||||
case string:
|
case string:
|
||||||
gstr = out
|
gstr = out
|
||||||
|
@ -270,14 +229,12 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
||||||
} else {
|
} else {
|
||||||
groklabel = n.Grok.RegexpName
|
groklabel = n.Grok.RegexpName
|
||||||
}
|
}
|
||||||
|
|
||||||
grok := n.Grok.RunTimeRegexp.Parse(gstr)
|
grok := n.Grok.RunTimeRegexp.Parse(gstr)
|
||||||
if len(grok) > 0 {
|
if len(grok) > 0 {
|
||||||
/*tag explicitly that the *current* node had a successful grok pattern. it's important to know success state*/
|
/*tag explicitly that the *current* node had a successful grok pattern. it's important to know success state*/
|
||||||
NodeHasOKGrok = true
|
NodeHasOKGrok = true
|
||||||
|
|
||||||
clog.Debugf("+ Grok '%s' returned %d entries to merge in Parsed", groklabel, len(grok))
|
clog.Debugf("+ Grok '%s' returned %d entries to merge in Parsed", groklabel, len(grok))
|
||||||
// We managed to grok stuff, merged into parse
|
//We managed to grok stuff, merged into parse
|
||||||
for k, v := range grok {
|
for k, v := range grok {
|
||||||
clog.Debugf("\t.Parsed['%s'] = '%s'", k, v)
|
clog.Debugf("\t.Parsed['%s'] = '%s'", k, v)
|
||||||
p.Parsed[k] = v
|
p.Parsed[k] = v
|
||||||
|
@ -289,37 +246,34 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
// grok failed, node failed
|
//grok failed, node failed
|
||||||
clog.Debugf("+ Grok '%s' didn't return data on '%s'", groklabel, gstr)
|
clog.Debugf("+ Grok '%s' didn't return data on '%s'", groklabel, gstr)
|
||||||
NodeState = false
|
NodeState = false
|
||||||
}
|
}
|
||||||
|
|
||||||
} else {
|
} else {
|
||||||
clog.Tracef("! No grok pattern : %p", n.Grok.RunTimeRegexp)
|
clog.Tracef("! No grok pattern : %p", n.Grok.RunTimeRegexp)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Process the stash (data collection) if : a grok was present and succeeded, or if there is no grok
|
//Process the stash (data collection) if : a grok was present and succeeded, or if there is no grok
|
||||||
if NodeHasOKGrok || n.Grok.RunTimeRegexp == nil {
|
if NodeHasOKGrok || n.Grok.RunTimeRegexp == nil {
|
||||||
for idx, stash := range n.Stash {
|
for idx, stash := range n.Stash {
|
||||||
var (
|
var value string
|
||||||
key string
|
var key string
|
||||||
value string
|
|
||||||
)
|
|
||||||
|
|
||||||
if stash.ValueExpression == nil {
|
if stash.ValueExpression == nil {
|
||||||
clog.Warningf("Stash %d has no value expression, skipping", idx)
|
clog.Warningf("Stash %d has no value expression, skipping", idx)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
if stash.KeyExpression == nil {
|
if stash.KeyExpression == nil {
|
||||||
clog.Warningf("Stash %d has no key expression, skipping", idx)
|
clog.Warningf("Stash %d has no key expression, skipping", idx)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
// collect the data
|
//collect the data
|
||||||
output, err := exprhelpers.Run(stash.ValueExpression, cachedExprEnv, clog, n.Debug)
|
output, err := exprhelpers.Run(stash.ValueExpression, cachedExprEnv, clog, n.Debug)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
clog.Warningf("Error while running stash val expression : %v", err)
|
clog.Warningf("Error while running stash val expression : %v", err)
|
||||||
}
|
}
|
||||||
// can we expect anything else than a string ?
|
//can we expect anything else than a string ?
|
||||||
switch output := output.(type) {
|
switch output := output.(type) {
|
||||||
case string:
|
case string:
|
||||||
value = output
|
value = output
|
||||||
|
@ -328,12 +282,12 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
// collect the key
|
//collect the key
|
||||||
output, err = exprhelpers.Run(stash.KeyExpression, cachedExprEnv, clog, n.Debug)
|
output, err = exprhelpers.Run(stash.KeyExpression, cachedExprEnv, clog, n.Debug)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
clog.Warningf("Error while running stash key expression : %v", err)
|
clog.Warningf("Error while running stash key expression : %v", err)
|
||||||
}
|
}
|
||||||
// can we expect anything else than a string ?
|
//can we expect anything else than a string ?
|
||||||
switch output := output.(type) {
|
switch output := output.(type) {
|
||||||
case string:
|
case string:
|
||||||
key = output
|
key = output
|
||||||
|
@ -345,7 +299,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Iterate on leafs
|
//Iterate on leafs
|
||||||
for _, leaf := range n.LeavesNodes {
|
for _, leaf := range n.LeavesNodes {
|
||||||
ret, err := leaf.process(p, ctx, cachedExprEnv)
|
ret, err := leaf.process(p, ctx, cachedExprEnv)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -353,9 +307,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
||||||
clog.Debugf("Event leaving node : ko")
|
clog.Debugf("Event leaving node : ko")
|
||||||
return false, err
|
return false, err
|
||||||
}
|
}
|
||||||
|
|
||||||
clog.Tracef("\tsub-node (%s) ret : %v (strategy:%s)", leaf.rn, ret, n.OnSuccess)
|
clog.Tracef("\tsub-node (%s) ret : %v (strategy:%s)", leaf.rn, ret, n.OnSuccess)
|
||||||
|
|
||||||
if ret {
|
if ret {
|
||||||
NodeState = true
|
NodeState = true
|
||||||
/* if child is successful, stop processing */
|
/* if child is successful, stop processing */
|
||||||
|
@ -376,14 +328,12 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
||||||
|
|
||||||
clog.Tracef("State after nodes : %v", NodeState)
|
clog.Tracef("State after nodes : %v", NodeState)
|
||||||
|
|
||||||
// grok or leafs failed, don't process statics
|
//grok or leafs failed, don't process statics
|
||||||
if !NodeState {
|
if !NodeState {
|
||||||
if n.Name != "" {
|
if n.Name != "" {
|
||||||
NodesHitsKo.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc()
|
NodesHitsKo.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc()
|
||||||
}
|
}
|
||||||
|
|
||||||
clog.Debugf("Event leaving node : ko")
|
clog.Debugf("Event leaving node : ko")
|
||||||
|
|
||||||
return NodeState, nil
|
return NodeState, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -410,10 +360,9 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
||||||
if NodeState {
|
if NodeState {
|
||||||
clog.Debugf("Event leaving node : ok")
|
clog.Debugf("Event leaving node : ok")
|
||||||
log.Tracef("node is successful, check strategy")
|
log.Tracef("node is successful, check strategy")
|
||||||
|
|
||||||
if n.OnSuccess == "next_stage" {
|
if n.OnSuccess == "next_stage" {
|
||||||
idx := stageidx(p.Stage, ctx.Stages)
|
idx := stageidx(p.Stage, ctx.Stages)
|
||||||
// we're at the last stage
|
//we're at the last stage
|
||||||
if idx+1 == len(ctx.Stages) {
|
if idx+1 == len(ctx.Stages) {
|
||||||
clog.Debugf("node reached the last stage : %s", p.Stage)
|
clog.Debugf("node reached the last stage : %s", p.Stage)
|
||||||
} else {
|
} else {
|
||||||
|
@ -426,16 +375,15 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
||||||
} else {
|
} else {
|
||||||
clog.Debugf("Event leaving node : ko")
|
clog.Debugf("Event leaving node : ko")
|
||||||
}
|
}
|
||||||
|
|
||||||
clog.Tracef("Node successful, continue")
|
clog.Tracef("Node successful, continue")
|
||||||
|
|
||||||
return NodeState, nil
|
return NodeState, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
||||||
var err error
|
var err error
|
||||||
|
var valid bool
|
||||||
|
|
||||||
valid := false
|
valid = false
|
||||||
|
|
||||||
dumpr := spew.ConfigState{MaxDepth: 1, DisablePointerAddresses: true}
|
dumpr := spew.ConfigState{MaxDepth: 1, DisablePointerAddresses: true}
|
||||||
n.rn = seed.Generate()
|
n.rn = seed.Generate()
|
||||||
|
@ -445,11 +393,10 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
||||||
/* if the node has debugging enabled, create a specific logger with debug
|
/* if the node has debugging enabled, create a specific logger with debug
|
||||||
that will be used only for processing this node ;) */
|
that will be used only for processing this node ;) */
|
||||||
if n.Debug {
|
if n.Debug {
|
||||||
clog := log.New()
|
var clog = log.New()
|
||||||
if err = types.ConfigureLogger(clog); err != nil {
|
if err = types.ConfigureLogger(clog); err != nil {
|
||||||
log.Fatalf("While creating bucket-specific logger : %s", err)
|
log.Fatalf("While creating bucket-specific logger : %s", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
clog.SetLevel(log.DebugLevel)
|
clog.SetLevel(log.DebugLevel)
|
||||||
n.Logger = clog.WithFields(log.Fields{
|
n.Logger = clog.WithFields(log.Fields{
|
||||||
"id": n.rn,
|
"id": n.rn,
|
||||||
|
@ -467,7 +414,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
||||||
|
|
||||||
n.Logger.Tracef("Compiling : %s", dumpr.Sdump(n))
|
n.Logger.Tracef("Compiling : %s", dumpr.Sdump(n))
|
||||||
|
|
||||||
// compile filter if present
|
//compile filter if present
|
||||||
if n.Filter != "" {
|
if n.Filter != "" {
|
||||||
n.RunTimeFilter, err = expr.Compile(n.Filter, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
|
n.RunTimeFilter, err = expr.Compile(n.Filter, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -478,15 +425,12 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
||||||
/* handle pattern_syntax and groks */
|
/* handle pattern_syntax and groks */
|
||||||
for _, pattern := range n.SubGroks {
|
for _, pattern := range n.SubGroks {
|
||||||
n.Logger.Tracef("Adding subpattern '%s' : '%s'", pattern.Key, pattern.Value)
|
n.Logger.Tracef("Adding subpattern '%s' : '%s'", pattern.Key, pattern.Value)
|
||||||
|
|
||||||
if err = pctx.Grok.Add(pattern.Key.(string), pattern.Value.(string)); err != nil {
|
if err = pctx.Grok.Add(pattern.Key.(string), pattern.Value.(string)); err != nil {
|
||||||
if errors.Is(err, grokky.ErrAlreadyExist) {
|
if errors.Is(err, grokky.ErrAlreadyExist) {
|
||||||
n.Logger.Warningf("grok '%s' already registred", pattern.Key)
|
n.Logger.Warningf("grok '%s' already registred", pattern.Key)
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|
||||||
n.Logger.Errorf("Unable to compile subpattern %s : %v", pattern.Key, err)
|
n.Logger.Errorf("Unable to compile subpattern %s : %v", pattern.Key, err)
|
||||||
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -494,36 +438,28 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
||||||
/* load grok by name or compile in-place */
|
/* load grok by name or compile in-place */
|
||||||
if n.Grok.RegexpName != "" {
|
if n.Grok.RegexpName != "" {
|
||||||
n.Logger.Tracef("+ Regexp Compilation '%s'", n.Grok.RegexpName)
|
n.Logger.Tracef("+ Regexp Compilation '%s'", n.Grok.RegexpName)
|
||||||
|
|
||||||
n.Grok.RunTimeRegexp, err = pctx.Grok.Get(n.Grok.RegexpName)
|
n.Grok.RunTimeRegexp, err = pctx.Grok.Get(n.Grok.RegexpName)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("unable to find grok '%s' : %v", n.Grok.RegexpName, err)
|
return fmt.Errorf("unable to find grok '%s' : %v", n.Grok.RegexpName, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if n.Grok.RunTimeRegexp == nil {
|
if n.Grok.RunTimeRegexp == nil {
|
||||||
return fmt.Errorf("empty grok '%s'", n.Grok.RegexpName)
|
return fmt.Errorf("empty grok '%s'", n.Grok.RegexpName)
|
||||||
}
|
}
|
||||||
|
|
||||||
n.Logger.Tracef("%s regexp: %s", n.Grok.RegexpName, n.Grok.RunTimeRegexp.String())
|
n.Logger.Tracef("%s regexp: %s", n.Grok.RegexpName, n.Grok.RunTimeRegexp.String())
|
||||||
|
|
||||||
valid = true
|
valid = true
|
||||||
} else if n.Grok.RegexpValue != "" {
|
} else if n.Grok.RegexpValue != "" {
|
||||||
if strings.HasSuffix(n.Grok.RegexpValue, "\n") {
|
if strings.HasSuffix(n.Grok.RegexpValue, "\n") {
|
||||||
n.Logger.Debugf("Beware, pattern ends with \\n : '%s'", n.Grok.RegexpValue)
|
n.Logger.Debugf("Beware, pattern ends with \\n : '%s'", n.Grok.RegexpValue)
|
||||||
}
|
}
|
||||||
|
|
||||||
n.Grok.RunTimeRegexp, err = pctx.Grok.Compile(n.Grok.RegexpValue)
|
n.Grok.RunTimeRegexp, err = pctx.Grok.Compile(n.Grok.RegexpValue)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("failed to compile grok '%s': %v", n.Grok.RegexpValue, err)
|
return fmt.Errorf("failed to compile grok '%s': %v", n.Grok.RegexpValue, err)
|
||||||
}
|
}
|
||||||
|
|
||||||
if n.Grok.RunTimeRegexp == nil {
|
if n.Grok.RunTimeRegexp == nil {
|
||||||
// We shouldn't be here because compilation succeeded, so regexp shouldn't be nil
|
// We shouldn't be here because compilation succeeded, so regexp shouldn't be nil
|
||||||
return fmt.Errorf("grok compilation failure: %s", n.Grok.RegexpValue)
|
return fmt.Errorf("grok compilation failure: %s", n.Grok.RegexpValue)
|
||||||
}
|
}
|
||||||
|
|
||||||
n.Logger.Tracef("%s regexp : %s", n.Grok.RegexpValue, n.Grok.RunTimeRegexp.String())
|
n.Logger.Tracef("%s regexp : %s", n.Grok.RegexpValue, n.Grok.RunTimeRegexp.String())
|
||||||
|
|
||||||
valid = true
|
valid = true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -537,7 +473,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
/* load grok statics */
|
/* load grok statics */
|
||||||
// compile expr statics if present
|
//compile expr statics if present
|
||||||
for idx := range n.Grok.Statics {
|
for idx := range n.Grok.Statics {
|
||||||
if n.Grok.Statics[idx].ExpValue != "" {
|
if n.Grok.Statics[idx].ExpValue != "" {
|
||||||
n.Grok.Statics[idx].RunTimeValue, err = expr.Compile(n.Grok.Statics[idx].ExpValue,
|
n.Grok.Statics[idx].RunTimeValue, err = expr.Compile(n.Grok.Statics[idx].ExpValue,
|
||||||
|
@ -546,7 +482,6 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
valid = true
|
valid = true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -570,7 +505,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
||||||
}
|
}
|
||||||
|
|
||||||
logLvl := n.Logger.Logger.GetLevel()
|
logLvl := n.Logger.Logger.GetLevel()
|
||||||
// init the cache, does it make sense to create it here just to be sure everything is fine ?
|
//init the cache, does it make sense to create it here just to be sure everything is fine ?
|
||||||
if err = cache.CacheInit(cache.CacheCfg{
|
if err = cache.CacheInit(cache.CacheCfg{
|
||||||
Size: n.Stash[i].MaxMapSize,
|
Size: n.Stash[i].MaxMapSize,
|
||||||
TTL: n.Stash[i].TTLVal,
|
TTL: n.Stash[i].TTLVal,
|
||||||
|
@ -591,18 +526,14 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
||||||
if !n.LeavesNodes[idx].Debug && n.Debug {
|
if !n.LeavesNodes[idx].Debug && n.Debug {
|
||||||
n.LeavesNodes[idx].Debug = true
|
n.LeavesNodes[idx].Debug = true
|
||||||
}
|
}
|
||||||
|
|
||||||
if !n.LeavesNodes[idx].Profiling && n.Profiling {
|
if !n.LeavesNodes[idx].Profiling && n.Profiling {
|
||||||
n.LeavesNodes[idx].Profiling = true
|
n.LeavesNodes[idx].Profiling = true
|
||||||
}
|
}
|
||||||
|
|
||||||
n.LeavesNodes[idx].Stage = n.Stage
|
n.LeavesNodes[idx].Stage = n.Stage
|
||||||
|
|
||||||
err = n.LeavesNodes[idx].compile(pctx, ectx)
|
err = n.LeavesNodes[idx].compile(pctx, ectx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
valid = true
|
valid = true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -615,7 +546,6 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
valid = true
|
valid = true
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -624,15 +554,13 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
valid = valid || whitelistValid
|
valid = valid || whitelistValid
|
||||||
|
|
||||||
if !valid {
|
if !valid {
|
||||||
/* node is empty, error force return */
|
/* node is empty, error force return */
|
||||||
n.Logger.Error("Node is empty or invalid, abort")
|
n.Logger.Error("Node is empty or invalid, abort")
|
||||||
n.Stage = ""
|
n.Stage = ""
|
||||||
|
return fmt.Errorf("Node is empty")
|
||||||
return errors.New("Node is empty")
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if err := n.validate(pctx, ectx); err != nil {
|
if err := n.validate(pctx, ectx); err != nil {
|
||||||
|
|
|
@ -129,7 +129,7 @@ func testOneParser(pctx *UnixParserCtx, ectx EnricherCtx, dir string, b *testing
|
||||||
count = b.N
|
count = b.N
|
||||||
b.ResetTimer()
|
b.ResetTimer()
|
||||||
}
|
}
|
||||||
for n := 0; n < count; n++ {
|
for range count {
|
||||||
if testFile(tests, *pctx, pnodes) != true {
|
if testFile(tests, *pctx, pnodes) != true {
|
||||||
return fmt.Errorf("test failed !")
|
return fmt.Errorf("test failed !")
|
||||||
}
|
}
|
||||||
|
@ -239,7 +239,7 @@ func matchEvent(expected types.Event, out types.Event, debug bool) ([]string, bo
|
||||||
valid = true
|
valid = true
|
||||||
}
|
}
|
||||||
|
|
||||||
for mapIdx := 0; mapIdx < len(expectMaps); mapIdx++ {
|
for mapIdx := range len(expectMaps) {
|
||||||
for expKey, expVal := range expectMaps[mapIdx] {
|
for expKey, expVal := range expectMaps[mapIdx] {
|
||||||
if outVal, ok := outMaps[mapIdx][expKey]; ok {
|
if outVal, ok := outMaps[mapIdx][expKey]; ok {
|
||||||
if outVal == expVal { //ok entry
|
if outVal == expVal { //ok entry
|
||||||
|
|
Loading…
Reference in a new issue