Compare commits
8 commits
1.6.2-down
...
master
Author | SHA1 | Date | |
---|---|---|---|
|
05b54687b6 | ||
|
c4473839c4 | ||
|
d2c4bc55fc | ||
|
2abc078e53 | ||
|
ceb4479ec4 | ||
|
845d4542bb | ||
|
f4ed7b3520 | ||
|
60431804d8 |
31 changed files with 311 additions and 163 deletions
2
.github/workflows/bats-hub.yml
vendored
2
.github/workflows/bats-hub.yml
vendored
|
@ -33,7 +33,7 @@ jobs:
|
|||
- name: "Set up Go"
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.21.9"
|
||||
go-version: "1.22.2"
|
||||
|
||||
- name: "Install bats dependencies"
|
||||
env:
|
||||
|
|
2
.github/workflows/bats-mysql.yml
vendored
2
.github/workflows/bats-mysql.yml
vendored
|
@ -36,7 +36,7 @@ jobs:
|
|||
- name: "Set up Go"
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.21.9"
|
||||
go-version: "1.22.2"
|
||||
|
||||
- name: "Install bats dependencies"
|
||||
env:
|
||||
|
|
2
.github/workflows/bats-postgres.yml
vendored
2
.github/workflows/bats-postgres.yml
vendored
|
@ -45,7 +45,7 @@ jobs:
|
|||
- name: "Set up Go"
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.21.9"
|
||||
go-version: "1.22.2"
|
||||
|
||||
- name: "Install bats dependencies"
|
||||
env:
|
||||
|
|
2
.github/workflows/bats-sqlite-coverage.yml
vendored
2
.github/workflows/bats-sqlite-coverage.yml
vendored
|
@ -28,7 +28,7 @@ jobs:
|
|||
- name: "Set up Go"
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.21.9"
|
||||
go-version: "1.22.2"
|
||||
|
||||
- name: "Install bats dependencies"
|
||||
env:
|
||||
|
|
2
.github/workflows/ci-windows-build-msi.yml
vendored
2
.github/workflows/ci-windows-build-msi.yml
vendored
|
@ -35,7 +35,7 @@ jobs:
|
|||
- name: "Set up Go"
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.21.9"
|
||||
go-version: "1.22.2"
|
||||
|
||||
- name: Build
|
||||
run: make windows_installer BUILD_RE2_WASM=1
|
||||
|
|
2
.github/workflows/codeql-analysis.yml
vendored
2
.github/workflows/codeql-analysis.yml
vendored
|
@ -52,7 +52,7 @@ jobs:
|
|||
- name: "Set up Go"
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.21.9"
|
||||
go-version: "1.22.2"
|
||||
cache-dependency-path: "**/go.sum"
|
||||
|
||||
# Initializes the CodeQL tools for scanning.
|
||||
|
|
2
.github/workflows/go-tests-windows.yml
vendored
2
.github/workflows/go-tests-windows.yml
vendored
|
@ -34,7 +34,7 @@ jobs:
|
|||
- name: "Set up Go"
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.21.9"
|
||||
go-version: "1.22.2"
|
||||
|
||||
- name: Build
|
||||
run: |
|
||||
|
|
2
.github/workflows/go-tests.yml
vendored
2
.github/workflows/go-tests.yml
vendored
|
@ -126,7 +126,7 @@ jobs:
|
|||
- name: "Set up Go"
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.21.9"
|
||||
go-version: "1.22.2"
|
||||
|
||||
- name: Create localstack streams
|
||||
run: |
|
||||
|
|
|
@ -25,7 +25,7 @@ jobs:
|
|||
- name: "Set up Go"
|
||||
uses: actions/setup-go@v5
|
||||
with:
|
||||
go-version: "1.21.9"
|
||||
go-version: "1.22.2"
|
||||
|
||||
- name: Build the binaries
|
||||
run: |
|
||||
|
|
|
@ -3,7 +3,7 @@
|
|||
linters-settings:
|
||||
cyclop:
|
||||
# lower this after refactoring
|
||||
max-complexity: 53
|
||||
max-complexity: 48
|
||||
|
||||
gci:
|
||||
sections:
|
||||
|
@ -22,7 +22,7 @@ linters-settings:
|
|||
|
||||
gocyclo:
|
||||
# lower this after refactoring
|
||||
min-complexity: 49
|
||||
min-complexity: 48
|
||||
|
||||
funlen:
|
||||
# Checks the number of lines in a function.
|
||||
|
@ -82,18 +82,6 @@ linters-settings:
|
|||
- "!**/pkg/apiserver/controllers/v1/errors.go"
|
||||
yaml:
|
||||
files:
|
||||
- "!**/cmd/crowdsec-cli/alerts.go"
|
||||
- "!**/cmd/crowdsec-cli/capi.go"
|
||||
- "!**/cmd/crowdsec-cli/config_show.go"
|
||||
- "!**/cmd/crowdsec-cli/hubtest.go"
|
||||
- "!**/cmd/crowdsec-cli/lapi.go"
|
||||
- "!**/cmd/crowdsec-cli/simulation.go"
|
||||
- "!**/cmd/crowdsec/crowdsec.go"
|
||||
- "!**/cmd/notification-dummy/main.go"
|
||||
- "!**/cmd/notification-email/main.go"
|
||||
- "!**/cmd/notification-http/main.go"
|
||||
- "!**/cmd/notification-slack/main.go"
|
||||
- "!**/cmd/notification-splunk/main.go"
|
||||
- "!**/pkg/acquisition/acquisition.go"
|
||||
- "!**/pkg/acquisition/acquisition_test.go"
|
||||
- "!**/pkg/acquisition/modules/appsec/appsec.go"
|
||||
|
@ -151,6 +139,13 @@ linters:
|
|||
- structcheck
|
||||
- varcheck
|
||||
|
||||
#
|
||||
# Disabled until fixed for go 1.22
|
||||
#
|
||||
|
||||
- copyloopvar # copyloopvar is a linter detects places where loop variables are copied
|
||||
- intrange # intrange is a linter to find places where for loops could make use of an integer range.
|
||||
|
||||
#
|
||||
# Enabled
|
||||
#
|
||||
|
@ -159,7 +154,6 @@ linters:
|
|||
# - asciicheck # checks that all code identifiers does not have non-ASCII symbols in the name
|
||||
# - bidichk # Checks for dangerous unicode character sequences
|
||||
# - bodyclose # checks whether HTTP response body is closed successfully
|
||||
# - copyloopvar # copyloopvar is a linter detects places where loop variables are copied
|
||||
# - cyclop # checks function and package cyclomatic complexity
|
||||
# - decorder # check declaration order and count of types, constants, variables and functions
|
||||
# - depguard # Go linter that checks if package imports are in a list of acceptable packages
|
||||
|
@ -188,7 +182,6 @@ linters:
|
|||
# - importas # Enforces consistent import aliases
|
||||
# - ineffassign # Detects when assignments to existing variables are not used
|
||||
# - interfacebloat # A linter that checks the number of methods inside an interface.
|
||||
# - intrange # intrange is a linter to find places where for loops could make use of an integer range.
|
||||
# - lll # Reports long lines
|
||||
# - loggercheck # (logrlint): Checks key value pairs for common logger libraries (kitlog,klog,logr,zap).
|
||||
# - logrlint # Check logr arguments.
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
# vim: set ft=dockerfile:
|
||||
FROM golang:1.21.9-alpine3.18 AS build
|
||||
FROM golang:1.22.2-alpine3.18 AS build
|
||||
|
||||
ARG BUILD_VERSION
|
||||
|
||||
|
|
|
@ -1,5 +1,5 @@
|
|||
# vim: set ft=dockerfile:
|
||||
FROM golang:1.21.9-bookworm AS build
|
||||
FROM golang:1.22.2-bookworm AS build
|
||||
|
||||
ARG BUILD_VERSION
|
||||
|
||||
|
|
|
@ -21,7 +21,7 @@ stages:
|
|||
- task: GoTool@0
|
||||
displayName: "Install Go"
|
||||
inputs:
|
||||
version: '1.21.9'
|
||||
version: '1.22.2'
|
||||
|
||||
- pwsh: |
|
||||
choco install -y make
|
||||
|
|
|
@ -17,7 +17,7 @@ import (
|
|||
"github.com/go-openapi/strfmt"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"gopkg.in/yaml.v2"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/crowdsecurity/go-cs-lib/version"
|
||||
|
||||
|
|
|
@ -10,7 +10,7 @@ import (
|
|||
"github.com/go-openapi/strfmt"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"gopkg.in/yaml.v2"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/crowdsecurity/go-cs-lib/version"
|
||||
|
||||
|
@ -85,7 +85,6 @@ func (cli *cliCapi) register(capiUserPrefix string, outputFile string) error {
|
|||
URL: apiurl,
|
||||
VersionPrefix: CAPIURLPrefix,
|
||||
}, nil)
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("api client register ('%s'): %w", types.CAPIBaseURL, err)
|
||||
}
|
||||
|
|
|
@ -14,7 +14,7 @@ import (
|
|||
"github.com/fatih/color"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"gopkg.in/yaml.v2"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/dumps"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/emoji"
|
||||
|
@ -136,7 +136,7 @@ cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios
|
|||
nucleiFileName := fmt.Sprintf("%s.yaml", testName)
|
||||
nucleiFilePath := filepath.Join(testPath, nucleiFileName)
|
||||
|
||||
nucleiFile, err := os.OpenFile(nucleiFilePath, os.O_RDWR|os.O_CREATE, 0755)
|
||||
nucleiFile, err := os.OpenFile(nucleiFilePath, os.O_RDWR|os.O_CREATE, 0o755)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
|
@ -13,7 +13,7 @@ import (
|
|||
"github.com/go-openapi/strfmt"
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"gopkg.in/yaml.v2"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/crowdsecurity/go-cs-lib/version"
|
||||
|
||||
|
|
|
@ -8,7 +8,7 @@ import (
|
|||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"github.com/spf13/cobra"
|
||||
"gopkg.in/yaml.v2"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
|
||||
|
@ -74,7 +74,7 @@ func (cli *cliSimulation) NewEnableCmd() *cobra.Command {
|
|||
|
||||
if len(args) > 0 {
|
||||
for _, scenario := range args {
|
||||
var item = hub.GetItem(cwhub.SCENARIOS, scenario)
|
||||
item := hub.GetItem(cwhub.SCENARIOS, scenario)
|
||||
if item == nil {
|
||||
log.Errorf("'%s' doesn't exist or is not a scenario", scenario)
|
||||
continue
|
||||
|
|
|
@ -319,7 +319,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
|
|||
`,
|
||||
Args: cobra.NoArgs,
|
||||
DisableAutoGenTag: true,
|
||||
Run: func(_ *cobra.Command, _ []string) {
|
||||
RunE: func(_ *cobra.Command, _ []string) error {
|
||||
var err error
|
||||
var skipHub, skipDB, skipCAPI, skipLAPI, skipAgent bool
|
||||
infos := map[string][]byte{
|
||||
|
@ -473,15 +473,19 @@ cscli support dump -f /tmp/crowdsec-support.zip
|
|||
|
||||
err = zipWriter.Close()
|
||||
if err != nil {
|
||||
log.Fatalf("could not finalize zip file: %s", err)
|
||||
return fmt.Errorf("could not finalize zip file: %s", err)
|
||||
}
|
||||
|
||||
if outFile == "-" {
|
||||
_, err = os.Stdout.Write(w.Bytes())
|
||||
return err
|
||||
}
|
||||
err = os.WriteFile(outFile, w.Bytes(), 0o600)
|
||||
if err != nil {
|
||||
log.Fatalf("could not write zip file to %s: %s", outFile, err)
|
||||
return fmt.Errorf("could not write zip file to %s: %s", outFile, err)
|
||||
}
|
||||
|
||||
log.Infof("Written zip file to %s", outFile)
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
|
|
|
@ -9,7 +9,7 @@ import (
|
|||
"time"
|
||||
|
||||
log "github.com/sirupsen/logrus"
|
||||
"gopkg.in/yaml.v2"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/crowdsecurity/go-cs-lib/trace"
|
||||
|
||||
|
@ -207,7 +207,7 @@ func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, hub *cwhub
|
|||
}
|
||||
|
||||
func dumpBucketsPour() {
|
||||
fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "bucketpour-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
|
||||
fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "bucketpour-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o666)
|
||||
if err != nil {
|
||||
log.Fatalf("open: %s", err)
|
||||
}
|
||||
|
@ -230,7 +230,7 @@ func dumpBucketsPour() {
|
|||
}
|
||||
|
||||
func dumpParserState() {
|
||||
fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "parser-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
|
||||
fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "parser-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o666)
|
||||
if err != nil {
|
||||
log.Fatalf("open: %s", err)
|
||||
}
|
||||
|
@ -253,7 +253,7 @@ func dumpParserState() {
|
|||
}
|
||||
|
||||
func dumpOverflowState() {
|
||||
fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "bucket-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0666)
|
||||
fd, err := os.OpenFile(filepath.Join(parser.DumpFolder, "bucket-dump.yaml"), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o666)
|
||||
if err != nil {
|
||||
log.Fatalf("open: %s", err)
|
||||
}
|
||||
|
|
|
@ -5,10 +5,11 @@ import (
|
|||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
||||
"github.com/hashicorp/go-hclog"
|
||||
plugin "github.com/hashicorp/go-plugin"
|
||||
"gopkg.in/yaml.v2"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
||||
)
|
||||
|
||||
type PluginConfig struct {
|
||||
|
@ -32,6 +33,7 @@ func (s *DummyPlugin) Notify(ctx context.Context, notification *protobufs.Notifi
|
|||
if _, ok := s.PluginConfigByName[notification.Name]; !ok {
|
||||
return nil, fmt.Errorf("invalid plugin config name %s", notification.Name)
|
||||
}
|
||||
|
||||
cfg := s.PluginConfigByName[notification.Name]
|
||||
|
||||
if cfg.LogLevel != nil && *cfg.LogLevel != "" {
|
||||
|
@ -42,19 +44,22 @@ func (s *DummyPlugin) Notify(ctx context.Context, notification *protobufs.Notifi
|
|||
logger.Debug(notification.Text)
|
||||
|
||||
if cfg.OutputFile != nil && *cfg.OutputFile != "" {
|
||||
f, err := os.OpenFile(*cfg.OutputFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
|
||||
f, err := os.OpenFile(*cfg.OutputFile, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644)
|
||||
if err != nil {
|
||||
logger.Error(fmt.Sprintf("Cannot open notification file: %s", err))
|
||||
}
|
||||
|
||||
if _, err := f.WriteString(notification.Text + "\n"); err != nil {
|
||||
f.Close()
|
||||
logger.Error(fmt.Sprintf("Cannot write notification to file: %s", err))
|
||||
}
|
||||
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
logger.Error(fmt.Sprintf("Cannot close notification file: %s", err))
|
||||
}
|
||||
}
|
||||
|
||||
fmt.Println(notification.Text)
|
||||
|
||||
return &protobufs.Empty{}, nil
|
||||
|
@ -64,11 +69,12 @@ func (s *DummyPlugin) Configure(ctx context.Context, config *protobufs.Config) (
|
|||
d := PluginConfig{}
|
||||
err := yaml.Unmarshal(config.Config, &d)
|
||||
s.PluginConfigByName[d.Name] = d
|
||||
|
||||
return &protobufs.Empty{}, err
|
||||
}
|
||||
|
||||
func main() {
|
||||
var handshake = plugin.HandshakeConfig{
|
||||
handshake := plugin.HandshakeConfig{
|
||||
ProtocolVersion: 1,
|
||||
MagicCookieKey: "CROWDSEC_PLUGIN_KEY",
|
||||
MagicCookieValue: os.Getenv("CROWDSEC_PLUGIN_KEY"),
|
||||
|
|
|
@ -2,15 +2,17 @@ package main
|
|||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
||||
"github.com/hashicorp/go-hclog"
|
||||
plugin "github.com/hashicorp/go-plugin"
|
||||
mail "github.com/xhit/go-simple-mail/v2"
|
||||
"gopkg.in/yaml.v2"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
||||
)
|
||||
|
||||
var baseLogger hclog.Logger = hclog.New(&hclog.LoggerOptions{
|
||||
|
@ -72,19 +74,20 @@ func (n *EmailPlugin) Configure(ctx context.Context, config *protobufs.Config) (
|
|||
}
|
||||
|
||||
if d.Name == "" {
|
||||
return nil, fmt.Errorf("name is required")
|
||||
return nil, errors.New("name is required")
|
||||
}
|
||||
|
||||
if d.SMTPHost == "" {
|
||||
return nil, fmt.Errorf("SMTP host is not set")
|
||||
return nil, errors.New("SMTP host is not set")
|
||||
}
|
||||
|
||||
if d.ReceiverEmails == nil || len(d.ReceiverEmails) == 0 {
|
||||
return nil, fmt.Errorf("receiver emails are not set")
|
||||
return nil, errors.New("receiver emails are not set")
|
||||
}
|
||||
|
||||
n.ConfigByName[d.Name] = d
|
||||
baseLogger.Debug(fmt.Sprintf("Email plugin '%s' use SMTP host '%s:%d'", d.Name, d.SMTPHost, d.SMTPPort))
|
||||
|
||||
return &protobufs.Empty{}, nil
|
||||
}
|
||||
|
||||
|
@ -92,6 +95,7 @@ func (n *EmailPlugin) Notify(ctx context.Context, notification *protobufs.Notifi
|
|||
if _, ok := n.ConfigByName[notification.Name]; !ok {
|
||||
return nil, fmt.Errorf("invalid plugin config name %s", notification.Name)
|
||||
}
|
||||
|
||||
cfg := n.ConfigByName[notification.Name]
|
||||
|
||||
logger := baseLogger.Named(cfg.Name)
|
||||
|
@ -117,6 +121,7 @@ func (n *EmailPlugin) Notify(ctx context.Context, notification *protobufs.Notifi
|
|||
server.ConnectTimeout, err = time.ParseDuration(cfg.ConnectTimeout)
|
||||
if err != nil {
|
||||
logger.Warn(fmt.Sprintf("invalid connect timeout '%s', using default '10s'", cfg.ConnectTimeout))
|
||||
|
||||
server.ConnectTimeout = 10 * time.Second
|
||||
}
|
||||
}
|
||||
|
@ -125,15 +130,18 @@ func (n *EmailPlugin) Notify(ctx context.Context, notification *protobufs.Notifi
|
|||
server.SendTimeout, err = time.ParseDuration(cfg.SendTimeout)
|
||||
if err != nil {
|
||||
logger.Warn(fmt.Sprintf("invalid send timeout '%s', using default '10s'", cfg.SendTimeout))
|
||||
|
||||
server.SendTimeout = 10 * time.Second
|
||||
}
|
||||
}
|
||||
|
||||
logger.Debug("making smtp connection")
|
||||
|
||||
smtpClient, err := server.Connect()
|
||||
if err != nil {
|
||||
return &protobufs.Empty{}, err
|
||||
}
|
||||
|
||||
logger.Debug("smtp connection done")
|
||||
|
||||
email := mail.NewMSG()
|
||||
|
@ -146,12 +154,14 @@ func (n *EmailPlugin) Notify(ctx context.Context, notification *protobufs.Notifi
|
|||
if err != nil {
|
||||
return &protobufs.Empty{}, err
|
||||
}
|
||||
|
||||
logger.Info(fmt.Sprintf("sent email to %v", cfg.ReceiverEmails))
|
||||
|
||||
return &protobufs.Empty{}, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
var handshake = plugin.HandshakeConfig{
|
||||
handshake := plugin.HandshakeConfig{
|
||||
ProtocolVersion: 1,
|
||||
MagicCookieKey: "CROWDSEC_PLUGIN_KEY",
|
||||
MagicCookieValue: os.Getenv("CROWDSEC_PLUGIN_KEY"),
|
||||
|
|
|
@ -12,10 +12,11 @@ import (
|
|||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
||||
"github.com/hashicorp/go-hclog"
|
||||
plugin "github.com/hashicorp/go-plugin"
|
||||
"gopkg.in/yaml.v2"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
||||
)
|
||||
|
||||
type PluginConfig struct {
|
||||
|
@ -90,18 +91,23 @@ func getTLSClient(c *PluginConfig) error {
|
|||
|
||||
tlsConfig.Certificates = []tls.Certificate{cert}
|
||||
}
|
||||
|
||||
transport := &http.Transport{
|
||||
TLSClientConfig: tlsConfig,
|
||||
}
|
||||
|
||||
if c.UnixSocket != "" {
|
||||
logger.Info(fmt.Sprintf("Using socket '%s'", c.UnixSocket))
|
||||
|
||||
transport.DialContext = func(_ context.Context, _, _ string) (net.Conn, error) {
|
||||
return net.Dial("unix", strings.TrimSuffix(c.UnixSocket, "/"))
|
||||
}
|
||||
}
|
||||
|
||||
c.Client = &http.Client{
|
||||
Transport: transport,
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
|
@ -109,6 +115,7 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
|
|||
if _, ok := s.PluginConfigByName[notification.Name]; !ok {
|
||||
return nil, fmt.Errorf("invalid plugin config name %s", notification.Name)
|
||||
}
|
||||
|
||||
cfg := s.PluginConfigByName[notification.Name]
|
||||
|
||||
if cfg.LogLevel != nil && *cfg.LogLevel != "" {
|
||||
|
@ -121,11 +128,14 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
|
|||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
for headerName, headerValue := range cfg.Headers {
|
||||
logger.Debug(fmt.Sprintf("adding header %s: %s", headerName, headerValue))
|
||||
request.Header.Add(headerName, headerValue)
|
||||
}
|
||||
|
||||
logger.Debug(fmt.Sprintf("making HTTP %s call to %s with body %s", cfg.Method, cfg.URL, notification.Text))
|
||||
|
||||
resp, err := cfg.Client.Do(request.WithContext(ctx))
|
||||
if err != nil {
|
||||
logger.Error(fmt.Sprintf("Failed to make HTTP request : %s", err))
|
||||
|
@ -135,7 +145,7 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
|
|||
|
||||
respData, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to read response body got error %s", err)
|
||||
return nil, fmt.Errorf("failed to read response body got error %w", err)
|
||||
}
|
||||
|
||||
logger.Debug(fmt.Sprintf("got response %s", string(respData)))
|
||||
|
@ -143,6 +153,7 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
|
|||
if resp.StatusCode < 200 || resp.StatusCode >= 300 {
|
||||
logger.Warn(fmt.Sprintf("HTTP server returned non 200 status code: %d", resp.StatusCode))
|
||||
logger.Debug(fmt.Sprintf("HTTP server returned body: %s", string(respData)))
|
||||
|
||||
return &protobufs.Empty{}, nil
|
||||
}
|
||||
|
||||
|
@ -151,21 +162,25 @@ func (s *HTTPPlugin) Notify(ctx context.Context, notification *protobufs.Notific
|
|||
|
||||
func (s *HTTPPlugin) Configure(ctx context.Context, config *protobufs.Config) (*protobufs.Empty, error) {
|
||||
d := PluginConfig{}
|
||||
|
||||
err := yaml.Unmarshal(config.Config, &d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
err = getTLSClient(&d)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
s.PluginConfigByName[d.Name] = d
|
||||
logger.Debug(fmt.Sprintf("HTTP plugin '%s' use URL '%s'", d.Name, d.URL))
|
||||
|
||||
return &protobufs.Empty{}, err
|
||||
}
|
||||
|
||||
func main() {
|
||||
var handshake = plugin.HandshakeConfig{
|
||||
handshake := plugin.HandshakeConfig{
|
||||
ProtocolVersion: 1,
|
||||
MagicCookieKey: "CROWDSEC_PLUGIN_KEY",
|
||||
MagicCookieValue: os.Getenv("CROWDSEC_PLUGIN_KEY"),
|
||||
|
|
|
@ -5,12 +5,12 @@ import (
|
|||
"fmt"
|
||||
"os"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
||||
"github.com/hashicorp/go-hclog"
|
||||
plugin "github.com/hashicorp/go-plugin"
|
||||
|
||||
"github.com/slack-go/slack"
|
||||
"gopkg.in/yaml.v2"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
||||
)
|
||||
|
||||
type PluginConfig struct {
|
||||
|
@ -33,13 +33,16 @@ func (n *Notify) Notify(ctx context.Context, notification *protobufs.Notificatio
|
|||
if _, ok := n.ConfigByName[notification.Name]; !ok {
|
||||
return nil, fmt.Errorf("invalid plugin config name %s", notification.Name)
|
||||
}
|
||||
|
||||
cfg := n.ConfigByName[notification.Name]
|
||||
|
||||
if cfg.LogLevel != nil && *cfg.LogLevel != "" {
|
||||
logger.SetLevel(hclog.LevelFromString(*cfg.LogLevel))
|
||||
}
|
||||
|
||||
logger.Info(fmt.Sprintf("found notify signal for %s config", notification.Name))
|
||||
logger.Debug(fmt.Sprintf("posting to %s webhook, message %s", cfg.Webhook, notification.Text))
|
||||
|
||||
err := slack.PostWebhookContext(ctx, n.ConfigByName[notification.Name].Webhook, &slack.WebhookMessage{
|
||||
Text: notification.Text,
|
||||
})
|
||||
|
@ -52,16 +55,19 @@ func (n *Notify) Notify(ctx context.Context, notification *protobufs.Notificatio
|
|||
|
||||
func (n *Notify) Configure(ctx context.Context, config *protobufs.Config) (*protobufs.Empty, error) {
|
||||
d := PluginConfig{}
|
||||
|
||||
if err := yaml.Unmarshal(config.Config, &d); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
n.ConfigByName[d.Name] = d
|
||||
logger.Debug(fmt.Sprintf("Slack plugin '%s' use URL '%s'", d.Name, d.Webhook))
|
||||
|
||||
return &protobufs.Empty{}, nil
|
||||
}
|
||||
|
||||
func main() {
|
||||
var handshake = plugin.HandshakeConfig{
|
||||
handshake := plugin.HandshakeConfig{
|
||||
ProtocolVersion: 1,
|
||||
MagicCookieKey: "CROWDSEC_PLUGIN_KEY",
|
||||
MagicCookieValue: os.Getenv("CROWDSEC_PLUGIN_KEY"),
|
||||
|
|
|
@ -10,11 +10,11 @@ import (
|
|||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
||||
"github.com/hashicorp/go-hclog"
|
||||
plugin "github.com/hashicorp/go-plugin"
|
||||
"gopkg.in/yaml.v3"
|
||||
|
||||
"gopkg.in/yaml.v2"
|
||||
"github.com/crowdsecurity/crowdsec/pkg/protobufs"
|
||||
)
|
||||
|
||||
var logger hclog.Logger = hclog.New(&hclog.LoggerOptions{
|
||||
|
@ -44,6 +44,7 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio
|
|||
if _, ok := s.PluginConfigByName[notification.Name]; !ok {
|
||||
return &protobufs.Empty{}, fmt.Errorf("splunk invalid config name %s", notification.Name)
|
||||
}
|
||||
|
||||
cfg := s.PluginConfigByName[notification.Name]
|
||||
|
||||
if cfg.LogLevel != nil && *cfg.LogLevel != "" {
|
||||
|
@ -53,6 +54,7 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio
|
|||
logger.Info(fmt.Sprintf("received notify signal for %s config", notification.Name))
|
||||
|
||||
p := Payload{Event: notification.Text}
|
||||
|
||||
data, err := json.Marshal(p)
|
||||
if err != nil {
|
||||
return &protobufs.Empty{}, err
|
||||
|
@ -65,6 +67,7 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio
|
|||
|
||||
req.Header.Add("Authorization", fmt.Sprintf("Splunk %s", cfg.Token))
|
||||
logger.Debug(fmt.Sprintf("posting event %s to %s", string(data), req.URL))
|
||||
|
||||
resp, err := s.Client.Do(req.WithContext(ctx))
|
||||
if err != nil {
|
||||
return &protobufs.Empty{}, err
|
||||
|
@ -73,15 +76,19 @@ func (s *Splunk) Notify(ctx context.Context, notification *protobufs.Notificatio
|
|||
if resp.StatusCode != http.StatusOK {
|
||||
content, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return &protobufs.Empty{}, fmt.Errorf("got non 200 response and failed to read error %s", err)
|
||||
return &protobufs.Empty{}, fmt.Errorf("got non 200 response and failed to read error %w", err)
|
||||
}
|
||||
|
||||
return &protobufs.Empty{}, fmt.Errorf("got non 200 response %s", string(content))
|
||||
}
|
||||
|
||||
respData, err := io.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return &protobufs.Empty{}, fmt.Errorf("failed to read response body got error %s", err)
|
||||
return &protobufs.Empty{}, fmt.Errorf("failed to read response body got error %w", err)
|
||||
}
|
||||
|
||||
logger.Debug(fmt.Sprintf("got response %s", string(respData)))
|
||||
|
||||
return &protobufs.Empty{}, nil
|
||||
}
|
||||
|
||||
|
@ -90,11 +97,12 @@ func (s *Splunk) Configure(ctx context.Context, config *protobufs.Config) (*prot
|
|||
err := yaml.Unmarshal(config.Config, &d)
|
||||
s.PluginConfigByName[d.Name] = d
|
||||
logger.Debug(fmt.Sprintf("Splunk plugin '%s' use URL '%s'", d.Name, d.URL))
|
||||
|
||||
return &protobufs.Empty{}, err
|
||||
}
|
||||
|
||||
func main() {
|
||||
var handshake = plugin.HandshakeConfig{
|
||||
handshake := plugin.HandshakeConfig{
|
||||
ProtocolVersion: 1,
|
||||
MagicCookieKey: "CROWDSEC_PLUGIN_KEY",
|
||||
MagicCookieValue: os.Getenv("CROWDSEC_PLUGIN_KEY"),
|
||||
|
|
2
go.mod
2
go.mod
|
@ -1,6 +1,6 @@
|
|||
module github.com/crowdsecurity/crowdsec
|
||||
|
||||
go 1.21
|
||||
go 1.22
|
||||
|
||||
// Don't use the toolchain directive to avoid uncontrolled downloads during
|
||||
// a build, especially in sandboxed environments (freebsd, gentoo...).
|
||||
|
|
|
@ -76,26 +76,24 @@ func (c *Config) LoadDBConfig(inCli bool) error {
|
|||
if c.DbConfig.UseWal == nil {
|
||||
dbDir := filepath.Dir(c.DbConfig.DbPath)
|
||||
isNetwork, fsType, err := types.IsNetworkFS(dbDir)
|
||||
if err != nil {
|
||||
switch {
|
||||
case err != nil:
|
||||
log.Warnf("unable to determine if database is on network filesystem: %s", err)
|
||||
log.Warning("You are using sqlite without WAL, this can have a performance impact. If you do not store the database in a network share, set db_config.use_wal to true. Set explicitly to false to disable this warning.")
|
||||
return nil
|
||||
}
|
||||
if isNetwork {
|
||||
case isNetwork:
|
||||
log.Debugf("database is on network filesystem (%s), setting useWal to false", fsType)
|
||||
c.DbConfig.UseWal = ptr.Of(false)
|
||||
} else {
|
||||
default:
|
||||
log.Debugf("database is on local filesystem (%s), setting useWal to true", fsType)
|
||||
c.DbConfig.UseWal = ptr.Of(true)
|
||||
}
|
||||
} else if *c.DbConfig.UseWal {
|
||||
dbDir := filepath.Dir(c.DbConfig.DbPath)
|
||||
isNetwork, fsType, err := types.IsNetworkFS(dbDir)
|
||||
if err != nil {
|
||||
switch {
|
||||
case err != nil:
|
||||
log.Warnf("unable to determine if database is on network filesystem: %s", err)
|
||||
return nil
|
||||
}
|
||||
if isNetwork {
|
||||
case isNetwork:
|
||||
log.Warnf("database seems to be stored on a network share (%s), but useWal is set to true. Proceed at your own risk.", fsType)
|
||||
}
|
||||
}
|
||||
|
|
|
@ -636,14 +636,24 @@ func (c *Client) createAlertChunk(machineID string, owner *ent.Machine, alerts [
|
|||
if len(alertItem.Meta) > 0 {
|
||||
metaBulk := make([]*ent.MetaCreate, len(alertItem.Meta))
|
||||
for i, metaItem := range alertItem.Meta {
|
||||
key := metaItem.Key
|
||||
value := metaItem.Value
|
||||
if len(metaItem.Value) > 4095 {
|
||||
c.Log.Warningf("truncated meta %s : value too long", metaItem.Key)
|
||||
value = value[:4095]
|
||||
}
|
||||
if len(metaItem.Key) > 255 {
|
||||
c.Log.Warningf("truncated meta %s : key too long", metaItem.Key)
|
||||
key = key[:255]
|
||||
}
|
||||
metaBulk[i] = c.Ent.Meta.Create().
|
||||
SetKey(metaItem.Key).
|
||||
SetValue(metaItem.Value)
|
||||
SetKey(key).
|
||||
SetValue(value)
|
||||
}
|
||||
|
||||
metas, err = c.Ent.Meta.CreateBulk(metaBulk...).Save(c.CTX)
|
||||
if err != nil {
|
||||
return nil, errors.Wrapf(BulkError, "creating alert meta: %s", err)
|
||||
c.Log.Warningf("error creating alert meta: %s", err)
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
@ -22,69 +22,70 @@ import (
|
|||
|
||||
type Node struct {
|
||||
FormatVersion string `yaml:"format"`
|
||||
//Enable config + runtime debug of node via config o/
|
||||
// Enable config + runtime debug of node via config o/
|
||||
Debug bool `yaml:"debug,omitempty"`
|
||||
//If enabled, the node (and its child) will report their own statistics
|
||||
// If enabled, the node (and its child) will report their own statistics
|
||||
Profiling bool `yaml:"profiling,omitempty"`
|
||||
//Name, author, description and reference(s) for parser pattern
|
||||
// Name, author, description and reference(s) for parser pattern
|
||||
Name string `yaml:"name,omitempty"`
|
||||
Author string `yaml:"author,omitempty"`
|
||||
Description string `yaml:"description,omitempty"`
|
||||
References []string `yaml:"references,omitempty"`
|
||||
//if debug is present in the node, keep its specific Logger in runtime structure
|
||||
// if debug is present in the node, keep its specific Logger in runtime structure
|
||||
Logger *log.Entry `yaml:"-"`
|
||||
//This is mostly a hack to make writing less repetitive.
|
||||
//relying on stage, we know which field to parse, and we
|
||||
//can also promote log to next stage on success
|
||||
// This is mostly a hack to make writing less repetitive.
|
||||
// relying on stage, we know which field to parse, and we
|
||||
// can also promote log to next stage on success
|
||||
Stage string `yaml:"stage,omitempty"`
|
||||
//OnSuccess allows to tag a node to be able to move log to next stage on success
|
||||
// OnSuccess allows to tag a node to be able to move log to next stage on success
|
||||
OnSuccess string `yaml:"onsuccess,omitempty"`
|
||||
rn string //this is only for us in debug, a random generated name for each node
|
||||
//Filter is executed at runtime (with current log line as context)
|
||||
//and must succeed or node is exited
|
||||
rn string // this is only for us in debug, a random generated name for each node
|
||||
// Filter is executed at runtime (with current log line as context)
|
||||
// and must succeed or node is exited
|
||||
Filter string `yaml:"filter,omitempty"`
|
||||
RunTimeFilter *vm.Program `yaml:"-" json:"-"` //the actual compiled filter
|
||||
//If node has leafs, execute all of them until one asks for a 'break'
|
||||
RunTimeFilter *vm.Program `yaml:"-" json:"-"` // the actual compiled filter
|
||||
// If node has leafs, execute all of them until one asks for a 'break'
|
||||
LeavesNodes []Node `yaml:"nodes,omitempty"`
|
||||
//Flag used to describe when to 'break' or return an 'error'
|
||||
// Flag used to describe when to 'break' or return an 'error'
|
||||
EnrichFunctions EnricherCtx
|
||||
|
||||
/* If the node is actually a leaf, it can have : grok, enrich, statics */
|
||||
//pattern_syntax are named grok patterns that are re-utilized over several grok patterns
|
||||
// pattern_syntax are named grok patterns that are re-utilized over several grok patterns
|
||||
SubGroks yaml.MapSlice `yaml:"pattern_syntax,omitempty"`
|
||||
|
||||
//Holds a grok pattern
|
||||
// Holds a grok pattern
|
||||
Grok GrokPattern `yaml:"grok,omitempty"`
|
||||
//Statics can be present in any type of node and is executed last
|
||||
// Statics can be present in any type of node and is executed last
|
||||
Statics []ExtraField `yaml:"statics,omitempty"`
|
||||
//Stash allows to capture data from the log line and store it in an accessible cache
|
||||
// Stash allows to capture data from the log line and store it in an accessible cache
|
||||
Stash []DataCapture `yaml:"stash,omitempty"`
|
||||
//Whitelists
|
||||
// Whitelists
|
||||
Whitelist Whitelist `yaml:"whitelist,omitempty"`
|
||||
Data []*types.DataSource `yaml:"data,omitempty"`
|
||||
}
|
||||
|
||||
func (n *Node) validate(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
||||
|
||||
//stage is being set automagically
|
||||
// stage is being set automagically
|
||||
if n.Stage == "" {
|
||||
return fmt.Errorf("stage needs to be an existing stage")
|
||||
return errors.New("stage needs to be an existing stage")
|
||||
}
|
||||
|
||||
/* "" behaves like continue */
|
||||
if n.OnSuccess != "continue" && n.OnSuccess != "next_stage" && n.OnSuccess != "" {
|
||||
return fmt.Errorf("onsuccess '%s' not continue,next_stage", n.OnSuccess)
|
||||
}
|
||||
|
||||
if n.Filter != "" && n.RunTimeFilter == nil {
|
||||
return fmt.Errorf("non-empty filter '%s' was not compiled", n.Filter)
|
||||
}
|
||||
|
||||
if n.Grok.RunTimeRegexp != nil || n.Grok.TargetField != "" {
|
||||
if n.Grok.TargetField == "" && n.Grok.ExpValue == "" {
|
||||
return fmt.Errorf("grok requires 'expression' or 'apply_on'")
|
||||
return errors.New("grok requires 'expression' or 'apply_on'")
|
||||
}
|
||||
|
||||
if n.Grok.RegexpName == "" && n.Grok.RegexpValue == "" {
|
||||
return fmt.Errorf("grok needs 'pattern' or 'name'")
|
||||
return errors.New("grok needs 'pattern' or 'name'")
|
||||
}
|
||||
}
|
||||
|
||||
|
@ -93,6 +94,7 @@ func (n *Node) validate(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
|||
if static.ExpValue == "" {
|
||||
return fmt.Errorf("static %d : when method is set, expression must be present", idx)
|
||||
}
|
||||
|
||||
if _, ok := ectx.Registered[static.Method]; !ok {
|
||||
log.Warningf("the method '%s' doesn't exist or the plugin has not been initialized", static.Method)
|
||||
}
|
||||
|
@ -100,6 +102,7 @@ func (n *Node) validate(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
|||
if static.Meta == "" && static.Parsed == "" && static.TargetByName == "" {
|
||||
return fmt.Errorf("static %d : at least one of meta/event/target must be set", idx)
|
||||
}
|
||||
|
||||
if static.Value == "" && static.RunTimeValue == nil {
|
||||
return fmt.Errorf("static %d value or expression must be set", idx)
|
||||
}
|
||||
|
@ -110,40 +113,44 @@ func (n *Node) validate(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
|||
if stash.Name == "" {
|
||||
return fmt.Errorf("stash %d : name must be set", idx)
|
||||
}
|
||||
|
||||
if stash.Value == "" {
|
||||
return fmt.Errorf("stash %s : value expression must be set", stash.Name)
|
||||
}
|
||||
|
||||
if stash.Key == "" {
|
||||
return fmt.Errorf("stash %s : key expression must be set", stash.Name)
|
||||
}
|
||||
|
||||
if stash.TTL == "" {
|
||||
return fmt.Errorf("stash %s : ttl must be set", stash.Name)
|
||||
}
|
||||
|
||||
if stash.Strategy == "" {
|
||||
stash.Strategy = "LRU"
|
||||
}
|
||||
//should be configurable
|
||||
// should be configurable
|
||||
if stash.MaxMapSize == 0 {
|
||||
stash.MaxMapSize = 100
|
||||
}
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[string]interface{}) (bool, error) {
|
||||
var NodeState bool
|
||||
var NodeHasOKGrok bool
|
||||
func (n *Node) processFilter(cachedExprEnv map[string]interface{}) (bool, error) {
|
||||
clog := n.Logger
|
||||
if n.RunTimeFilter == nil {
|
||||
clog.Tracef("Node has not filter, enter")
|
||||
return true, nil
|
||||
}
|
||||
|
||||
cachedExprEnv := expressionEnv
|
||||
|
||||
clog.Tracef("Event entering node")
|
||||
if n.RunTimeFilter != nil {
|
||||
//Evaluate node's filter
|
||||
// Evaluate node's filter
|
||||
output, err := exprhelpers.Run(n.RunTimeFilter, cachedExprEnv, clog, n.Debug)
|
||||
if err != nil {
|
||||
clog.Warningf("failed to run filter : %v", err)
|
||||
clog.Debugf("Event leaving node : ko")
|
||||
|
||||
return false, nil
|
||||
}
|
||||
|
||||
|
@ -156,26 +163,26 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
|||
default:
|
||||
clog.Warningf("Expr '%s' returned non-bool, abort : %T", n.Filter, output)
|
||||
clog.Debugf("Event leaving node : ko")
|
||||
|
||||
return false, nil
|
||||
}
|
||||
NodeState = true
|
||||
} else {
|
||||
clog.Tracef("Node has not filter, enter")
|
||||
NodeState = true
|
||||
}
|
||||
|
||||
if n.Name != "" {
|
||||
NodesHits.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc()
|
||||
}
|
||||
exprErr := error(nil)
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (n *Node) processWhitelist(cachedExprEnv map[string]interface{}, p *types.Event) (bool, error) {
|
||||
var exprErr error
|
||||
|
||||
isWhitelisted := n.CheckIPsWL(p)
|
||||
if !isWhitelisted {
|
||||
isWhitelisted, exprErr = n.CheckExprWL(cachedExprEnv, p)
|
||||
}
|
||||
|
||||
if exprErr != nil {
|
||||
// Previous code returned nil if there was an error, so we keep this behavior
|
||||
return false, nil //nolint:nilerr
|
||||
}
|
||||
|
||||
if isWhitelisted && !p.Whitelisted {
|
||||
p.Whitelisted = true
|
||||
p.WhitelistReason = n.Whitelist.Reason
|
||||
|
@ -185,18 +192,51 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
|||
for k := range p.Overflow.Sources {
|
||||
ips = append(ips, k)
|
||||
}
|
||||
clog.Infof("Ban for %s whitelisted, reason [%s]", strings.Join(ips, ","), n.Whitelist.Reason)
|
||||
|
||||
n.Logger.Infof("Ban for %s whitelisted, reason [%s]", strings.Join(ips, ","), n.Whitelist.Reason)
|
||||
|
||||
p.Overflow.Whitelisted = true
|
||||
}
|
||||
}
|
||||
|
||||
//Process grok if present, should be exclusive with nodes :)
|
||||
return isWhitelisted, nil
|
||||
}
|
||||
|
||||
func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[string]interface{}) (bool, error) {
|
||||
var NodeHasOKGrok bool
|
||||
|
||||
clog := n.Logger
|
||||
|
||||
cachedExprEnv := expressionEnv
|
||||
|
||||
clog.Tracef("Event entering node")
|
||||
|
||||
NodeState, err := n.processFilter(cachedExprEnv)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
if !NodeState {
|
||||
return false, nil
|
||||
}
|
||||
|
||||
if n.Name != "" {
|
||||
NodesHits.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc()
|
||||
}
|
||||
|
||||
isWhitelisted, err := n.processWhitelist(cachedExprEnv, p)
|
||||
if err != nil {
|
||||
return false, err
|
||||
}
|
||||
|
||||
// Process grok if present, should be exclusive with nodes :)
|
||||
gstr := ""
|
||||
|
||||
if n.Grok.RunTimeRegexp != nil {
|
||||
clog.Tracef("Processing grok pattern : %s : %p", n.Grok.RegexpName, n.Grok.RunTimeRegexp)
|
||||
//for unparsed, parsed etc. set sensible defaults to reduce user hassle
|
||||
// for unparsed, parsed etc. set sensible defaults to reduce user hassle
|
||||
if n.Grok.TargetField != "" {
|
||||
//it's a hack to avoid using real reflect
|
||||
// it's a hack to avoid using real reflect
|
||||
if n.Grok.TargetField == "Line.Raw" {
|
||||
gstr = p.Line.Raw
|
||||
} else if val, ok := p.Parsed[n.Grok.TargetField]; ok {
|
||||
|
@ -211,6 +251,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
|||
clog.Warningf("failed to run RunTimeValue : %v", err)
|
||||
NodeState = false
|
||||
}
|
||||
|
||||
switch out := output.(type) {
|
||||
case string:
|
||||
gstr = out
|
||||
|
@ -229,12 +270,14 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
|||
} else {
|
||||
groklabel = n.Grok.RegexpName
|
||||
}
|
||||
|
||||
grok := n.Grok.RunTimeRegexp.Parse(gstr)
|
||||
if len(grok) > 0 {
|
||||
/*tag explicitly that the *current* node had a successful grok pattern. it's important to know success state*/
|
||||
NodeHasOKGrok = true
|
||||
|
||||
clog.Debugf("+ Grok '%s' returned %d entries to merge in Parsed", groklabel, len(grok))
|
||||
//We managed to grok stuff, merged into parse
|
||||
// We managed to grok stuff, merged into parse
|
||||
for k, v := range grok {
|
||||
clog.Debugf("\t.Parsed['%s'] = '%s'", k, v)
|
||||
p.Parsed[k] = v
|
||||
|
@ -246,34 +289,37 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
|||
return false, err
|
||||
}
|
||||
} else {
|
||||
//grok failed, node failed
|
||||
// grok failed, node failed
|
||||
clog.Debugf("+ Grok '%s' didn't return data on '%s'", groklabel, gstr)
|
||||
NodeState = false
|
||||
}
|
||||
|
||||
} else {
|
||||
clog.Tracef("! No grok pattern : %p", n.Grok.RunTimeRegexp)
|
||||
}
|
||||
|
||||
//Process the stash (data collection) if : a grok was present and succeeded, or if there is no grok
|
||||
// Process the stash (data collection) if : a grok was present and succeeded, or if there is no grok
|
||||
if NodeHasOKGrok || n.Grok.RunTimeRegexp == nil {
|
||||
for idx, stash := range n.Stash {
|
||||
var value string
|
||||
var key string
|
||||
var (
|
||||
key string
|
||||
value string
|
||||
)
|
||||
|
||||
if stash.ValueExpression == nil {
|
||||
clog.Warningf("Stash %d has no value expression, skipping", idx)
|
||||
continue
|
||||
}
|
||||
|
||||
if stash.KeyExpression == nil {
|
||||
clog.Warningf("Stash %d has no key expression, skipping", idx)
|
||||
continue
|
||||
}
|
||||
//collect the data
|
||||
// collect the data
|
||||
output, err := exprhelpers.Run(stash.ValueExpression, cachedExprEnv, clog, n.Debug)
|
||||
if err != nil {
|
||||
clog.Warningf("Error while running stash val expression : %v", err)
|
||||
}
|
||||
//can we expect anything else than a string ?
|
||||
// can we expect anything else than a string ?
|
||||
switch output := output.(type) {
|
||||
case string:
|
||||
value = output
|
||||
|
@ -282,12 +328,12 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
|||
continue
|
||||
}
|
||||
|
||||
//collect the key
|
||||
// collect the key
|
||||
output, err = exprhelpers.Run(stash.KeyExpression, cachedExprEnv, clog, n.Debug)
|
||||
if err != nil {
|
||||
clog.Warningf("Error while running stash key expression : %v", err)
|
||||
}
|
||||
//can we expect anything else than a string ?
|
||||
// can we expect anything else than a string ?
|
||||
switch output := output.(type) {
|
||||
case string:
|
||||
key = output
|
||||
|
@ -299,7 +345,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
|||
}
|
||||
}
|
||||
|
||||
//Iterate on leafs
|
||||
// Iterate on leafs
|
||||
for _, leaf := range n.LeavesNodes {
|
||||
ret, err := leaf.process(p, ctx, cachedExprEnv)
|
||||
if err != nil {
|
||||
|
@ -307,7 +353,9 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
|||
clog.Debugf("Event leaving node : ko")
|
||||
return false, err
|
||||
}
|
||||
|
||||
clog.Tracef("\tsub-node (%s) ret : %v (strategy:%s)", leaf.rn, ret, n.OnSuccess)
|
||||
|
||||
if ret {
|
||||
NodeState = true
|
||||
/* if child is successful, stop processing */
|
||||
|
@ -328,12 +376,14 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
|||
|
||||
clog.Tracef("State after nodes : %v", NodeState)
|
||||
|
||||
//grok or leafs failed, don't process statics
|
||||
// grok or leafs failed, don't process statics
|
||||
if !NodeState {
|
||||
if n.Name != "" {
|
||||
NodesHitsKo.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc()
|
||||
}
|
||||
|
||||
clog.Debugf("Event leaving node : ko")
|
||||
|
||||
return NodeState, nil
|
||||
}
|
||||
|
||||
|
@ -360,9 +410,10 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
|||
if NodeState {
|
||||
clog.Debugf("Event leaving node : ok")
|
||||
log.Tracef("node is successful, check strategy")
|
||||
|
||||
if n.OnSuccess == "next_stage" {
|
||||
idx := stageidx(p.Stage, ctx.Stages)
|
||||
//we're at the last stage
|
||||
// we're at the last stage
|
||||
if idx+1 == len(ctx.Stages) {
|
||||
clog.Debugf("node reached the last stage : %s", p.Stage)
|
||||
} else {
|
||||
|
@ -375,15 +426,16 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri
|
|||
} else {
|
||||
clog.Debugf("Event leaving node : ko")
|
||||
}
|
||||
|
||||
clog.Tracef("Node successful, continue")
|
||||
|
||||
return NodeState, nil
|
||||
}
|
||||
|
||||
func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
||||
var err error
|
||||
var valid bool
|
||||
|
||||
valid = false
|
||||
valid := false
|
||||
|
||||
dumpr := spew.ConfigState{MaxDepth: 1, DisablePointerAddresses: true}
|
||||
n.rn = seed.Generate()
|
||||
|
@ -393,10 +445,11 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
|||
/* if the node has debugging enabled, create a specific logger with debug
|
||||
that will be used only for processing this node ;) */
|
||||
if n.Debug {
|
||||
var clog = log.New()
|
||||
clog := log.New()
|
||||
if err = types.ConfigureLogger(clog); err != nil {
|
||||
log.Fatalf("While creating bucket-specific logger : %s", err)
|
||||
}
|
||||
|
||||
clog.SetLevel(log.DebugLevel)
|
||||
n.Logger = clog.WithFields(log.Fields{
|
||||
"id": n.rn,
|
||||
|
@ -414,7 +467,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
|||
|
||||
n.Logger.Tracef("Compiling : %s", dumpr.Sdump(n))
|
||||
|
||||
//compile filter if present
|
||||
// compile filter if present
|
||||
if n.Filter != "" {
|
||||
n.RunTimeFilter, err = expr.Compile(n.Filter, exprhelpers.GetExprOptions(map[string]interface{}{"evt": &types.Event{}})...)
|
||||
if err != nil {
|
||||
|
@ -425,12 +478,15 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
|||
/* handle pattern_syntax and groks */
|
||||
for _, pattern := range n.SubGroks {
|
||||
n.Logger.Tracef("Adding subpattern '%s' : '%s'", pattern.Key, pattern.Value)
|
||||
|
||||
if err = pctx.Grok.Add(pattern.Key.(string), pattern.Value.(string)); err != nil {
|
||||
if errors.Is(err, grokky.ErrAlreadyExist) {
|
||||
n.Logger.Warningf("grok '%s' already registred", pattern.Key)
|
||||
continue
|
||||
}
|
||||
|
||||
n.Logger.Errorf("Unable to compile subpattern %s : %v", pattern.Key, err)
|
||||
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
@ -438,28 +494,36 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
|||
/* load grok by name or compile in-place */
|
||||
if n.Grok.RegexpName != "" {
|
||||
n.Logger.Tracef("+ Regexp Compilation '%s'", n.Grok.RegexpName)
|
||||
|
||||
n.Grok.RunTimeRegexp, err = pctx.Grok.Get(n.Grok.RegexpName)
|
||||
if err != nil {
|
||||
return fmt.Errorf("unable to find grok '%s' : %v", n.Grok.RegexpName, err)
|
||||
}
|
||||
|
||||
if n.Grok.RunTimeRegexp == nil {
|
||||
return fmt.Errorf("empty grok '%s'", n.Grok.RegexpName)
|
||||
}
|
||||
|
||||
n.Logger.Tracef("%s regexp: %s", n.Grok.RegexpName, n.Grok.RunTimeRegexp.String())
|
||||
|
||||
valid = true
|
||||
} else if n.Grok.RegexpValue != "" {
|
||||
if strings.HasSuffix(n.Grok.RegexpValue, "\n") {
|
||||
n.Logger.Debugf("Beware, pattern ends with \\n : '%s'", n.Grok.RegexpValue)
|
||||
}
|
||||
|
||||
n.Grok.RunTimeRegexp, err = pctx.Grok.Compile(n.Grok.RegexpValue)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to compile grok '%s': %v", n.Grok.RegexpValue, err)
|
||||
}
|
||||
|
||||
if n.Grok.RunTimeRegexp == nil {
|
||||
// We shouldn't be here because compilation succeeded, so regexp shouldn't be nil
|
||||
return fmt.Errorf("grok compilation failure: %s", n.Grok.RegexpValue)
|
||||
}
|
||||
|
||||
n.Logger.Tracef("%s regexp : %s", n.Grok.RegexpValue, n.Grok.RunTimeRegexp.String())
|
||||
|
||||
valid = true
|
||||
}
|
||||
|
||||
|
@ -473,7 +537,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
|||
}
|
||||
|
||||
/* load grok statics */
|
||||
//compile expr statics if present
|
||||
// compile expr statics if present
|
||||
for idx := range n.Grok.Statics {
|
||||
if n.Grok.Statics[idx].ExpValue != "" {
|
||||
n.Grok.Statics[idx].RunTimeValue, err = expr.Compile(n.Grok.Statics[idx].ExpValue,
|
||||
|
@ -482,6 +546,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
|||
return err
|
||||
}
|
||||
}
|
||||
|
||||
valid = true
|
||||
}
|
||||
|
||||
|
@ -505,7 +570,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
|||
}
|
||||
|
||||
logLvl := n.Logger.Logger.GetLevel()
|
||||
//init the cache, does it make sense to create it here just to be sure everything is fine ?
|
||||
// init the cache, does it make sense to create it here just to be sure everything is fine ?
|
||||
if err = cache.CacheInit(cache.CacheCfg{
|
||||
Size: n.Stash[i].MaxMapSize,
|
||||
TTL: n.Stash[i].TTLVal,
|
||||
|
@ -526,14 +591,18 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
|||
if !n.LeavesNodes[idx].Debug && n.Debug {
|
||||
n.LeavesNodes[idx].Debug = true
|
||||
}
|
||||
|
||||
if !n.LeavesNodes[idx].Profiling && n.Profiling {
|
||||
n.LeavesNodes[idx].Profiling = true
|
||||
}
|
||||
|
||||
n.LeavesNodes[idx].Stage = n.Stage
|
||||
|
||||
err = n.LeavesNodes[idx].compile(pctx, ectx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
valid = true
|
||||
}
|
||||
|
||||
|
@ -546,6 +615,7 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
|||
return err
|
||||
}
|
||||
}
|
||||
|
||||
valid = true
|
||||
}
|
||||
|
||||
|
@ -554,13 +624,15 @@ func (n *Node) compile(pctx *UnixParserCtx, ectx EnricherCtx) error {
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
valid = valid || whitelistValid
|
||||
|
||||
if !valid {
|
||||
/* node is empty, error force return */
|
||||
n.Logger.Error("Node is empty or invalid, abort")
|
||||
n.Stage = ""
|
||||
return fmt.Errorf("Node is empty")
|
||||
|
||||
return errors.New("Node is empty")
|
||||
}
|
||||
|
||||
if err := n.validate(pctx, ectx); err != nil {
|
||||
|
|
|
@ -1,9 +1,10 @@
|
|||
//go:build !windows
|
||||
//go:build !windows && !freebsd
|
||||
|
||||
package types
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"golang.org/x/sys/unix"
|
||||
)
|
||||
|
||||
|
@ -92,6 +93,7 @@ var fsTypeMapping map[int64]string = map[int64]string{
|
|||
0xabba1974: "xenfs",
|
||||
0x012ff7b4: "xenix",
|
||||
0x58465342: "xfs",
|
||||
0x2fc12fc1: "zfs",
|
||||
}
|
||||
|
||||
func GetFSType(path string) (string, error) {
|
||||
|
|
25
pkg/types/getfstype_freebsd.go
Normal file
25
pkg/types/getfstype_freebsd.go
Normal file
|
@ -0,0 +1,25 @@
|
|||
//go:build freebsd
|
||||
|
||||
package types
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"syscall"
|
||||
)
|
||||
|
||||
func GetFSType(path string) (string, error) {
|
||||
var fsStat syscall.Statfs_t
|
||||
|
||||
if err := syscall.Statfs(path, &fsStat); err != nil {
|
||||
return "", fmt.Errorf("failed to get filesystem type: %w", err)
|
||||
}
|
||||
|
||||
bs := fsStat.Fstypename
|
||||
|
||||
b := make([]byte, len(bs))
|
||||
for i, v := range bs {
|
||||
b[i] = byte(v)
|
||||
}
|
||||
|
||||
return string(b), nil
|
||||
}
|
Loading…
Reference in a new issue