From 534328ca30eec7731e7d8e9140bb67f80102500c Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 23 May 2023 10:52:47 +0200 Subject: [PATCH] decouple bouncer dependencies: use go-cs-lib/pkg/* (#2216) * decouple bouncer dependencies: use go-cs-lib/pkg/trace * decouple bouncer dependencies: use go-cs-lib/pkg/version * decouple bouncer dependencies: use go-cs-lib/pkg/yamlpatch * decouple bouncer dependencies: use go-cs-lib/pkg/csstring * unused import --- Makefile | 6 +- cmd/crowdsec-cli/alerts.go | 5 +- cmd/crowdsec-cli/capi.go | 7 +- cmd/crowdsec-cli/console.go | 5 +- cmd/crowdsec-cli/decisions.go | 5 +- cmd/crowdsec-cli/lapi.go | 7 +- cmd/crowdsec-cli/metrics.go | 4 +- cmd/crowdsec-cli/notifications.go | 5 +- cmd/crowdsec-cli/support.go | 4 +- cmd/crowdsec-cli/utils.go | 4 +- cmd/crowdsec/api.go | 7 +- cmd/crowdsec/crowdsec.go | 12 +- cmd/crowdsec/metrics.go | 9 +- cmd/crowdsec/output.go | 5 +- cmd/crowdsec/run_in_svc.go | 9 +- cmd/crowdsec/run_in_svc_windows.go | 9 +- cmd/crowdsec/serve.go | 4 +- debian/rules | 6 - go.mod | 5 +- go.sum | 10 +- pkg/acquisition/acquisition.go | 6 +- pkg/acquisition/modules/file/file.go | 6 +- .../modules/journalctl/journalctl.go | 6 +- pkg/acquisition/modules/kafka/kafka.go | 4 +- pkg/acquisition/modules/kinesis/kinesis.go | 6 +- .../modules/kubernetesaudit/k8s_audit.go | 4 +- pkg/acquisition/modules/syslog/syslog.go | 4 +- .../wineventlog/wineventlog_windows.go | 4 +- pkg/apiclient/alerts_service_test.go | 11 +- pkg/apiclient/auth_service_test.go | 11 +- pkg/apiclient/client_http_test.go | 7 +- pkg/apiclient/client_test.go | 13 +- pkg/apiclient/decisions_service_test.go | 5 +- pkg/apiclient/heartbeat.go | 5 +- pkg/apiserver/apic.go | 14 +- pkg/apiserver/apic_test.go | 21 +- pkg/apiserver/apiserver.go | 6 +- pkg/apiserver/apiserver_test.go | 5 +- pkg/apiserver/papi.go | 7 +- pkg/csconfig/api.go | 3 +- pkg/csconfig/config.go | 5 +- pkg/csconfig/profiles.go | 3 +- pkg/csconfig/simulation.go | 2 +- pkg/csplugin/broker.go | 3 +- pkg/csstring/expand.go | 75 ----- pkg/csstring/expand_test.go | 98 ------ pkg/cwversion/version.go | 42 +-- pkg/leakybucket/bucket.go | 4 +- pkg/leakybucket/manager_load.go | 2 +- pkg/metabase/api.go | 5 +- pkg/parser/stage.go | 2 +- pkg/types/utils.go | 40 --- pkg/yamlpatch/merge.go | 168 ---------- pkg/yamlpatch/merge_test.go | 238 ------------- pkg/yamlpatch/patcher.go | 173 ---------- pkg/yamlpatch/patcher_test.go | 313 ------------------ pkg/yamlpatch/testdata/base.yaml | 13 - pkg/yamlpatch/testdata/expect.yaml | 13 - pkg/yamlpatch/testdata/production.yaml | 13 - 59 files changed, 191 insertions(+), 1297 deletions(-) delete mode 100644 pkg/csstring/expand.go delete mode 100644 pkg/csstring/expand_test.go delete mode 100644 pkg/yamlpatch/merge.go delete mode 100644 pkg/yamlpatch/merge_test.go delete mode 100644 pkg/yamlpatch/patcher.go delete mode 100644 pkg/yamlpatch/patcher_test.go delete mode 100644 pkg/yamlpatch/testdata/base.yaml delete mode 100644 pkg/yamlpatch/testdata/expect.yaml delete mode 100644 pkg/yamlpatch/testdata/production.yaml diff --git a/Makefile b/Makefile index 05eb0586b..6cbb97109 100644 --- a/Makefile +++ b/Makefile @@ -27,10 +27,10 @@ BUILD_CMD = build GO_MODULE_NAME = github.com/crowdsecurity/crowdsec LD_OPTS_VARS= \ --X '$(GO_MODULE_NAME)/pkg/cwversion.Version=$(BUILD_VERSION)' \ --X '$(GO_MODULE_NAME)/pkg/cwversion.BuildDate=$(BUILD_TIMESTAMP)' \ +-X 'github.com/crowdsecurity/go-cs-lib/pkg/version.Version=$(BUILD_VERSION)' \ +-X 'github.com/crowdsecurity/go-cs-lib/pkg/version.BuildDate=$(BUILD_TIMESTAMP)' \ +-X 'github.com/crowdsecurity/go-cs-lib/pkg/version.Tag=$(BUILD_TAG)' \ -X '$(GO_MODULE_NAME)/pkg/cwversion.Codename=$(BUILD_CODENAME)' \ --X '$(GO_MODULE_NAME)/pkg/cwversion.Tag=$(BUILD_TAG)' \ -X '$(GO_MODULE_NAME)/pkg/csconfig.defaultConfigDir=$(DEFAULT_CONFIGDIR)' \ -X '$(GO_MODULE_NAME)/pkg/csconfig.defaultDataDir=$(DEFAULT_DATADIR)' diff --git a/cmd/crowdsec-cli/alerts.go b/cmd/crowdsec-cli/alerts.go index 0c55e89c6..25cb26515 100644 --- a/cmd/crowdsec-cli/alerts.go +++ b/cmd/crowdsec-cli/alerts.go @@ -20,8 +20,9 @@ import ( "github.com/spf13/cobra" "gopkg.in/yaml.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + "github.com/crowdsecurity/crowdsec/pkg/apiclient" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -219,7 +220,7 @@ func NewAlertsCmd() *cobra.Command { Client, err = apiclient.NewClient(&apiclient.Config{ MachineID: csConfig.API.Client.Credentials.Login, Password: strfmt.Password(csConfig.API.Client.Credentials.Password), - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, VersionPrefix: "v1", }) diff --git a/cmd/crowdsec-cli/capi.go b/cmd/crowdsec-cli/capi.go index 0cecad0fa..e67d33ce4 100644 --- a/cmd/crowdsec-cli/capi.go +++ b/cmd/crowdsec-cli/capi.go @@ -6,10 +6,11 @@ import ( "net/url" "os" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + "github.com/crowdsecurity/crowdsec/pkg/apiclient" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/crowdsecurity/crowdsec/pkg/fflag" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -70,7 +71,7 @@ func NewCapiRegisterCmd() *cobra.Command { _, err = apiclient.RegisterClient(&apiclient.Config{ MachineID: capiUser, Password: password, - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiurl, VersionPrefix: CAPIURLPrefix, }, nil) @@ -164,7 +165,7 @@ func NewCapiStatusCmd() *cobra.Command { log.Fatalf("no scenarios installed, abort") } - Client, err = apiclient.NewDefaultClient(apiurl, CAPIURLPrefix, fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), nil) + Client, err = apiclient.NewDefaultClient(apiurl, CAPIURLPrefix, fmt.Sprintf("crowdsec/%s", version.String()), nil) if err != nil { log.Fatalf("init default client: %s", err) } diff --git a/cmd/crowdsec-cli/console.go b/cmd/crowdsec-cli/console.go index 6bd4ef88b..f6c51a33e 100644 --- a/cmd/crowdsec-cli/console.go +++ b/cmd/crowdsec-cli/console.go @@ -16,10 +16,11 @@ import ( "github.com/spf13/cobra" "gopkg.in/yaml.v3" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + "github.com/crowdsecurity/crowdsec/pkg/apiclient" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/crowdsecurity/crowdsec/pkg/fflag" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -101,7 +102,7 @@ After running this command your will need to validate the enrollment in the weba MachineID: csConfig.API.Server.OnlineClient.Credentials.Login, Password: password, Scenarios: scenarios, - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, VersionPrefix: "v3", }) diff --git a/cmd/crowdsec-cli/decisions.go b/cmd/crowdsec-cli/decisions.go index dc289c171..e11c56ad3 100644 --- a/cmd/crowdsec-cli/decisions.go +++ b/cmd/crowdsec-cli/decisions.go @@ -19,8 +19,9 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + "github.com/crowdsecurity/crowdsec/pkg/apiclient" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -120,7 +121,7 @@ func NewDecisionsCmd() *cobra.Command { Client, err = apiclient.NewClient(&apiclient.Config{ MachineID: csConfig.API.Client.Credentials.Login, Password: password, - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiurl, VersionPrefix: "v1", }) diff --git a/cmd/crowdsec-cli/lapi.go b/cmd/crowdsec-cli/lapi.go index ae6d3f33f..6034e5a7e 100644 --- a/cmd/crowdsec-cli/lapi.go +++ b/cmd/crowdsec-cli/lapi.go @@ -14,11 +14,12 @@ import ( "github.com/spf13/cobra" "gopkg.in/yaml.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + "github.com/crowdsecurity/crowdsec/pkg/alertcontext" "github.com/crowdsecurity/crowdsec/pkg/apiclient" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/parser" @@ -51,7 +52,7 @@ func runLapiStatus(cmd *cobra.Command, args []string) error { Client, err = apiclient.NewDefaultClient(apiurl, LAPIURLPrefix, - fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + fmt.Sprintf("crowdsec/%s", version.String()), nil) if err != nil { log.Fatalf("init default client: %s", err) @@ -122,7 +123,7 @@ func runLapiRegister(cmd *cobra.Command, args []string) error { _, err = apiclient.RegisterClient(&apiclient.Config{ MachineID: lapiUser, Password: password, - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiurl, VersionPrefix: LAPIURLPrefix, }, nil) diff --git a/cmd/crowdsec-cli/metrics.go b/cmd/crowdsec-cli/metrics.go index 03592a9f7..f2577db8f 100644 --- a/cmd/crowdsec-cli/metrics.go +++ b/cmd/crowdsec-cli/metrics.go @@ -17,7 +17,7 @@ import ( "github.com/spf13/cobra" "gopkg.in/yaml.v2" - "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" ) // FormatPrometheusMetrics is a complete rip from prom2json @@ -32,7 +32,7 @@ func FormatPrometheusMetrics(out io.Writer, url string, formatType string) error // Timeout early if the server doesn't even return the headers. transport.ResponseHeaderTimeout = time.Minute go func() { - defer types.CatchPanic("crowdsec/ShowPrometheus") + defer trace.CatchPanic("crowdsec/ShowPrometheus") err := prom2json.FetchMetricFamilies(url, mfChan, transport) if err != nil { log.Fatalf("failed to fetch prometheus metrics : %v", err) diff --git a/cmd/crowdsec-cli/notifications.go b/cmd/crowdsec-cli/notifications.go index f9b51b381..ebbefd6b7 100644 --- a/cmd/crowdsec-cli/notifications.go +++ b/cmd/crowdsec-cli/notifications.go @@ -20,11 +20,12 @@ import ( "github.com/spf13/cobra" "gopkg.in/tomb.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + "github.com/crowdsecurity/crowdsec/pkg/apiclient" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csplugin" "github.com/crowdsecurity/crowdsec/pkg/csprofiles" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" ) @@ -273,7 +274,7 @@ cscli notifications reinject -a '{"remediation": true,"scenario":"not client, err := apiclient.NewClient(&apiclient.Config{ MachineID: csConfig.API.Client.Credentials.Login, Password: strfmt.Password(csConfig.API.Client.Credentials.Password), - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, VersionPrefix: "v1", }) diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index e7110ae70..013abf4b2 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -18,6 +18,8 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + "github.com/crowdsecurity/crowdsec/pkg/apiclient" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/cwversion" @@ -182,7 +184,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str Client, err = apiclient.NewDefaultClient(apiurl, prefix, - fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + fmt.Sprintf("crowdsec/%s", version.String()), nil) if err != nil { return []byte(fmt.Sprintf("could not init client: %s", err)) diff --git a/cmd/crowdsec-cli/utils.go b/cmd/crowdsec-cli/utils.go index 01817ac09..fa16cf4bd 100644 --- a/cmd/crowdsec-cli/utils.go +++ b/cmd/crowdsec-cli/utils.go @@ -21,6 +21,8 @@ import ( "github.com/texttheater/golang-levenshtein/levenshtein" "gopkg.in/yaml.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -515,7 +517,7 @@ func GetPrometheusMetric(url string) []*prom2json.Family { transport.ResponseHeaderTimeout = time.Minute go func() { - defer types.CatchPanic("crowdsec/GetPrometheusMetric") + defer trace.CatchPanic("crowdsec/GetPrometheusMetric") err := prom2json.FetchMetricFamilies(url, mfChan, transport) if err != nil { log.Fatalf("failed to fetch prometheus metrics : %v", err) diff --git a/cmd/crowdsec/api.go b/cmd/crowdsec/api.go index daae1ca7a..3ce249d4c 100644 --- a/cmd/crowdsec/api.go +++ b/cmd/crowdsec/api.go @@ -7,9 +7,10 @@ import ( "github.com/pkg/errors" log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/crowdsec/pkg/apiserver" "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/types" ) func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) { @@ -52,9 +53,9 @@ func initAPIServer(cConfig *csconfig.Config) (*apiserver.APIServer, error) { func serveAPIServer(apiServer *apiserver.APIServer, apiReady chan bool) { apiTomb.Go(func() error { - defer types.CatchPanic("crowdsec/serveAPIServer") + defer trace.CatchPanic("crowdsec/serveAPIServer") go func() { - defer types.CatchPanic("crowdsec/runAPIServer") + defer trace.CatchPanic("crowdsec/runAPIServer") log.Debugf("serving API after %s ms", time.Since(crowdsecT0)) if err := apiServer.Run(apiReady); err != nil { log.Fatal(err) diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 10ac48aba..8b4487e15 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -8,6 +8,8 @@ import ( "path/filepath" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/crowdsec/pkg/acquisition" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" @@ -53,7 +55,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error { parserWg.Add(1) for i := 0; i < cConfig.Crowdsec.ParserRoutinesCount; i++ { parsersTomb.Go(func() error { - defer types.CatchPanic("crowdsec/runParse") + defer trace.CatchPanic("crowdsec/runParse") if err := runParse(inputLineChan, inputEventChan, *parsers.Ctx, parsers.Nodes); err != nil { //this error will never happen as parser.Parse is not able to return errors log.Fatalf("starting parse error : %s", err) return err @@ -79,7 +81,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error { for i := 0; i < cConfig.Crowdsec.BucketsRoutinesCount; i++ { bucketsTomb.Go(func() error { - defer types.CatchPanic("crowdsec/runPour") + defer trace.CatchPanic("crowdsec/runPour") if err := runPour(inputEventChan, holders, buckets, cConfig); err != nil { log.Fatalf("starting pour error : %s", err) return err @@ -97,7 +99,7 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error { outputWg.Add(1) for i := 0; i < cConfig.Crowdsec.OutputRoutinesCount; i++ { outputsTomb.Go(func() error { - defer types.CatchPanic("crowdsec/runOutput") + defer trace.CatchPanic("crowdsec/runOutput") if err := runOutput(inputEventChan, outputEventChan, buckets, *parsers.Povfwctx, parsers.Povfwnodes, *cConfig.API.Client.Credentials); err != nil { log.Fatalf("starting outputs error : %s", err) return err @@ -132,9 +134,9 @@ func runCrowdsec(cConfig *csconfig.Config, parsers *parser.Parsers) error { func serveCrowdsec(parsers *parser.Parsers, cConfig *csconfig.Config, agentReady chan bool) { crowdsecTomb.Go(func() error { - defer types.CatchPanic("crowdsec/serveCrowdsec") + defer trace.CatchPanic("crowdsec/serveCrowdsec") go func() { - defer types.CatchPanic("crowdsec/runCrowdsec") + defer trace.CatchPanic("crowdsec/runCrowdsec") // this logs every time, even at config reload log.Debugf("running agent after %s ms", time.Since(crowdsecT0)) agentReady <- true diff --git a/cmd/crowdsec/metrics.go b/cmd/crowdsec/metrics.go index 6b549603f..8e87eecd0 100644 --- a/cmd/crowdsec/metrics.go +++ b/cmd/crowdsec/metrics.go @@ -9,15 +9,16 @@ import ( "github.com/prometheus/client_golang/prometheus/promhttp" log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + v1 "github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers/v1" "github.com/crowdsecurity/crowdsec/pkg/cache" "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" "github.com/crowdsecurity/crowdsec/pkg/parser" - "github.com/crowdsecurity/crowdsec/pkg/types" ) /*prometheus*/ @@ -61,7 +62,7 @@ var globalCsInfo = prometheus.NewGauge( prometheus.GaugeOpts{ Name: "cs_info", Help: "Information about Crowdsec.", - ConstLabels: prometheus.Labels{"version": cwversion.VersionStr()}, + ConstLabels: prometheus.Labels{"version": version.String()}, }, ) @@ -187,7 +188,7 @@ func servePrometheus(config *csconfig.PrometheusCfg, dbClient *database.Client, return } - defer types.CatchPanic("crowdsec/servePrometheus") + defer trace.CatchPanic("crowdsec/servePrometheus") http.Handle("/metrics", computeDynamicMetrics(promhttp.Handler(), dbClient)) <-apiReady diff --git a/cmd/crowdsec/output.go b/cmd/crowdsec/output.go index efeab7720..17cc99827 100644 --- a/cmd/crowdsec/output.go +++ b/cmd/crowdsec/output.go @@ -7,10 +7,11 @@ import ( "sync" "time" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + "github.com/crowdsecurity/crowdsec/pkg/apiclient" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/parser" @@ -88,7 +89,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky MachineID: apiConfig.Login, Password: password, Scenarios: scenarios, - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, PapiURL: papiURL, VersionPrefix: "v1", diff --git a/cmd/crowdsec/run_in_svc.go b/cmd/crowdsec/run_in_svc.go index d655afe88..a5ab996b5 100644 --- a/cmd/crowdsec/run_in_svc.go +++ b/cmd/crowdsec/run_in_svc.go @@ -10,10 +10,11 @@ import ( log "github.com/sirupsen/logrus" "github.com/sirupsen/logrus/hooks/writer" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/crowdsecurity/crowdsec/pkg/database" - "github.com/crowdsecurity/crowdsec/pkg/types" ) func StartRunSvc() error { @@ -22,7 +23,7 @@ func StartRunSvc() error { err error ) - defer types.CatchPanic("crowdsec/StartRunSvc") + defer trace.CatchPanic("crowdsec/StartRunSvc") // Set a default logger with level=fatal on stderr, // in addition to the one we configure afterwards @@ -38,7 +39,7 @@ func StartRunSvc() error { return err } - log.Infof("Crowdsec %s", cwversion.VersionStr()) + log.Infof("Crowdsec %s", version.String()) apiReady := make(chan bool, 1) agentReady := make(chan bool, 1) diff --git a/cmd/crowdsec/run_in_svc_windows.go b/cmd/crowdsec/run_in_svc_windows.go index 54287f770..c51d24147 100644 --- a/cmd/crowdsec/run_in_svc_windows.go +++ b/cmd/crowdsec/run_in_svc_windows.go @@ -7,17 +7,18 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/sys/windows/svc" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/crowdsecurity/crowdsec/pkg/database" - "github.com/crowdsecurity/crowdsec/pkg/types" ) func StartRunSvc() error { const svcName = "CrowdSec" const svcDescription = "Crowdsec IPS/IDS" - defer types.CatchPanic("crowdsec/StartRunSvc") + defer trace.CatchPanic("crowdsec/StartRunSvc") isRunninginService, err := svc.IsWindowsService() if err != nil { @@ -66,7 +67,7 @@ func WindowsRun() error { return err } // Configure logging - log.Infof("Crowdsec %s", cwversion.VersionStr()) + log.Infof("Crowdsec %s", version.String()) apiReady := make(chan bool, 1) agentReady := make(chan bool, 1) diff --git a/cmd/crowdsec/serve.go b/cmd/crowdsec/serve.go index 464af6327..5e2e8b720 100644 --- a/cmd/crowdsec/serve.go +++ b/cmd/crowdsec/serve.go @@ -11,6 +11,8 @@ import ( log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" @@ -226,7 +228,7 @@ func HandleSignals(cConfig *csconfig.Config) error { exitChan := make(chan error) go func() { - defer types.CatchPanic("crowdsec/HandleSignals") + defer trace.CatchPanic("crowdsec/HandleSignals") Loop: for { s := <-signalChan diff --git a/debian/rules b/debian/rules index 9f9258a2f..6683e5443 100755 --- a/debian/rules +++ b/debian/rules @@ -4,12 +4,6 @@ export DEB_VERSION=$(shell dpkg-parsechangelog | egrep '^Version:' | cut -f 2 -d export BUILD_VERSION=v${DEB_VERSION}-debian-pragmatic export GO111MODULE=on -# LD_OPTS=-ldflags "-s -w -X github.com/crowdsecurity/crowdsec/pkg/cwversion.Version=$(BUILD_VERSION) \ -# -X github.com/crowdsecurity/crowdsec/pkg/cwversion.BuildDate=$(BUILD_TIMESTAMP) \ -# -X github.com/crowdsecurity/crowdsec/pkg/cwversion.Codename=$(BUILD_CODENAME) \ -# -X github.com/crowdsecurity/crowdsec/pkg/cwversion.Tag=$(BUILD_TAG) \ -# -X github.com/crowdsecurity/crowdsec/pkg/cwversion.GoVersion=$(BUILD_GOVERSION)" - %: dh $@ diff --git a/go.mod b/go.mod index 882a20ca6..1ab6b74c6 100644 --- a/go.mod +++ b/go.mod @@ -48,9 +48,9 @@ require ( github.com/prometheus/client_model v0.3.0 github.com/prometheus/prom2json v1.3.0 github.com/r3labs/diff/v2 v2.14.1 - github.com/sirupsen/logrus v1.9.0 + github.com/sirupsen/logrus v1.9.2 github.com/spf13/cobra v1.7.0 - github.com/stretchr/testify v1.8.2 + github.com/stretchr/testify v1.8.3 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 google.golang.org/grpc v1.47.0 @@ -101,6 +101,7 @@ require ( github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect + github.com/crowdsecurity/go-cs-lib v0.0.0-20230522120244-fa545c12e7ee // indirect github.com/docker/distribution v2.8.2+incompatible // indirect github.com/docker/go-units v0.4.0 // indirect github.com/gin-contrib/sse v0.1.0 // indirect diff --git a/go.sum b/go.sum index ca8f2f5cf..e7fc2bf43 100644 --- a/go.sum +++ b/go.sum @@ -172,6 +172,8 @@ github.com/creack/pty v1.1.11 h1:07n33Z8lZxZ2qwegKbObQohDhXDQxiMMz1NOUGYlesw= github.com/creack/pty v1.1.11/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU= github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26/go.mod h1:zpv7r+7KXwgVUZnUNjyP22zc/D7LKjyoY02weH2RBbk= +github.com/crowdsecurity/go-cs-lib v0.0.0-20230522120244-fa545c12e7ee h1:YD/SYJ0otjlyAxuLRzJNXTQyXcpccefTvaHextBT9mQ= +github.com/crowdsecurity/go-cs-lib v0.0.0-20230522120244-fa545c12e7ee/go.mod h1:9JJLSpGj1ZXnROV3xAcJvS/HTaUvuA8K3gGOpO4tfVc= github.com/crowdsecurity/grokky v0.2.1 h1:t4VYnDlAd0RjDM2SlILalbwfCrQxtJSMGdQOR0zwkE4= github.com/crowdsecurity/grokky v0.2.1/go.mod h1:33usDIYzGDsgX1kHAThCbseso6JuWNJXOzRQDGXHtWM= github.com/crowdsecurity/machineid v1.0.2 h1:wpkpsUghJF8Khtmn/tg6GxgdhLA1Xflerh5lirI+bdc= @@ -841,8 +843,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.2 h1:oxx1eChJGI6Uks2ZC4W1zpLlVgqB8ner4EuQwV4Ik1Y= +github.com/sirupsen/logrus v1.9.2/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -881,8 +883,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.2 h1:+h33VjcLVPDHtOdpUCuF+7gSuG3yGIftsP1YvFihtJ8= -github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tetratelabs/wazero v1.0.0-rc.2 h1:OA3UUynnoqxrjCQ94mpAtdO4/oMxFQVNL2BXDMOc66Q= github.com/tetratelabs/wazero v1.0.0-rc.2/go.mod h1:wYx2gNRg8/WihJfSDxA1TIL8H+GkfLYm+bIfbblu9VQ= diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index bf22a3fb3..a26f4381b 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -15,6 +15,8 @@ import ( tomb "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" cloudwatchacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/cloudwatch" dockeracquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/docker" @@ -241,7 +243,7 @@ func GetMetrics(sources []DataSource, aggregated bool) error { } func transform(transformChan chan types.Event, output chan types.Event, AcquisTomb *tomb.Tomb, transformRuntime *vm.Program, logger *log.Entry) { - defer types.CatchPanic("crowdsec/acquis") + defer trace.CatchPanic("crowdsec/acquis") logger.Infof("transformer started") for { select { @@ -296,7 +298,7 @@ func StartAcquisition(sources []DataSource, output chan types.Event, AcquisTomb log.Debugf("starting one source %d/%d ->> %T", i, len(sources), subsrc) AcquisTomb.Go(func() error { - defer types.CatchPanic("crowdsec/acquis") + defer trace.CatchPanic("crowdsec/acquis") var err error outChan := output diff --git a/pkg/acquisition/modules/file/file.go b/pkg/acquisition/modules/file/file.go index dde29f245..c24b17332 100644 --- a/pkg/acquisition/modules/file/file.go +++ b/pkg/acquisition/modules/file/file.go @@ -14,6 +14,8 @@ import ( "strings" "time" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/fsnotify/fsnotify" "github.com/nxadm/tail" "github.com/pkg/errors" @@ -336,7 +338,7 @@ func (f *FileSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) er } f.tails[file] = true t.Go(func() error { - defer types.CatchPanic("crowdsec/acquis/file/live/fsnotify") + defer trace.CatchPanic("crowdsec/acquis/file/live/fsnotify") return f.tailFile(out, t, tail) }) } @@ -419,7 +421,7 @@ func (f *FileSource) monitorNewFiles(out chan types.Event, t *tomb.Tomb) error { } f.tails[event.Name] = true t.Go(func() error { - defer types.CatchPanic("crowdsec/acquis/tailfile") + defer trace.CatchPanic("crowdsec/acquis/tailfile") return f.tailFile(out, t, tail) }) } diff --git a/pkg/acquisition/modules/journalctl/journalctl.go b/pkg/acquisition/modules/journalctl/journalctl.go index 9858381e6..7882cb7c2 100644 --- a/pkg/acquisition/modules/journalctl/journalctl.go +++ b/pkg/acquisition/modules/journalctl/journalctl.go @@ -15,6 +15,8 @@ import ( "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -257,7 +259,7 @@ func (j *JournalCtlSource) GetName() string { } func (j *JournalCtlSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb) error { - defer types.CatchPanic("crowdsec/acquis/journalctl/oneshot") + defer trace.CatchPanic("crowdsec/acquis/journalctl/oneshot") err := j.runJournalCtl(out, t) j.logger.Debug("Oneshot journalctl acquisition is done") return err @@ -266,7 +268,7 @@ func (j *JournalCtlSource) OneShotAcquisition(out chan types.Event, t *tomb.Tomb func (j *JournalCtlSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error { t.Go(func() error { - defer types.CatchPanic("crowdsec/acquis/journalctl/streaming") + defer trace.CatchPanic("crowdsec/acquis/journalctl/streaming") return j.runJournalCtl(out, t) }) return nil diff --git a/pkg/acquisition/modules/kafka/kafka.go b/pkg/acquisition/modules/kafka/kafka.go index af9251163..6fb5ab509 100644 --- a/pkg/acquisition/modules/kafka/kafka.go +++ b/pkg/acquisition/modules/kafka/kafka.go @@ -17,6 +17,8 @@ import ( "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -190,7 +192,7 @@ func (k *KafkaSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) e k.logger.Infof("start reader on topic '%s'", k.Config.Topic) t.Go(func() error { - defer types.CatchPanic("crowdsec/acquis/kafka/live") + defer trace.CatchPanic("crowdsec/acquis/kafka/live") return k.RunReader(out, t) }) diff --git a/pkg/acquisition/modules/kinesis/kinesis.go b/pkg/acquisition/modules/kinesis/kinesis.go index 7700975b9..60cdc3751 100644 --- a/pkg/acquisition/modules/kinesis/kinesis.go +++ b/pkg/acquisition/modules/kinesis/kinesis.go @@ -19,6 +19,8 @@ import ( "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -490,7 +492,7 @@ func (k *KinesisSource) ReadFromStream(out chan types.Event, t *tomb.Tomb) error for _, shard := range shards.Shards { shardId := *shard.ShardId k.shardReaderTomb.Go(func() error { - defer types.CatchPanic("crowdsec/acquis/kinesis/streaming/shard") + defer trace.CatchPanic("crowdsec/acquis/kinesis/streaming/shard") return k.ReadFromShard(out, shardId) }) } @@ -514,7 +516,7 @@ func (k *KinesisSource) ReadFromStream(out chan types.Event, t *tomb.Tomb) error func (k *KinesisSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error { t.Go(func() error { - defer types.CatchPanic("crowdsec/acquis/kinesis/streaming") + defer trace.CatchPanic("crowdsec/acquis/kinesis/streaming") if k.Config.UseEnhancedFanOut { return k.EnhancedRead(out, t) } else { diff --git a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go index 15a1a1033..f65a0aa57 100644 --- a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go +++ b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go @@ -8,6 +8,8 @@ import ( "net/http" "strings" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/pkg/errors" @@ -133,7 +135,7 @@ func (ka *KubernetesAuditSource) OneShotAcquisition(out chan types.Event, t *tom func (ka *KubernetesAuditSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error { ka.outChan = out t.Go(func() error { - defer types.CatchPanic("crowdsec/acquis/k8s-audit/live") + defer trace.CatchPanic("crowdsec/acquis/k8s-audit/live") ka.logger.Infof("Starting k8s-audit server on %s:%d%s", ka.config.ListenAddr, ka.config.ListenPort, ka.config.WebhookPath) t.Go(func() error { err := ka.server.ListenAndServe() diff --git a/pkg/acquisition/modules/syslog/syslog.go b/pkg/acquisition/modules/syslog/syslog.go index cc93c3e2b..948f3d005 100644 --- a/pkg/acquisition/modules/syslog/syslog.go +++ b/pkg/acquisition/modules/syslog/syslog.go @@ -12,6 +12,8 @@ import ( "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog/internal/parser/rfc3164" "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog/internal/parser/rfc5424" @@ -142,7 +144,7 @@ func (s *SyslogSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) } s.serverTomb = s.server.StartServer() t.Go(func() error { - defer types.CatchPanic("crowdsec/acquis/syslog/live") + defer trace.CatchPanic("crowdsec/acquis/syslog/live") return s.handleSyslogMsg(out, t, c) }) return nil diff --git a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go index ba5829d8a..3a78a1932 100644 --- a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go +++ b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go @@ -17,6 +17,8 @@ import ( "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -321,7 +323,7 @@ func (w *WinEventLogSource) CanRun() error { func (w *WinEventLogSource) StreamingAcquisition(out chan types.Event, t *tomb.Tomb) error { t.Go(func() error { - defer types.CatchPanic("crowdsec/acquis/wineventlog/streaming") + defer trace.CatchPanic("crowdsec/acquis/wineventlog/streaming") return w.getEvents(out, t) }) return nil diff --git a/pkg/apiclient/alerts_service_test.go b/pkg/apiclient/alerts_service_test.go index 4270851d5..f4ec8cabe 100644 --- a/pkg/apiclient/alerts_service_test.go +++ b/pkg/apiclient/alerts_service_test.go @@ -8,7 +8,8 @@ import ( "reflect" "testing" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + "github.com/crowdsecurity/crowdsec/pkg/models" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -31,7 +32,7 @@ func TestAlertsListAsMachine(t *testing.T) { client, err := NewClient(&Config{ MachineID: "test_login", Password: "test_password", - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, VersionPrefix: "v1", }) @@ -233,7 +234,7 @@ func TestAlertsGetAsMachine(t *testing.T) { client, err := NewClient(&Config{ MachineID: "test_login", Password: "test_password", - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, VersionPrefix: "v1", }) @@ -423,7 +424,7 @@ func TestAlertsCreateAsMachine(t *testing.T) { client, err := NewClient(&Config{ MachineID: "test_login", Password: "test_password", - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, VersionPrefix: "v1", }) @@ -467,7 +468,7 @@ func TestAlertsDeleteAsMachine(t *testing.T) { client, err := NewClient(&Config{ MachineID: "test_login", Password: "test_password", - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, VersionPrefix: "v1", }) diff --git a/pkg/apiclient/auth_service_test.go b/pkg/apiclient/auth_service_test.go index 1e3e83e04..6236cf041 100644 --- a/pkg/apiclient/auth_service_test.go +++ b/pkg/apiclient/auth_service_test.go @@ -10,7 +10,8 @@ import ( "net/url" "testing" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + "github.com/crowdsecurity/crowdsec/pkg/models" log "github.com/sirupsen/logrus" "github.com/stretchr/testify/assert" @@ -86,7 +87,7 @@ func TestWatcherRegister(t *testing.T) { clientconfig := Config{ MachineID: "test_login", Password: "test_password", - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, VersionPrefix: "v1", } @@ -128,7 +129,7 @@ func TestWatcherAuth(t *testing.T) { clientConfig := &Config{ MachineID: "test_login", Password: "test_password", - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, VersionPrefix: "v1", Scenarios: []string{"crowdsecurity/test"}, @@ -216,7 +217,7 @@ func TestWatcherUnregister(t *testing.T) { mycfg := &Config{ MachineID: "test_login", Password: "test_password", - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, VersionPrefix: "v1", Scenarios: []string{"crowdsecurity/test"}, @@ -270,7 +271,7 @@ func TestWatcherEnroll(t *testing.T) { mycfg := &Config{ MachineID: "test_login", Password: "test_password", - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, VersionPrefix: "v1", Scenarios: []string{"crowdsecurity/test"}, diff --git a/pkg/apiclient/client_http_test.go b/pkg/apiclient/client_http_test.go index f9ad57210..c50769041 100644 --- a/pkg/apiclient/client_http_test.go +++ b/pkg/apiclient/client_http_test.go @@ -8,7 +8,8 @@ import ( "testing" "time" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + "github.com/stretchr/testify/assert" ) @@ -23,7 +24,7 @@ func TestNewRequestInvalid(t *testing.T) { client, err := NewClient(&Config{ MachineID: "test_login", Password: "test_password", - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, VersionPrefix: "v1", }) @@ -56,7 +57,7 @@ func TestNewRequestTimeout(t *testing.T) { client, err := NewClient(&Config{ MachineID: "test_login", Password: "test_password", - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, VersionPrefix: "v1", }) diff --git a/pkg/apiclient/client_test.go b/pkg/apiclient/client_test.go index 3b55dce07..ef52a60ab 100644 --- a/pkg/apiclient/client_test.go +++ b/pkg/apiclient/client_test.go @@ -11,8 +11,9 @@ import ( "github.com/stretchr/testify/assert" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" log "github.com/sirupsen/logrus" + + "github.com/crowdsecurity/go-cs-lib/pkg/version" ) /*this is a ripoff of google/go-github approach : @@ -55,7 +56,7 @@ func TestNewClientOk(t *testing.T) { client, err := NewClient(&Config{ MachineID: "test_login", Password: "test_password", - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, VersionPrefix: "v1", }) @@ -92,7 +93,7 @@ func TestNewClientKo(t *testing.T) { client, err := NewClient(&Config{ MachineID: "test_login", Password: "test_password", - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, VersionPrefix: "v1", }) @@ -143,7 +144,7 @@ func TestNewClientRegisterKO(t *testing.T) { _, err = RegisterClient(&Config{ MachineID: "test_login", Password: "test_password", - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, VersionPrefix: "v1", }, &http.Client{}) @@ -173,7 +174,7 @@ func TestNewClientRegisterOK(t *testing.T) { client, err := RegisterClient(&Config{ MachineID: "test_login", Password: "test_password", - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, VersionPrefix: "v1", }, &http.Client{}) @@ -201,7 +202,7 @@ func TestNewClientBadAnswer(t *testing.T) { _, err = RegisterClient(&Config{ MachineID: "test_login", Password: "test_password", - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, VersionPrefix: "v1", }, &http.Client{}) diff --git a/pkg/apiclient/decisions_service_test.go b/pkg/apiclient/decisions_service_test.go index 35d819d5a..7af2bf81f 100644 --- a/pkg/apiclient/decisions_service_test.go +++ b/pkg/apiclient/decisions_service_test.go @@ -8,7 +8,8 @@ import ( "reflect" "testing" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/modelscapi" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -495,7 +496,7 @@ func TestDeleteDecisions(t *testing.T) { client, err := NewClient(&Config{ MachineID: "test_login", Password: "test_password", - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, VersionPrefix: "v1", }) diff --git a/pkg/apiclient/heartbeat.go b/pkg/apiclient/heartbeat.go index dc7c256e3..497ccb7eb 100644 --- a/pkg/apiclient/heartbeat.go +++ b/pkg/apiclient/heartbeat.go @@ -6,9 +6,10 @@ import ( "net/http" "time" - "github.com/crowdsecurity/crowdsec/pkg/types" log "github.com/sirupsen/logrus" tomb "gopkg.in/tomb.v2" + + "github.com/crowdsecurity/go-cs-lib/pkg/trace" ) type HeartBeatService service @@ -32,7 +33,7 @@ func (h *HeartBeatService) Ping(ctx context.Context) (bool, *Response, error) { func (h *HeartBeatService) StartHeartBeat(ctx context.Context, t *tomb.Tomb) { t.Go(func() error { - defer types.CatchPanic("crowdsec/apiClient/heartbeat") + defer trace.CatchPanic("crowdsec/apiClient/heartbeat") hbTimer := time.NewTicker(1 * time.Minute) for { select { diff --git a/pkg/apiserver/apic.go b/pkg/apiserver/apic.go index 5e8c714a1..11104221c 100644 --- a/pkg/apiserver/apic.go +++ b/pkg/apiserver/apic.go @@ -17,9 +17,11 @@ import ( log "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + "github.com/crowdsecurity/crowdsec/pkg/apiclient" "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/database/ent" "github.com/crowdsecurity/crowdsec/pkg/database/ent/alert" @@ -193,7 +195,7 @@ func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, con ret.apiClient, err = apiclient.NewClient(&apiclient.Config{ MachineID: config.Credentials.Login, Password: password, - UserAgent: fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + UserAgent: fmt.Sprintf("crowdsec/%s", version.String()), URL: apiURL, PapiURL: papiURL, VersionPrefix: "v3", @@ -231,7 +233,7 @@ func NewAPIC(config *csconfig.OnlineApiClientCfg, dbClient *database.Client, con // keep track of all alerts in cache and push it to CAPI every PushInterval. func (a *apic) Push() error { - defer types.CatchPanic("lapi/pushToAPIC") + defer trace.CatchPanic("lapi/pushToAPIC") var cache models.AddSignalsRequest ticker := time.NewTicker(a.pushIntervalFirst) @@ -778,7 +780,7 @@ func setAlertScenario(add_counters map[string]map[string]int, delete_counters ma } func (a *apic) Pull() error { - defer types.CatchPanic("lapi/pullFromAPIC") + defer trace.CatchPanic("lapi/pullFromAPIC") toldOnce := false for { @@ -820,7 +822,7 @@ func (a *apic) Pull() error { func (a *apic) GetMetrics() (*models.Metrics, error) { metric := &models.Metrics{ - ApilVersion: types.StrPtr(cwversion.VersionStr()), + ApilVersion: types.StrPtr(version.String()), Machines: make([]*models.MetricsAgentInfo, 0), Bouncers: make([]*models.MetricsBouncerInfo, 0), } @@ -861,7 +863,7 @@ func (a *apic) GetMetrics() (*models.Metrics, error) { } func (a *apic) SendMetrics(stop chan (bool)) { - defer types.CatchPanic("lapi/metricsToAPIC") + defer trace.CatchPanic("lapi/metricsToAPIC") ticker := time.NewTicker(a.metricsIntervalFirst) diff --git a/pkg/apiserver/apic_test.go b/pkg/apiserver/apic_test.go index 965ef0378..dd89e2be5 100644 --- a/pkg/apiserver/apic_test.go +++ b/pkg/apiserver/apic_test.go @@ -20,10 +20,11 @@ import ( "github.com/stretchr/testify/require" "gopkg.in/tomb.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + "github.com/crowdsecurity/crowdsec/pkg/apiclient" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cstest" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/database/ent/decision" "github.com/crowdsecurity/crowdsec/pkg/database/ent/machine" @@ -293,7 +294,7 @@ func TestAPICGetMetrics(t *testing.T) { machineIDs: []string{}, bouncers: []string{}, expectedMetric: &models.Metrics{ - ApilVersion: types.StrPtr(cwversion.VersionStr()), + ApilVersion: types.StrPtr(version.String()), Bouncers: []*models.MetricsBouncerInfo{}, Machines: []*models.MetricsAgentInfo{}, }, @@ -303,7 +304,7 @@ func TestAPICGetMetrics(t *testing.T) { machineIDs: []string{"a", "b", "c"}, bouncers: []string{"1", "2", "3"}, expectedMetric: &models.Metrics{ - ApilVersion: types.StrPtr(cwversion.VersionStr()), + ApilVersion: types.StrPtr(version.String()), Bouncers: []*models.MetricsBouncerInfo{ { CustomName: "1", @@ -661,7 +662,7 @@ func TestAPICWhitelists(t *testing.T) { apic, err := apiclient.NewDefaultClient( url, "/api", - fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + fmt.Sprintf("crowdsec/%s", version.String()), nil, ) require.NoError(t, err) @@ -792,7 +793,7 @@ func TestAPICPullTop(t *testing.T) { apic, err := apiclient.NewDefaultClient( url, "/api", - fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + fmt.Sprintf("crowdsec/%s", version.String()), nil, ) require.NoError(t, err) @@ -874,7 +875,7 @@ func TestAPICPullTopBLCacheFirstCall(t *testing.T) { apic, err := apiclient.NewDefaultClient( url, "/api", - fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + fmt.Sprintf("crowdsec/%s", version.String()), nil, ) require.NoError(t, err) @@ -961,7 +962,7 @@ func TestAPICPullTopBLCacheForceCall(t *testing.T) { apic, err := apiclient.NewDefaultClient( url, "/api", - fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + fmt.Sprintf("crowdsec/%s", version.String()), nil, ) require.NoError(t, err) @@ -1036,7 +1037,7 @@ func TestAPICPush(t *testing.T) { apic, err := apiclient.NewDefaultClient( url, "/api", - fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + fmt.Sprintf("crowdsec/%s", version.String()), nil, ) require.NoError(t, err) @@ -1111,7 +1112,7 @@ func TestAPICSendMetrics(t *testing.T) { apiClient, err := apiclient.NewDefaultClient( url, "/api", - fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + fmt.Sprintf("crowdsec/%s", version.String()), nil, ) require.NoError(t, err) @@ -1179,7 +1180,7 @@ func TestAPICPull(t *testing.T) { apic, err := apiclient.NewDefaultClient( url, "/api", - fmt.Sprintf("crowdsec/%s", cwversion.VersionStr()), + fmt.Sprintf("crowdsec/%s", version.String()), nil, ) require.NoError(t, err) diff --git a/pkg/apiserver/apiserver.go b/pkg/apiserver/apiserver.go index 278a776be..2c69f48ed 100644 --- a/pkg/apiserver/apiserver.go +++ b/pkg/apiserver/apiserver.go @@ -12,6 +12,8 @@ import ( "strings" "time" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/crowdsec/pkg/apiclient" "github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers" v1 "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1" @@ -87,7 +89,7 @@ func CustomRecoveryWithWriter() gin.HandlerFunc { log.Warningf("client %s disconnected : %s", c.ClientIP(), err) c.Abort() } else { - filename := types.WriteStackTrace(err) + filename := trace.WriteStackTrace(err) log.Warningf("client %s error : %s", c.ClientIP(), err) log.Warningf("stacktrace written to %s, please join to your issue", filename) c.AbortWithStatus(http.StatusInternalServerError) @@ -325,7 +327,7 @@ func (s *APIServer) GetTLSConfig() (*tls.Config, error) { } func (s *APIServer) Run(apiReady chan bool) error { - defer types.CatchPanic("lapi/runServer") + defer trace.CatchPanic("lapi/runServer") tlsCfg, err := s.GetTLSConfig() if err != nil { return errors.Wrap(err, "while creating TLS config") diff --git a/pkg/apiserver/apiserver_test.go b/pkg/apiserver/apiserver_test.go index 3eaaab651..136b0d919 100644 --- a/pkg/apiserver/apiserver_test.go +++ b/pkg/apiserver/apiserver_test.go @@ -11,8 +11,9 @@ import ( "testing" "time" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + middlewares "github.com/crowdsecurity/crowdsec/pkg/apiserver/middlewares/v1" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/go-openapi/strfmt" @@ -33,7 +34,7 @@ var MachineTest = models.WatcherAuthRequest{ Password: &testPassword, } -var UserAgent = fmt.Sprintf("crowdsec-test/%s", cwversion.Version) +var UserAgent = fmt.Sprintf("crowdsec-test/%s", version.Version) var emptyBody = strings.NewReader("") func LoadTestConfig() csconfig.Config { diff --git a/pkg/apiserver/papi.go b/pkg/apiserver/papi.go index 33028b13c..e59160aee 100644 --- a/pkg/apiserver/papi.go +++ b/pkg/apiserver/papi.go @@ -8,6 +8,8 @@ import ( "sync" "time" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/crowdsec/pkg/apiclient" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/database" @@ -217,8 +219,7 @@ func (p *Papi) PullOnce(since time.Time, sync bool) error { // PullPAPI is the long polling client for real-time decisions from PAPI func (p *Papi) Pull() error { - - defer types.CatchPanic("lapi/PullPAPI") + defer trace.CatchPanic("lapi/PullPAPI") p.Logger.Infof("Starting Polling API Pull") lastTimestamp := time.Time{} @@ -270,7 +271,7 @@ func (p *Papi) Pull() error { } func (p *Papi) SyncDecisions() error { - defer types.CatchPanic("lapi/syncDecisionsToCAPI") + defer trace.CatchPanic("lapi/syncDecisionsToCAPI") var cache models.DecisionsDeleteRequest ticker := time.NewTicker(p.SyncInterval) diff --git a/pkg/csconfig/api.go b/pkg/csconfig/api.go index bcc3c5b93..0c0605b80 100644 --- a/pkg/csconfig/api.go +++ b/pkg/csconfig/api.go @@ -13,9 +13,10 @@ import ( log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/yamlpatch" + "github.com/crowdsecurity/crowdsec/pkg/apiclient" "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/crowdsecurity/crowdsec/pkg/yamlpatch" ) type APICfg struct { diff --git a/pkg/csconfig/config.go b/pkg/csconfig/config.go index b226dde5b..b94a2e975 100644 --- a/pkg/csconfig/config.go +++ b/pkg/csconfig/config.go @@ -9,9 +9,10 @@ import ( log "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" - "github.com/crowdsecurity/crowdsec/pkg/csstring" + "github.com/crowdsecurity/go-cs-lib/pkg/yamlpatch" + "github.com/crowdsecurity/go-cs-lib/pkg/csstring" + "github.com/crowdsecurity/crowdsec/pkg/types" - "github.com/crowdsecurity/crowdsec/pkg/yamlpatch" ) // defaultConfigDir is the base path to all configuration files, to be overridden in the Makefile */ diff --git a/pkg/csconfig/profiles.go b/pkg/csconfig/profiles.go index 16a4e454d..41725bcf2 100644 --- a/pkg/csconfig/profiles.go +++ b/pkg/csconfig/profiles.go @@ -5,8 +5,9 @@ import ( "fmt" "io" + "github.com/crowdsecurity/go-cs-lib/pkg/yamlpatch" + "github.com/crowdsecurity/crowdsec/pkg/models" - "github.com/crowdsecurity/crowdsec/pkg/yamlpatch" "github.com/pkg/errors" "gopkg.in/yaml.v2" ) diff --git a/pkg/csconfig/simulation.go b/pkg/csconfig/simulation.go index 69c520c5c..f291a4e16 100644 --- a/pkg/csconfig/simulation.go +++ b/pkg/csconfig/simulation.go @@ -4,7 +4,7 @@ import ( "fmt" "path/filepath" - "github.com/crowdsecurity/crowdsec/pkg/yamlpatch" + "github.com/crowdsecurity/go-cs-lib/pkg/yamlpatch" "gopkg.in/yaml.v2" ) diff --git a/pkg/csplugin/broker.go b/pkg/csplugin/broker.go index 716368a92..fcde41831 100644 --- a/pkg/csplugin/broker.go +++ b/pkg/csplugin/broker.go @@ -19,8 +19,9 @@ import ( "gopkg.in/tomb.v2" "gopkg.in/yaml.v2" + "github.com/crowdsecurity/go-cs-lib/pkg/csstring" + "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/csstring" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/protobufs" "github.com/crowdsecurity/crowdsec/pkg/types" diff --git a/pkg/csstring/expand.go b/pkg/csstring/expand.go deleted file mode 100644 index 6f9037af0..000000000 --- a/pkg/csstring/expand.go +++ /dev/null @@ -1,75 +0,0 @@ -package csstring - -func seekClosingBracket(s string, i int) int { - for ; i < len(s); i++ { - if s[i] == '}' { - return i - } - } - - return -1 -} - -func seekEndVarname(s string, i int) int { - // envvar names are more strict but this is good enough - for ; i < len(s); i++ { - if (s[i] < 'a' || s[i] > 'z') && (s[i] < 'A' || s[i] > 'Z') && (s[i] < '0' || s[i] > '9') && s[i] != '_' { - break - } - } - - return i -} - -func replaceVarBracket(s string, i int, mapping func(string) (string, bool)) string { - j := seekClosingBracket(s, i+2) - if j < 0 { - return s - } - - if j < len(s) { - varName := s[i+2 : j] - if val, ok := mapping(varName); ok { - s = s[:i] + val + s[j+1:] - } - } - - return s -} - -func replaceVar(s string, i int, mapping func(string) (string, bool)) string { - if s[i+1] == '{' { - return replaceVarBracket(s, i, mapping) - } - - j := seekEndVarname(s, i+1) - if j < 0 { - return s - } - - if j > i+1 { - varName := s[i+1 : j] - if val, ok := mapping(varName); ok { - s = s[:i] + val + s[j:] - } - } - - return s -} - -// StrictExpand replaces ${var} or $var in the string according to the mapping -// function, like os.Expand. The difference is that the mapping function -// returns a boolean indicating whether the variable was found. -// If the variable was not found, the string is not modified. -// -// Whereas os.ExpandEnv uses os.Getenv, here we can use os.LookupEnv -// to distinguish between an empty variable and an undefined one. -func StrictExpand(s string, mapping func(string) (string, bool)) string { - for i := 0; i < len(s); i++ { - if s[i] == '$' { - s = replaceVar(s, i, mapping) - } - } - - return s -} diff --git a/pkg/csstring/expand_test.go b/pkg/csstring/expand_test.go deleted file mode 100644 index 72a281be9..000000000 --- a/pkg/csstring/expand_test.go +++ /dev/null @@ -1,98 +0,0 @@ -package csstring_test - -import ( - "fmt" - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/crowdsecurity/crowdsec/pkg/csstring" -) - -func TestStrictExpand(t *testing.T) { - t.Parallel() - - testenv := func(key string) (string, bool) { - switch key { - case "USER": - return "testuser", true - case "HOME": - return "/home/testuser", true - case "empty": - return "", true - default: - return "", false - } - } - - home, _ := testenv("HOME") - user, _ := testenv("USER") - - tests := []struct { - input string - expected string - }{ - { - input: "$HOME", - expected: home, - }, - { - input: "${USER}", - expected: user, - }, - { - input: "Hello, $USER!", - expected: fmt.Sprintf("Hello, %s!", user), - }, - { - input: "My home directory is ${HOME}", - expected: fmt.Sprintf("My home directory is %s", home), - }, - { - input: "This is a $SINGLE_VAR string with ${HOME}", - expected: fmt.Sprintf("This is a $SINGLE_VAR string with %s", home), - }, - { - input: "This is a $SINGLE_VAR string with $HOME", - expected: fmt.Sprintf("This is a $SINGLE_VAR string with %s", home), - }, - { - input: "This variable does not exist: $NON_EXISTENT_VAR", - expected: "This variable does not exist: $NON_EXISTENT_VAR", - }, - { - input: "This is a $MULTI_VAR string with ${HOME} and ${USER}", - expected: fmt.Sprintf("This is a $MULTI_VAR string with %s and %s", home, user), - }, - { - input: "This is a ${MULTI_VAR} string with $HOME and $USER", - expected: fmt.Sprintf("This is a ${MULTI_VAR} string with %s and %s", home, user), - }, - { - input: "This is a plain string with no variables", - expected: "This is a plain string with no variables", - }, - { - input: "$empty", - expected: "", - }, - { - input: "", - expected: "", - }, - { - input: "$USER:$empty:$HOME", - expected: fmt.Sprintf("%s::%s", user, home), - }, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.input, func(t *testing.T) { - t.Parallel() - - output := csstring.StrictExpand(tc.input, testenv) - assert.Equal(t, tc.expected, output) - }) - } -} diff --git a/pkg/cwversion/version.go b/pkg/cwversion/version.go index cb1f06896..f910159fa 100644 --- a/pkg/cwversion/version.go +++ b/pkg/cwversion/version.go @@ -8,27 +8,13 @@ import ( "runtime" "strings" - version "github.com/hashicorp/go-version" + goversion "github.com/hashicorp/go-version" + + "github.com/crowdsecurity/go-cs-lib/pkg/version" ) -/* - -Given a version number MAJOR.MINOR.PATCH, increment the: - - MAJOR version when you make incompatible API changes, - MINOR version when you add functionality in a backwards compatible manner, and - PATCH version when you make backwards compatible bug fixes. - -Additional labels for pre-release and build metadata are available as extensions to the MAJOR.MINOR.PATCH format. - -*/ - var ( - Version string // = "v0.0.0" Codename string // = "SoumSoum" - BuildDate string // = "0000-00-00_00:00:00" - Tag string // = "dev" - GoVersion = runtime.Version()[2:] // = "1.13" System = runtime.GOOS // = "linux" Constraint_parser = ">= 1.0, <= 2.0" Constraint_scenario = ">= 1.0, < 3.0" @@ -38,19 +24,19 @@ var ( func ShowStr() string { ret := "" - ret += fmt.Sprintf("version: %s-%s\n", Version, Tag) + ret += fmt.Sprintf("version: %s-%s\n", version.Version, version.Tag) ret += fmt.Sprintf("Codename: %s\n", Codename) - ret += fmt.Sprintf("BuildDate: %s\n", BuildDate) - ret += fmt.Sprintf("GoVersion: %s\n", GoVersion) + ret += fmt.Sprintf("BuildDate: %s\n", version.BuildDate) + ret += fmt.Sprintf("GoVersion: %s\n", version.GoVersion) ret += fmt.Sprintf("Platform: %s\n", System) return ret } func Show() { - log.Printf("version: %s-%s", Version, Tag) + log.Printf("version: %s-%s", version.Version, version.Tag) log.Printf("Codename: %s", Codename) - log.Printf("BuildDate: %s", BuildDate) - log.Printf("GoVersion: %s", GoVersion) + log.Printf("BuildDate: %s", version.BuildDate) + log.Printf("GoVersion: %s", version.GoVersion) log.Printf("Platform: %s\n", System) log.Printf("Constraint_parser: %s", Constraint_parser) log.Printf("Constraint_scenario: %s", Constraint_scenario) @@ -59,21 +45,21 @@ func Show() { } func VersionStr() string { - return fmt.Sprintf("%s-%s-%s", Version, System, Tag) + return fmt.Sprintf("%s-%s-%s", version.Version, System, version.Tag) } func VersionStrip() string { - version := strings.Split(Version, "~") + version := strings.Split(version.Version, "~") version = strings.Split(version[0], "-") return version[0] } -func Statisfies(strvers string, constraint string) (bool, error) { - vers, err := version.NewVersion(strvers) +func Satisfies(strvers string, constraint string) (bool, error) { + vers, err := goversion.NewVersion(strvers) if err != nil { return false, fmt.Errorf("failed to parse '%s' : %v", strvers, err) } - constraints, err := version.NewConstraint(constraint) + constraints, err := goversion.NewConstraint(constraint) if err != nil { return false, fmt.Errorf("failed to parse constraint '%s'", constraint) } diff --git a/pkg/leakybucket/bucket.go b/pkg/leakybucket/bucket.go index 8ce03e41a..004d5b9d8 100644 --- a/pkg/leakybucket/bucket.go +++ b/pkg/leakybucket/bucket.go @@ -6,6 +6,8 @@ import ( "sync/atomic" "time" + "github.com/crowdsecurity/go-cs-lib/pkg/trace" + "github.com/crowdsecurity/crowdsec/pkg/time/rate" "github.com/crowdsecurity/crowdsec/pkg/types" "github.com/davecgh/go-spew/spew" @@ -202,7 +204,7 @@ func LeakRoutine(leaky *Leaky) error { firstEvent = true ) - defer types.CatchPanic(fmt.Sprintf("crowdsec/LeakRoutine/%s", leaky.Name)) + defer trace.CatchPanic(fmt.Sprintf("crowdsec/LeakRoutine/%s", leaky.Name)) BucketsCurrentCount.With(prometheus.Labels{"name": leaky.Name}).Inc() defer BucketsCurrentCount.With(prometheus.Labels{"name": leaky.Name}).Dec() diff --git a/pkg/leakybucket/manager_load.go b/pkg/leakybucket/manager_load.go index 3789c749d..1e212f815 100644 --- a/pkg/leakybucket/manager_load.go +++ b/pkg/leakybucket/manager_load.go @@ -200,7 +200,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb. log.Tracef("no version in %s : %s, assuming '1.0'", bucketFactory.Name, f) bucketFactory.FormatVersion = "1.0" } - ok, err := cwversion.Statisfies(bucketFactory.FormatVersion, cwversion.Constraint_scenario) + ok, err := cwversion.Satisfies(bucketFactory.FormatVersion, cwversion.Constraint_scenario) if err != nil { log.Fatalf("Failed to check version : %s", err) } diff --git a/pkg/metabase/api.go b/pkg/metabase/api.go index 4e33b9349..99cbf9ec7 100644 --- a/pkg/metabase/api.go +++ b/pkg/metabase/api.go @@ -6,7 +6,8 @@ import ( "net/http" "time" - "github.com/crowdsecurity/crowdsec/pkg/cwversion" + "github.com/crowdsecurity/go-cs-lib/pkg/version" + "github.com/dghubble/sling" log "github.com/sirupsen/logrus" ) @@ -37,7 +38,7 @@ var ( func NewAPIClient(url string) (*APIClient, error) { httpClient := &http.Client{Timeout: 20 * time.Second} return &APIClient{ - CTX: sling.New().Client(httpClient).Base(url).Set("User-Agent", fmt.Sprintf("crowdsec/%s", cwversion.VersionStr())), + CTX: sling.New().Client(httpClient).Base(url).Set("User-Agent", fmt.Sprintf("crowdsec/%s", version.String())), Client: httpClient, }, nil } diff --git a/pkg/parser/stage.go b/pkg/parser/stage.go index 4ffa50f2f..3bdaed1f0 100644 --- a/pkg/parser/stage.go +++ b/pkg/parser/stage.go @@ -83,7 +83,7 @@ func LoadStages(stageFiles []Stagefile, pctx *UnixParserCtx, ectx EnricherCtx) ( log.Tracef("no version in %s, assuming '1.0'", node.Name) node.FormatVersion = "1.0" } - ok, err := cwversion.Statisfies(node.FormatVersion, cwversion.Constraint_parser) + ok, err := cwversion.Satisfies(node.FormatVersion, cwversion.Constraint_parser) if err != nil { log.Fatalf("Failed to check version : %s", err) } diff --git a/pkg/types/utils.go b/pkg/types/utils.go index 342fa6372..3c1d22b16 100644 --- a/pkg/types/utils.go +++ b/pkg/types/utils.go @@ -9,15 +9,12 @@ import ( "os" "path/filepath" "regexp" - "runtime/debug" "strconv" "strings" "time" log "github.com/sirupsen/logrus" "gopkg.in/natefinch/lumberjack.v2" - - "github.com/crowdsecurity/crowdsec/pkg/cwversion" ) var logFormatter log.Formatter @@ -25,7 +22,6 @@ var LogOutput *lumberjack.Logger //io.Writer var logLevel log.Level func SetDefaultLoggerConfig(cfgMode string, cfgFolder string, cfgLevel log.Level, maxSize int, maxFiles int, maxAge int, compress *bool, forceColors bool) error { - /*Configure logs*/ if cfgMode == "file" { _maxsize := 500 @@ -87,7 +83,6 @@ func ConfigureLogger(clog *log.Logger) error { } func Clone(a, b interface{}) error { - buff := new(bytes.Buffer) enc := gob.NewEncoder(buff) dec := gob.NewDecoder(buff) @@ -100,40 +95,6 @@ func Clone(a, b interface{}) error { return nil } -func WriteStackTrace(iErr interface{}) string { - tmpfile, err := os.CreateTemp("", "crowdsec-crash.*.txt") - if err != nil { - log.Fatal(err) - } - if _, err := tmpfile.Write([]byte(fmt.Sprintf("error : %+v\n", iErr))); err != nil { - tmpfile.Close() - log.Fatal(err) - } - if _, err := tmpfile.Write([]byte(cwversion.ShowStr())); err != nil { - tmpfile.Close() - log.Fatal(err) - } - if _, err := tmpfile.Write(debug.Stack()); err != nil { - tmpfile.Close() - log.Fatal(err) - } - if err := tmpfile.Close(); err != nil { - log.Fatal(err) - } - return tmpfile.Name() -} - -//CatchPanic is a util func that we should call from all go-routines to ensure proper stacktrace handling -func CatchPanic(component string) { - if r := recover(); r != nil { - log.Errorf("crowdsec - goroutine %s crashed : %s", component, r) - log.Errorf("please report this error to https://github.com/crowdsecurity/crowdsec/") - filename := WriteStackTrace(r) - log.Errorf("stacktrace/report is written to %s : please join it to your issue", filename) - log.Fatalf("crowdsec stopped") - } -} - func ParseDuration(d string) (time.Duration, error) { durationStr := d if strings.HasSuffix(d, "d") { @@ -181,7 +142,6 @@ func copyFileContents(src, dst string) (err error) { /*copy the file, ioutile doesn't offer the feature*/ func CopyFile(sourceSymLink, destinationFile string) (err error) { - sourceFile, err := filepath.EvalSymlinks(sourceSymLink) if err != nil { log.Infof("Not a symlink : %s", err) diff --git a/pkg/yamlpatch/merge.go b/pkg/yamlpatch/merge.go deleted file mode 100644 index 8a61b6470..000000000 --- a/pkg/yamlpatch/merge.go +++ /dev/null @@ -1,168 +0,0 @@ -// -// from https://github.com/uber-go/config/tree/master/internal/merge -// -// Copyright (c) 2019 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package yamlpatch - -import ( - "bytes" - "fmt" - "io" - - "github.com/pkg/errors" - - yaml "gopkg.in/yaml.v2" -) - -type ( - // YAML has three fundamental types. When unmarshaled into interface{}, - // they're represented like this. - mapping = map[interface{}]interface{} - sequence = []interface{} -) - -// YAML deep-merges any number of YAML sources, with later sources taking -// priority over earlier ones. -// -// Maps are deep-merged. For example, -// {"one": 1, "two": 2} + {"one": 42, "three": 3} -// == {"one": 42, "two": 2, "three": 3} -// Sequences are replaced. For example, -// {"foo": [1, 2, 3]} + {"foo": [4, 5, 6]} -// == {"foo": [4, 5, 6]} -// -// In non-strict mode, duplicate map keys are allowed within a single source, -// with later values overwriting previous ones. Attempting to merge -// mismatched types (e.g., merging a sequence into a map) replaces the old -// value with the new. -// -// Enabling strict mode returns errors in both of the above cases. -func YAML(sources [][]byte, strict bool) (*bytes.Buffer, error) { - var merged interface{} - var hasContent bool - for _, r := range sources { - d := yaml.NewDecoder(bytes.NewReader(r)) - d.SetStrict(strict) - - var contents interface{} - if err := d.Decode(&contents); err == io.EOF { - // Skip empty and comment-only sources, which we should handle - // differently from explicit nils. - continue - } else if err != nil { - return nil, fmt.Errorf("couldn't decode source: %v", err) - } - - hasContent = true - pair, err := merge(merged, contents, strict) - if err != nil { - return nil, err // error is already descriptive enough - } - merged = pair - } - - buf := &bytes.Buffer{} - if !hasContent { - // No sources had any content. To distinguish this from a source with just - // an explicit top-level null, return an empty buffer. - return buf, nil - } - enc := yaml.NewEncoder(buf) - if err := enc.Encode(merged); err != nil { - return nil, errors.Wrap(err, "couldn't re-serialize merged YAML") - } - return buf, nil -} - -func merge(into, from interface{}, strict bool) (interface{}, error) { - // It's possible to handle this with a mass of reflection, but we only need - // to merge whole YAML files. Since we're always unmarshaling into - // interface{}, we only need to handle a few types. This ends up being - // cleaner if we just handle each case explicitly. - if into == nil { - return from, nil - } - if from == nil { - // Allow higher-priority YAML to explicitly nil out lower-priority entries. - return nil, nil - } - if IsScalar(into) && IsScalar(from) { - return from, nil - } - if IsSequence(into) && IsSequence(from) { - return from, nil - } - if IsMapping(into) && IsMapping(from) { - return mergeMapping(into.(mapping), from.(mapping), strict) - } - // YAML types don't match, so no merge is possible. For backward - // compatibility, ignore mismatches unless we're in strict mode and return - // the higher-priority value. - if !strict { - return from, nil - } - return nil, fmt.Errorf("can't merge a %s into a %s", describe(from), describe(into)) -} - -func mergeMapping(into, from mapping, strict bool) (mapping, error) { - merged := make(mapping, len(into)) - for k, v := range into { - merged[k] = v - } - for k := range from { - m, err := merge(merged[k], from[k], strict) - if err != nil { - return nil, err - } - merged[k] = m - } - return merged, nil -} - -// IsMapping reports whether a type is a mapping in YAML, represented as a -// map[interface{}]interface{}. -func IsMapping(i interface{}) bool { - _, is := i.(mapping) - return is -} - -// IsSequence reports whether a type is a sequence in YAML, represented as an -// []interface{}. -func IsSequence(i interface{}) bool { - _, is := i.(sequence) - return is -} - -// IsScalar reports whether a type is a scalar value in YAML. -func IsScalar(i interface{}) bool { - return !IsMapping(i) && !IsSequence(i) -} - -func describe(i interface{}) string { - if IsMapping(i) { - return "mapping" - } - if IsSequence(i) { - return "sequence" - } - return "scalar" -} diff --git a/pkg/yamlpatch/merge_test.go b/pkg/yamlpatch/merge_test.go deleted file mode 100644 index e86f6fea7..000000000 --- a/pkg/yamlpatch/merge_test.go +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright (c) 2018 Uber Technologies, Inc. -// -// Permission is hereby granted, free of charge, to any person obtaining a copy -// of this software and associated documentation files (the "Software"), to deal -// in the Software without restriction, including without limitation the rights -// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -// copies of the Software, and to permit persons to whom the Software is -// furnished to do so, subject to the following conditions: -// -// The above copyright notice and this permission notice shall be included in -// all copies or substantial portions of the Software. -// -// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN -// THE SOFTWARE. - -package yamlpatch - -import ( - "bytes" - "os" - "strings" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - yaml "gopkg.in/yaml.v2" -) - -func trimcr(s string) string { - return strings.ReplaceAll(s, "\r\n", "\n") -} - -func mustRead(t testing.TB, fname string) []byte { - contents, err := os.ReadFile(fname) - require.NoError(t, err, "failed to read file: %s", fname) - return contents -} - -func dump(t testing.TB, actual, expected string) { - // It's impossible to debug YAML if the actual and expected values are - // printed on a single line. - t.Logf("Actual:\n\n%s\n\n", actual) - t.Logf("Expected:\n\n%s\n\n", expected) -} - -func strip(s string) string { - // It's difficult to write string constants that are valid YAML. Normalize - // strings for ease of testing. - s = strings.TrimSpace(s) - s = strings.Replace(s, "\t", " ", -1) - return s -} - -func canonicalize(t testing.TB, s string) string { - // round-trip to canonicalize formatting - var i interface{} - require.NoError(t, - yaml.Unmarshal([]byte(strip(s)), &i), - "canonicalize: couldn't unmarshal YAML", - ) - formatted, err := yaml.Marshal(i) - require.NoError(t, err, "canonicalize: couldn't marshal YAML") - return string(bytes.TrimSpace(formatted)) -} - -func unmarshal(t testing.TB, s string) interface{} { - var i interface{} - require.NoError(t, yaml.Unmarshal([]byte(strip(s)), &i), "unmarshaling failed") - return i -} - -func succeeds(t testing.TB, strict bool, left, right, expect string) { - l, r := unmarshal(t, left), unmarshal(t, right) - m, err := merge(l, r, strict) - require.NoError(t, err, "merge failed") - - actualBytes, err := yaml.Marshal(m) - require.NoError(t, err, "couldn't marshal merged structure") - actual := canonicalize(t, string(actualBytes)) - expect = canonicalize(t, expect) - if !assert.Equal(t, expect, actual) { - dump(t, actual, expect) - } -} - -func fails(t testing.TB, strict bool, left, right string) { - _, err := merge(unmarshal(t, left), unmarshal(t, right), strict) - assert.Error(t, err, "merge succeeded") -} - -func TestIntegration(t *testing.T) { - base := mustRead(t, "testdata/base.yaml") - prod := mustRead(t, "testdata/production.yaml") - expect := mustRead(t, "testdata/expect.yaml") - - merged, err := YAML([][]byte{base, prod}, true /* strict */) - require.NoError(t, err, "merge failed") - - if !assert.Equal(t, trimcr(string(expect)), merged.String(), "unexpected contents") { - dump(t, merged.String(), string(expect)) - } -} - -func TestEmpty(t *testing.T) { - full := []byte("foo: bar\n") - null := []byte("~") - - tests := []struct { - desc string - sources [][]byte - expect string - }{ - {"empty base", [][]byte{nil, full}, string(full)}, - {"empty override", [][]byte{full, nil}, string(full)}, - {"both empty", [][]byte{nil, nil}, ""}, - {"null base", [][]byte{null, full}, string(full)}, - {"null override", [][]byte{full, null}, "null\n"}, - {"empty base and null override", [][]byte{nil, null}, "null\n"}, - {"null base and empty override", [][]byte{null, nil}, "null\n"}, - } - - for _, tt := range tests { - tt := tt - t.Run(tt.desc, func(t *testing.T) { - merged, err := YAML(tt.sources, true /* strict */) - require.NoError(t, err, "merge failed") - assert.Equal(t, tt.expect, merged.String(), "wrong contents after merge") - }) - } -} - -func TestSuccess(t *testing.T) { - left := ` -fun: [maserati, porsche] -practical: {toyota: camry, honda: accord} -occupants: - honda: {driver: jane, backseat: [nate]} - ` - right := ` -fun: [lamborghini, porsche] -practical: {honda: civic, nissan: altima} -occupants: - honda: {passenger: arthur, backseat: [nora]} - ` - expect := ` -fun: [lamborghini, porsche] -practical: {toyota: camry, honda: civic, nissan: altima} -occupants: - honda: {passenger: arthur, driver: jane, backseat: [nora]} - ` - succeeds(t, true, left, right, expect) - succeeds(t, false, left, right, expect) -} - -func TestErrors(t *testing.T) { - check := func(t testing.TB, strict bool, sources ...[]byte) error { - _, err := YAML(sources, strict) - return err - } - t.Run("tabs in source", func(t *testing.T) { - src := []byte("foo:\n\tbar:baz") - assert.Error(t, check(t, false, src), "expected error in permissive mode") - assert.Error(t, check(t, true, src), "expected error in strict mode") - }) - - t.Run("duplicated keys", func(t *testing.T) { - src := []byte("{foo: bar, foo: baz}") - assert.NoError(t, check(t, false, src), "expected success in permissive mode") - assert.Error(t, check(t, true, src), "expected error in permissive mode") - }) - - t.Run("merge error", func(t *testing.T) { - left := []byte("foo: [1, 2]") - right := []byte("foo: {bar: baz}") - assert.NoError(t, check(t, false, left, right), "expected success in permissive mode") - assert.Error(t, check(t, true, left, right), "expected error in strict mode") - }) -} - -func TestMismatchedTypes(t *testing.T) { - tests := []struct { - desc string - left, right string - }{ - {"sequence and mapping", "[one, two]", "{foo: bar}"}, - {"sequence and scalar", "[one, two]", "foo"}, - {"mapping and scalar", "{foo: bar}", "foo"}, - {"nested", "{foo: [one, two]}", "{foo: bar}"}, - } - - for _, tt := range tests { - tt := tt - t.Run(tt.desc+" strict", func(t *testing.T) { - fails(t, true, tt.left, tt.right) - }) - t.Run(tt.desc+" permissive", func(t *testing.T) { - // prefer the higher-priority value - succeeds(t, false, tt.left, tt.right, tt.right) - }) - } -} - -func TestBooleans(t *testing.T) { - // YAML helpfully interprets many strings as Booleans. - tests := []struct { - in, out string - }{ - {"yes", "true"}, - {"YES", "true"}, - {"on", "true"}, - {"ON", "true"}, - {"no", "false"}, - {"NO", "false"}, - {"off", "false"}, - {"OFF", "false"}, - } - - for _, tt := range tests { - tt := tt - t.Run(tt.in, func(t *testing.T) { - succeeds(t, true, "", tt.in, tt.out) - succeeds(t, false, "", tt.in, tt.out) - }) - } -} - -func TestExplicitNil(t *testing.T) { - base := `foo: {one: two}` - override := `foo: ~` - expect := `foo: ~` - succeeds(t, true, base, override, expect) - succeeds(t, false, base, override, expect) -} diff --git a/pkg/yamlpatch/patcher.go b/pkg/yamlpatch/patcher.go deleted file mode 100644 index 43a2a0cb8..000000000 --- a/pkg/yamlpatch/patcher.go +++ /dev/null @@ -1,173 +0,0 @@ -package yamlpatch - -import ( - "bytes" - "io" - "os" - - "github.com/pkg/errors" - log "github.com/sirupsen/logrus" - "gopkg.in/yaml.v2" -) - -type Patcher struct { - BaseFilePath string - PatchFilePath string - quiet bool -} - -func NewPatcher(filePath string, suffix string) *Patcher { - return &Patcher{ - BaseFilePath: filePath, - PatchFilePath: filePath + suffix, - quiet: false, - } -} - - -// SetQuiet sets the quiet flag, which will log as DEBUG_LEVEL instead of INFO -func (p *Patcher) SetQuiet(quiet bool) { - p.quiet = quiet -} - - -// read a single YAML file, check for errors (the merge package doesn't) then return the content as bytes. -func readYAML(filePath string) ([]byte, error) { - var content []byte - - var err error - - if content, err = os.ReadFile(filePath); err != nil { - return nil, errors.Wrap(err, "while reading yaml file") - } - - var yamlMap map[interface{}]interface{} - if err = yaml.Unmarshal(content, &yamlMap); err != nil { - return nil, errors.Wrap(err, filePath) - } - - return content, nil -} - -// MergedPatchContent reads a YAML file and, if it exists, its patch file, -// then merges them and returns it serialized. -func (p *Patcher) MergedPatchContent() ([]byte, error) { - var err error - - var base []byte - - base, err = readYAML(p.BaseFilePath) - if err != nil { - return nil, err - } - - var over []byte - - over, err = readYAML(p.PatchFilePath) - if errors.Is(err, os.ErrNotExist) { - return base, nil - } - - if err != nil { - return nil, err - } - - logf := log.Infof - if p.quiet { - logf = log.Debugf - } - logf("Patching yaml: '%s' with '%s'", p.BaseFilePath, p.PatchFilePath) - - var patched *bytes.Buffer - - // strict mode true, will raise errors for duplicate map keys and - // overriding with a different type - patched, err = YAML([][]byte{base, over}, true) - if err != nil { - return nil, err - } - - return patched.Bytes(), nil -} - -// read multiple YAML documents inside a file, and writes them to a buffer -// separated by the appropriate '---' terminators. -func decodeDocuments(file *os.File, buf *bytes.Buffer, finalDashes bool) error { - var ( - err error - docBytes []byte - ) - - dec := yaml.NewDecoder(file) - dec.SetStrict(true) - - dashTerminator := false - - for { - yml := make(map[interface{}]interface{}) - - err = dec.Decode(&yml) - if err != nil { - if errors.Is(err, io.EOF) { - break - } - return errors.Wrapf(err, "while decoding %s", file.Name()) - } - - docBytes, err = yaml.Marshal(&yml) - if err != nil { - return errors.Wrapf(err, "while marshaling %s", file.Name()) - } - - if dashTerminator { - buf.Write([]byte("---\n")) - } - - buf.Write(docBytes) - dashTerminator = true - } - if dashTerminator && finalDashes { - buf.Write([]byte("---\n")) - } - return nil -} - -// PrependedPatchContent collates the base .yaml file with the .yaml.patch, by putting -// the content of the patch BEFORE the base document. The result is a multi-document -// YAML in all cases, even if the base and patch files are single documents. -func (p *Patcher) PrependedPatchContent() ([]byte, error) { - var ( - result bytes.Buffer - patchFile *os.File - baseFile *os.File - err error - ) - - patchFile, err = os.Open(p.PatchFilePath) - // optional file, ignore if it does not exist - if err != nil && !errors.Is(err, os.ErrNotExist) { - return nil, errors.Wrapf(err, "while opening %s", p.PatchFilePath) - } - - if patchFile != nil { - if err = decodeDocuments(patchFile, &result, true); err != nil { - return nil, err - } - logf := log.Infof - if p.quiet { - logf = log.Debugf - } - logf("Prepending yaml: '%s' with '%s'", p.BaseFilePath, p.PatchFilePath) - } - - baseFile, err = os.Open(p.BaseFilePath) - if err != nil { - return nil, errors.Wrapf(err, "while opening %s", p.BaseFilePath) - } - - if err = decodeDocuments(baseFile, &result, false); err != nil { - return nil, err - } - - return result.Bytes(), nil -} diff --git a/pkg/yamlpatch/patcher_test.go b/pkg/yamlpatch/patcher_test.go deleted file mode 100644 index be4a855cf..000000000 --- a/pkg/yamlpatch/patcher_test.go +++ /dev/null @@ -1,313 +0,0 @@ -package yamlpatch_test - -import ( - "os" - "path/filepath" - "testing" - - "github.com/crowdsecurity/crowdsec/pkg/yamlpatch" - "github.com/stretchr/testify/require" -) - -// similar to the one in cstest, but with test number too. We cannot import -// cstest here because of circular dependency. -func requireErrorContains(t *testing.T, err error, expectedErr string) { - t.Helper() - - if expectedErr != "" { - require.ErrorContains(t, err, expectedErr) - - return - } - - require.NoError(t, err) -} - -func TestMergedPatchContent(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - base string - patch string - expected string - expectedErr string - }{ - { - "invalid yaml in base", - "notayaml", - "", - "", - "config.yaml: yaml: unmarshal errors:", - }, - { - "invalid yaml in base (detailed message)", - "notayaml", - "", - "", - "cannot unmarshal !!str `notayaml`", - }, - { - "invalid yaml in patch", - "", - "notayaml", - "", - "config.yaml.local: yaml: unmarshal errors:", - }, - { - "invalid yaml in patch (detailed message)", - "", - "notayaml", - "", - "cannot unmarshal !!str `notayaml`", - }, - { - "basic merge", - "{'first':{'one':1,'two':2},'second':{'three':3}}", - "{'first':{'one':10,'dos':2}}", - "{'first':{'one':10,'dos':2,'two':2},'second':{'three':3}}", - "", - }, - - // bools and zero values; here the "mergo" package had issues - // so we used something simpler. - - { - "bool merge - off if false", - "bool: on", - "bool: off", - "bool: false", - "", - }, - { - "bool merge - on is true", - "bool: off", - "bool: on", - "bool: true", - "", - }, - { - "string is not a bool - on to off", - "{'bool': 'on'}", - "{'bool': 'off'}", - "{'bool': 'off'}", - "", - }, - { - "string is not a bool - off to on", - "{'bool': 'off'}", - "{'bool': 'on'}", - "{'bool': 'on'}", - "", - }, - { - "bool merge - true to false", - "{'bool': true}", - "{'bool': false}", - "{'bool': false}", - "", - }, - { - "bool merge - false to true", - "{'bool': false}", - "{'bool': true}", - "{'bool': true}", - "", - }, - { - "string merge - value to value", - "{'string': 'value'}", - "{'string': ''}", - "{'string': ''}", - "", - }, - { - "sequence merge - value to empty", - "{'sequence': [1, 2]}", - "{'sequence': []}", - "{'sequence': []}", - "", - }, - { - "map merge - value to value", - "{'map': {'one': 1, 'two': 2}}", - "{'map': {}}", - "{'map': {'one': 1, 'two': 2}}", - "", - }, - - // mismatched types - - { - "can't merge a sequence into a mapping", - "map: {'key': 'value'}", - "map: ['value1', 'value2']", - "", - "can't merge a sequence into a mapping", - }, - { - "can't merge a scalar into a mapping", - "map: {'key': 'value'}", - "map: 3", - "", - "can't merge a scalar into a mapping", - }, - { - "can't merge a mapping into a sequence", - "sequence: ['value1', 'value2']", - "sequence: {'key': 'value'}", - "", - "can't merge a mapping into a sequence", - }, - { - "can't merge a scalar into a sequence", - "sequence: ['value1', 'value2']", - "sequence: 3", - "", - "can't merge a scalar into a sequence", - }, - { - "can't merge a sequence into a scalar", - "scalar: true", - "scalar: ['value1', 'value2']", - "", - "can't merge a sequence into a scalar", - }, - { - "can't merge a mapping into a scalar", - "scalar: true", - "scalar: {'key': 'value'}", - "", - "can't merge a mapping into a scalar", - }, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - dirPath, err := os.MkdirTemp("", "yamlpatch") - require.NoError(t, err) - - defer os.RemoveAll(dirPath) - - configPath := filepath.Join(dirPath, "config.yaml") - patchPath := filepath.Join(dirPath, "config.yaml.local") - err = os.WriteFile(configPath, []byte(tc.base), 0o600) - require.NoError(t, err) - - err = os.WriteFile(patchPath, []byte(tc.patch), 0o600) - require.NoError(t, err) - - patcher := yamlpatch.NewPatcher(configPath, ".local") - patchedBytes, err := patcher.MergedPatchContent() - requireErrorContains(t, err, tc.expectedErr) - require.YAMLEq(t, tc.expected, string(patchedBytes)) - }) - } -} - -func TestPrependedPatchContent(t *testing.T) { - t.Parallel() - - tests := []struct { - name string - base string - patch string - expected string - expectedErr string - }{ - // we test with scalars here, because YAMLeq does not work - // with multi-document files, so we need char-to-char comparison - // which is noisy with sequences and (unordered) mappings - { - "newlines are always appended, if missing, by yaml.Marshal()", - "foo: bar", - "", - "foo: bar\n", - "", - }, - { - "prepend empty document", - "foo: bar\n", - "", - "foo: bar\n", - "", - }, - { - "prepend a document to another", - "foo: bar", - "baz: qux", - "baz: qux\n---\nfoo: bar\n", - "", - }, - { - "prepend document with same key", - "foo: true", - "foo: false", - "foo: false\n---\nfoo: true\n", - "", - }, - { - "prepend multiple documents", - "one: 1\n---\ntwo: 2\n---\none: 3", - "four: 4\n---\none: 1.1", - "four: 4\n---\none: 1.1\n---\none: 1\n---\ntwo: 2\n---\none: 3\n", - "", - }, - { - "invalid yaml in base", - "blablabla", - "", - "", - "config.yaml: yaml: unmarshal errors:", - }, - { - "invalid yaml in base (detailed message)", - "blablabla", - "", - "", - "cannot unmarshal !!str `blablabla`", - }, - { - "invalid yaml in patch", - "", - "blablabla", - "", - "config.yaml.local: yaml: unmarshal errors:", - }, - { - "invalid yaml in patch (detailed message)", - "", - "blablabla", - "", - "cannot unmarshal !!str `blablabla`", - }, - } - - for _, tc := range tests { - tc := tc - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - dirPath, err := os.MkdirTemp("", "yamlpatch") - require.NoError(t, err) - - defer os.RemoveAll(dirPath) - - configPath := filepath.Join(dirPath, "config.yaml") - patchPath := filepath.Join(dirPath, "config.yaml.local") - - err = os.WriteFile(configPath, []byte(tc.base), 0o600) - require.NoError(t, err) - - err = os.WriteFile(patchPath, []byte(tc.patch), 0o600) - require.NoError(t, err) - - patcher := yamlpatch.NewPatcher(configPath, ".local") - patchedBytes, err := patcher.PrependedPatchContent() - requireErrorContains(t, err, tc.expectedErr) - // YAMLeq does not handle multiple documents - require.Equal(t, tc.expected, string(patchedBytes)) - }) - } -} diff --git a/pkg/yamlpatch/testdata/base.yaml b/pkg/yamlpatch/testdata/base.yaml deleted file mode 100644 index 4ac551ad5..000000000 --- a/pkg/yamlpatch/testdata/base.yaml +++ /dev/null @@ -1,13 +0,0 @@ -fun: - - maserati - - porsche - -practical: - toyota: camry - honda: accord - -occupants: - honda: - driver: jane - backseat: - - nate diff --git a/pkg/yamlpatch/testdata/expect.yaml b/pkg/yamlpatch/testdata/expect.yaml deleted file mode 100644 index c19091563..000000000 --- a/pkg/yamlpatch/testdata/expect.yaml +++ /dev/null @@ -1,13 +0,0 @@ -fun: -- lamborghini -- porsche -occupants: - honda: - backseat: - - nora - driver: jane - passenger: arthur -practical: - honda: civic - nissan: altima - toyota: camry diff --git a/pkg/yamlpatch/testdata/production.yaml b/pkg/yamlpatch/testdata/production.yaml deleted file mode 100644 index 7dab2aeee..000000000 --- a/pkg/yamlpatch/testdata/production.yaml +++ /dev/null @@ -1,13 +0,0 @@ -fun: - - lamborghini - - porsche - -practical: - honda: civic - nissan: altima - -occupants: - honda: - passenger: arthur - backseat: - - nora