From c76325b91b64137ee3df3f02b473270819f8d241 Mon Sep 17 00:00:00 2001 From: blotus Date: Tue, 19 Mar 2024 17:42:08 +0100 Subject: [PATCH 01/22] Update windows pipeline (#2909) --- azure-pipelines.yml | 154 ++++++++++++++++++++++++++++++-------------- make_chocolatey.ps1 | 4 +- make_installer.ps1 | 2 +- 3 files changed, 111 insertions(+), 49 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6151fe222..f90af6f17 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -15,15 +15,9 @@ pool: stages: - stage: Build jobs: - - job: + - job: Build displayName: "Build" steps: - - task: DotNetCoreCLI@2 - displayName: "Install SignClient" - inputs: - command: 'custom' - custom: 'tool' - arguments: 'install --global SignClient --version 1.3.155' - task: GoTool@0 displayName: "Install Go" inputs: @@ -39,24 +33,14 @@ stages: #we are not calling make windows_installer because we want to sign the binaries before they are added to the MSI script: | make build BUILD_RE2_WASM=1 - - task: AzureKeyVault@2 - inputs: - azureSubscription: 'Azure subscription 1(8a93ab40-7e99-445e-ad47-0f6a3e2ef546)' - KeyVaultName: 'CodeSigningSecrets' - SecretsFilter: 'CodeSigningUser,CodeSigningPassword' - RunAsPreJob: false - - - task: DownloadSecureFile@1 - inputs: - secureFile: appsettings.json - - - pwsh: | - SignClient.exe Sign --name "crowdsec-binaries" ` - --input "**/*.exe" --config (Join-Path -Path $(Agent.TempDirectory) -ChildPath "appsettings.json") ` - --user $(CodeSigningUser) --secret '$(CodeSigningPassword)' - displayName: "Sign Crowdsec binaries + plugins" + - pwsh: | $build_version=$env:BUILD_SOURCEBRANCHNAME + #Override the version if it's set in the pipeline + if ( ${env:USERBUILDVERSION} -ne "") + { + $build_version = ${env:USERBUILDVERSION} + } if ($build_version.StartsWith("v")) { $build_version = $build_version.Substring(1) @@ -69,35 +53,112 @@ stages: displayName: GetCrowdsecVersion name: GetCrowdsecVersion - pwsh: | - .\make_installer.ps1 -version '$(GetCrowdsecVersion.BuildVersion)' + Get-ChildItem -Path .\cmd -Directory | ForEach-Object { + $dirName = $_.Name + Get-ChildItem -Path .\cmd\$dirName -File -Filter '*.exe' | ForEach-Object { + $fileName = $_.Name + $destDir = Join-Path $(Build.ArtifactStagingDirectory) cmd\$dirName + New-Item -ItemType Directory -Path $destDir -Force + Copy-Item -Path .\cmd\$dirName\$fileName -Destination $destDir + } + } + displayName: "Copy binaries to staging directory" + - task: PublishPipelineArtifact@1 + inputs: + targetPath: '$(Build.ArtifactStagingDirectory)' + artifact: 'unsigned_binaries' + displayName: "Upload binaries artifact" + + - stage: Sign + dependsOn: Build + variables: + - group: 'FOSS Build Variables' + - name: BuildVersion + value: $[ stageDependencies.Build.Build.outputs['GetCrowdsecVersion.BuildVersion'] ] + condition: succeeded() + jobs: + - job: Sign + displayName: "Sign" + steps: + - download: current + artifact: unsigned_binaries + displayName: "Download binaries artifact" + - task: CopyFiles@2 + inputs: + SourceFolder: '$(Pipeline.Workspace)/unsigned_binaries' + TargetFolder: '$(Build.SourcesDirectory)' + displayName: "Copy binaries to workspace" + - task: DotNetCoreCLI@2 + displayName: "Install SignTool tool" + inputs: + command: 'custom' + custom: 'tool' + arguments: install --global sign --version 0.9.0-beta.23127.3 + - task: AzureKeyVault@2 + displayName: "Get signing parameters" + inputs: + azureSubscription: "Azure subscription" + KeyVaultName: "$(KeyVaultName)" + SecretsFilter: "TenantId,ClientId,ClientSecret,Certificate,KeyVaultUrl" + - pwsh: | + sign code azure-key-vault ` + "**/*.exe" ` + --base-directory "$(Build.SourcesDirectory)/cmd/" ` + --publisher-name "CrowdSec" ` + --description "CrowdSec" ` + --description-url "https://github.com/crowdsecurity/crowdsec" ` + --azure-key-vault-tenant-id "$(TenantId)" ` + --azure-key-vault-client-id "$(ClientId)" ` + --azure-key-vault-client-secret "$(ClientSecret)" ` + --azure-key-vault-certificate "$(Certificate)" ` + --azure-key-vault-url "$(KeyVaultUrl)" + displayName: "Sign crowdsec binaries" + - pwsh: | + .\make_installer.ps1 -version '$(BuildVersion)' displayName: "Build Crowdsec MSI" name: BuildMSI - - pwsh: | - .\make_chocolatey.ps1 -version '$(GetCrowdsecVersion.BuildVersion)' + .\make_chocolatey.ps1 -version '$(BuildVersion)' displayName: "Build Chocolatey nupkg" - - pwsh: | - SignClient.exe Sign --name "crowdsec-msi" ` - --input "*.msi" --config (Join-Path -Path $(Agent.TempDirectory) -ChildPath "appsettings.json") ` - --user $(CodeSigningUser) --secret '$(CodeSigningPassword)' - displayName: "Sign Crowdsec MSI" - - - task: PublishBuildArtifacts@1 + sign code azure-key-vault ` + "*.msi" ` + --base-directory "$(Build.SourcesDirectory)" ` + --publisher-name "CrowdSec" ` + --description "CrowdSec" ` + --description-url "https://github.com/crowdsecurity/crowdsec" ` + --azure-key-vault-tenant-id "$(TenantId)" ` + --azure-key-vault-client-id "$(ClientId)" ` + --azure-key-vault-client-secret "$(ClientSecret)" ` + --azure-key-vault-certificate "$(Certificate)" ` + --azure-key-vault-url "$(KeyVaultUrl)" + displayName: "Sign MSI package" + - pwsh: | + sign code azure-key-vault ` + "*.nupkg" ` + --base-directory "$(Build.SourcesDirectory)" ` + --publisher-name "CrowdSec" ` + --description "CrowdSec" ` + --description-url "https://github.com/crowdsecurity/crowdsec" ` + --azure-key-vault-tenant-id "$(TenantId)" ` + --azure-key-vault-client-id "$(ClientId)" ` + --azure-key-vault-client-secret "$(ClientSecret)" ` + --azure-key-vault-certificate "$(Certificate)" ` + --azure-key-vault-url "$(KeyVaultUrl)" + displayName: "Sign nuget package" + - task: PublishPipelineArtifact@1 inputs: - PathtoPublish: '$(Build.Repository.LocalPath)\\crowdsec_$(GetCrowdsecVersion.BuildVersion).msi' - ArtifactName: 'crowdsec.msi' - publishLocation: 'Container' - displayName: "Upload MSI artifact" - - - task: PublishBuildArtifacts@1 + targetPath: '$(Build.SourcesDirectory)/crowdsec_$(BuildVersion).msi' + artifact: 'signed_msi_package' + displayName: "Upload signed MSI artifact" + - task: PublishPipelineArtifact@1 inputs: - PathtoPublish: '$(Build.Repository.LocalPath)\\windows\\Chocolatey\\crowdsec\\crowdsec.$(GetCrowdsecVersion.BuildVersion).nupkg' - ArtifactName: 'crowdsec.nupkg' - publishLocation: 'Container' - displayName: "Upload nupkg artifact" + targetPath: '$(Build.SourcesDirectory)/crowdsec.$(BuildVersion).nupkg' + artifact: 'signed_nuget_package' + displayName: "Upload signed nuget artifact" + - stage: Publish - dependsOn: Build + dependsOn: Sign jobs: - deployment: "Publish" displayName: "Publish to GitHub" @@ -119,8 +180,7 @@ stages: assetUploadMode: 'replace' addChangeLog: false isPreRelease: true #we force prerelease because the pipeline is invoked on tag creation, which happens when we do a prerelease - #the .. is an ugly hack, but I can't find the var that gives D:\a\1 ... assets: | - $(Build.ArtifactStagingDirectory)\..\crowdsec.msi/*.msi - $(Build.ArtifactStagingDirectory)\..\crowdsec.nupkg/*.nupkg + $(Pipeline.Workspace)/signed_msi_package/*.msi + $(Pipeline.Workspace)/signed_nuget_package/*.nupkg condition: ne(variables['GetLatestPrelease.LatestPreRelease'], '') diff --git a/make_chocolatey.ps1 b/make_chocolatey.ps1 index 67f85c33d..cceed2840 100644 --- a/make_chocolatey.ps1 +++ b/make_chocolatey.ps1 @@ -15,4 +15,6 @@ if ($version.Contains("-")) Set-Location .\windows\Chocolatey\crowdsec Copy-Item ..\..\..\crowdsec_$version.msi tools\crowdsec.msi -choco pack --version $version \ No newline at end of file +choco pack --version $version + +Copy-Item crowdsec.$version.nupkg ..\..\..\ \ No newline at end of file diff --git a/make_installer.ps1 b/make_installer.ps1 index a20ffaf55..c927452ff 100644 --- a/make_installer.ps1 +++ b/make_installer.ps1 @@ -1,7 +1,7 @@ param ( $version ) -$env:Path += ";C:\Program Files (x86)\WiX Toolset v3.11\bin" +$env:Path += ";C:\Program Files (x86)\WiX Toolset v3.14\bin" if ($version.StartsWith("v")) { $version = $version.Substring(1) From d9f2a22ee5b5a982bbf7a2f970b5929944ad7f2b Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Wed, 20 Mar 2024 13:27:28 +0100 Subject: [PATCH 02/22] cscli metrics -> sort table order (#2908) --- cmd/crowdsec-cli/metrics.go | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/cmd/crowdsec-cli/metrics.go b/cmd/crowdsec-cli/metrics.go index 0f9234386..ad2b9ee79 100644 --- a/cmd/crowdsec-cli/metrics.go +++ b/cmd/crowdsec-cli/metrics.go @@ -272,9 +272,7 @@ func (ms metricStore) Format(out io.Writer, sections []string, formatType string // if no sections are specified, we want all of them if len(sections) == 0 { - for section := range ms { - sections = append(sections, section) - } + sections = maptools.SortedKeys(ms) } for _, section := range sections { @@ -283,7 +281,7 @@ func (ms metricStore) Format(out io.Writer, sections []string, formatType string switch formatType { case "human": - for section := range want { + for _, section := range maptools.SortedKeys(want) { want[section].Table(out, noUnit, showEmpty) } case "json": @@ -376,7 +374,7 @@ cscli metrics list`, } // expandAlias returns a list of sections. The input can be a list of sections or alias. -func (cli *cliMetrics) expandSectionGroups(args []string) []string { +func (cli *cliMetrics) expandAlias(args []string) []string { ret := []string{} for _, section := range args { @@ -422,7 +420,7 @@ cscli metrics show acquisition parsers scenarios stash -o json`, // Positional args are optional DisableAutoGenTag: true, RunE: func(_ *cobra.Command, args []string) error { - args = cli.expandSectionGroups(args) + args = cli.expandAlias(args) return cli.show(args, url, noUnit) }, } From 75a50c0c9d5b31cc6200e55f13c7b94ae1d5a41c Mon Sep 17 00:00:00 2001 From: "Thibault \"bui\" Koechlin" Date: Wed, 20 Mar 2024 14:02:29 +0100 Subject: [PATCH 03/22] improve a bit cscli examples when it comes to list mgmt (#2911) --- cmd/crowdsec-cli/alerts.go | 12 +++++++----- cmd/crowdsec-cli/decisions.go | 3 ++- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/cmd/crowdsec-cli/alerts.go b/cmd/crowdsec-cli/alerts.go index ce304bcc7..908466f9e 100644 --- a/cmd/crowdsec-cli/alerts.go +++ b/cmd/crowdsec-cli/alerts.go @@ -177,9 +177,9 @@ func (cli *cliAlerts) displayOneAlert(alert *models.Alert, withDetail bool) erro return nil } -type cliAlerts struct{ +type cliAlerts struct { client *apiclient.ApiClient - cfg configGetter + cfg configGetter } func NewCLIAlerts(getconfig configGetter) *cliAlerts { @@ -253,8 +253,10 @@ func (cli *cliAlerts) NewListCmd() *cobra.Command { Example: `cscli alerts list cscli alerts list --ip 1.2.3.4 cscli alerts list --range 1.2.3.0/24 +cscli alerts list --origin lists cscli alerts list -s crowdsecurity/ssh-bf cscli alerts list --type ban`, + Long: `List alerts with optional filters`, DisableAutoGenTag: true, RunE: func(cmd *cobra.Command, _ []string) error { if err := manageCliDecisionAlerts(alertListFilter.IPEquals, alertListFilter.RangeEquals, @@ -358,7 +360,7 @@ func (cli *cliAlerts) NewDeleteCmd() *cobra.Command { var ( ActiveDecision *bool AlertDeleteAll bool - delAlertByID string + delAlertByID string ) var alertDeleteFilter = apiclient.AlertsDeleteOpts{ @@ -449,7 +451,7 @@ cscli alerts delete -s crowdsecurity/ssh-bf"`, return nil }, } - + flags := cmd.Flags() flags.SortFlags = false flags.StringVar(alertDeleteFilter.ScopeEquals, "scope", "", "the scope (ie. ip,range)") @@ -520,7 +522,7 @@ func (cli *cliAlerts) NewInspectCmd() *cobra.Command { func (cli *cliAlerts) NewFlushCmd() *cobra.Command { var ( maxItems int - maxAge string + maxAge string ) cmd := &cobra.Command{ diff --git a/cmd/crowdsec-cli/decisions.go b/cmd/crowdsec-cli/decisions.go index d71653678..a97536ddc 100644 --- a/cmd/crowdsec-cli/decisions.go +++ b/cmd/crowdsec-cli/decisions.go @@ -195,7 +195,7 @@ func (cli *cliDecisions) newListCmd() *cobra.Command { Example: `cscli decisions list -i 1.2.3.4 cscli decisions list -r 1.2.3.0/24 cscli decisions list -s crowdsecurity/ssh-bf -cscli decisions list -t ban +cscli decisions list --origin lists --scenario list_name `, Args: cobra.ExactArgs(0), DisableAutoGenTag: true, @@ -436,6 +436,7 @@ func (cli *cliDecisions) newDeleteCmd() *cobra.Command { cscli decisions delete -i 1.2.3.4 cscli decisions delete --id 42 cscli decisions delete --type captcha +cscli decisions delete --origin lists --scenario list_name `, /*TBD : refaire le Long/Example*/ PreRunE: func(cmd *cobra.Command, _ []string) error { From 7779c7ff0c010537d9a1dc3a73abdb0e2c750c33 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Wed, 20 Mar 2024 15:46:14 +0100 Subject: [PATCH 04/22] hub update: reload crowdsec if only data files have changed (#2912) --- pkg/cwhub/dataset.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pkg/cwhub/dataset.go b/pkg/cwhub/dataset.go index 4612f3576..97fd9c5a0 100644 --- a/pkg/cwhub/dataset.go +++ b/pkg/cwhub/dataset.go @@ -32,6 +32,9 @@ func downloadFile(url string, destPath string) error { return fmt.Errorf("bad http code %d for %s", resp.StatusCode, url) } + // Download to a temporary location to avoid corrupting files + // that are currently in use or memory mapped. + tmpFile, err := os.CreateTemp(filepath.Dir(destPath), filepath.Base(destPath)+".*.tmp") if err != nil { return err @@ -57,6 +60,11 @@ func downloadFile(url string, destPath string) error { return err } + // a check on stdout is used while scripting to know if the hub has been upgraded + // and a configuration reload is required + // TODO: use a better way to communicate this + fmt.Printf("updated %s\n", filepath.Base(destPath)) + if err = os.Rename(tmpFileName, destPath); err != nil { return err } From 52f86c2d10f51a238b9e23961e65075a44c85fc9 Mon Sep 17 00:00:00 2001 From: blotus Date: Thu, 21 Mar 2024 11:39:37 +0100 Subject: [PATCH 05/22] add libinjection expr helpers (#2914) --- pkg/exprhelpers/expr_lib.go | 14 +++++++ pkg/exprhelpers/libinjection.go | 17 ++++++++ pkg/exprhelpers/libinjection_test.go | 60 ++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 pkg/exprhelpers/libinjection.go create mode 100644 pkg/exprhelpers/libinjection_test.go diff --git a/pkg/exprhelpers/expr_lib.go b/pkg/exprhelpers/expr_lib.go index db191b84a..520799287 100644 --- a/pkg/exprhelpers/expr_lib.go +++ b/pkg/exprhelpers/expr_lib.go @@ -441,6 +441,20 @@ var exprFuncs = []exprCustomFunc{ new(func(float64, float64) bool), }, }, + { + name: "LibInjectionIsSQLI", + function: LibInjectionIsSQLI, + signature: []interface{}{ + new(func(string) bool), + }, + }, + { + name: "LibInjectionIsXSS", + function: LibInjectionIsXSS, + signature: []interface{}{ + new(func(string) bool), + }, + }, } //go 1.20 "CutPrefix": strings.CutPrefix, diff --git a/pkg/exprhelpers/libinjection.go b/pkg/exprhelpers/libinjection.go new file mode 100644 index 000000000..e9f33e4f4 --- /dev/null +++ b/pkg/exprhelpers/libinjection.go @@ -0,0 +1,17 @@ +package exprhelpers + +import "github.com/corazawaf/libinjection-go" + +func LibInjectionIsSQLI(params ...any) (any, error) { + str := params[0].(string) + + ret, _ := libinjection.IsSQLi(str) + return ret, nil +} + +func LibInjectionIsXSS(params ...any) (any, error) { + str := params[0].(string) + + ret := libinjection.IsXSS(str) + return ret, nil +} diff --git a/pkg/exprhelpers/libinjection_test.go b/pkg/exprhelpers/libinjection_test.go new file mode 100644 index 000000000..7b4ab825d --- /dev/null +++ b/pkg/exprhelpers/libinjection_test.go @@ -0,0 +1,60 @@ +package exprhelpers + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestLibinjectionHelpers(t *testing.T) { + tests := []struct { + name string + function func(params ...any) (any, error) + params []any + expectResult any + }{ + { + name: "LibInjectionIsSQLI", + function: LibInjectionIsSQLI, + params: []any{"?__f__73=73&&__f__75=75&delivery=1&max=24.9&min=15.9&n=12&o=2&p=(select(0)from(select(sleep(15)))v)/*'%2B(select(0)from(select(sleep(15)))v)%2B'\x22%2B(select(0)from(select(sleep(15)))v)%2B\x22*/&rating=4"}, + expectResult: true, + }, + { + name: "LibInjectionIsSQLI - no match", + function: LibInjectionIsSQLI, + params: []any{"?bla=42&foo=bar"}, + expectResult: false, + }, + { + name: "LibInjectionIsSQLI - no match 2", + function: LibInjectionIsSQLI, + params: []any{"https://foo.com/asdkfj?bla=42&foo=bar"}, + expectResult: false, + }, + { + name: "LibInjectionIsXSS", + function: LibInjectionIsXSS, + params: []any{""}, + expectResult: true, + }, + { + name: "LibInjectionIsXSS - no match", + function: LibInjectionIsXSS, + params: []any{"?bla=42&foo=bar"}, + expectResult: false, + }, + { + name: "LibInjectionIsXSS - no match 2", + function: LibInjectionIsXSS, + params: []any{"https://foo.com/asdkfj?bla=42&foo[]=bar&foo"}, + expectResult: false, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result, _ := test.function(test.params...) + assert.Equal(t, test.expectResult, result) + }) + } +} From 2e1ddec107ed01937809d4f53b4ab8fa2e00f7e4 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 25 Mar 2024 10:40:41 +0100 Subject: [PATCH 06/22] cscli: Add user-agent to all hub requests (#2915) * cscli: Add user-agent to all hub requests * fix unit test and avoid httpmock * fix windows test --- pkg/cwhub/cwhub.go | 14 ++++++++++ pkg/cwhub/dataset_test.go | 58 +++++++++++++++++++++------------------ 2 files changed, 46 insertions(+), 26 deletions(-) diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index 9ce091fad..a7864d4c0 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -7,10 +7,24 @@ import ( "sort" "strings" "time" + + "github.com/crowdsecurity/go-cs-lib/version" ) +// hubTransport wraps a Transport to set a custom User-Agent. +type hubTransport struct { + http.RoundTripper +} + +func (t *hubTransport) RoundTrip(req *http.Request) (*http.Response, error) { + req.Header.Set("User-Agent", "crowdsec/"+version.String()) + return t.RoundTripper.RoundTrip(req) +} + +// hubClient is the HTTP client used to communicate with the CrowdSec Hub. var hubClient = &http.Client{ Timeout: 120 * time.Second, + Transport: &hubTransport{http.DefaultTransport}, } // safePath returns a joined path and ensures that it does not escape the base directory. diff --git a/pkg/cwhub/dataset_test.go b/pkg/cwhub/dataset_test.go index 93d3e3bf0..e48202e48 100644 --- a/pkg/cwhub/dataset_test.go +++ b/pkg/cwhub/dataset_test.go @@ -1,50 +1,56 @@ package cwhub import ( + "io" + "net/http" + "net/http/httptest" "os" + "path/filepath" "testing" - "github.com/jarcoal/httpmock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/go-cs-lib/cstest" ) func TestDownloadFile(t *testing.T) { - examplePath := "./example.txt" - defer os.Remove(examplePath) + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/xx": + w.WriteHeader(http.StatusOK) + _, _ = io.WriteString(w, "example content oneoneone") + default: + w.WriteHeader(http.StatusNotFound) + _, _ = io.WriteString(w, "not found") + } + })) + defer ts.Close() - httpmock.Activate() - defer httpmock.DeactivateAndReset() + dest := filepath.Join(t.TempDir(), "example.txt") + defer os.Remove(dest) - // OK - httpmock.RegisterResponder( - "GET", - "https://example.com/xx", - httpmock.NewStringResponder(200, "example content oneoneone"), - ) - - httpmock.RegisterResponder( - "GET", - "https://example.com/x", - httpmock.NewStringResponder(404, "not found"), - ) - - err := downloadFile("https://example.com/xx", examplePath) + err := downloadFile(ts.URL+"/xx", dest) require.NoError(t, err) - content, err := os.ReadFile(examplePath) + content, err := os.ReadFile(dest) assert.Equal(t, "example content oneoneone", string(content)) require.NoError(t, err) // bad uri - err = downloadFile("https://zz.com", examplePath) - require.Error(t, err) + err = downloadFile("https://zz.com", dest) + cstest.RequireErrorContains(t, err, "lookup zz.com") + cstest.RequireErrorContains(t, err, "no such host") // 404 - err = downloadFile("https://example.com/x", examplePath) - require.Error(t, err) + err = downloadFile(ts.URL+"/x", dest) + cstest.RequireErrorContains(t, err, "bad http code 404") // bad target - err = downloadFile("https://example.com/xx", "") - require.Error(t, err) + err = downloadFile(ts.URL+"/xx", "") + cstest.RequireErrorContains(t, err, cstest.PathNotFoundMessage) + + // destination directory does not exist + err = downloadFile(ts.URL+"/xx", filepath.Join(t.TempDir(), "missing/example.txt")) + cstest.RequireErrorContains(t, err, cstest.PathNotFoundMessage) } From f6bb8412c55426c7f459a85fa4f1b08fec4f487f Mon Sep 17 00:00:00 2001 From: Christian Kampka Date: Mon, 25 Mar 2024 16:20:16 +0100 Subject: [PATCH 07/22] Add patterns_dir configuration option (#2868) * Add patterns_dir configuration option * Update config.yaml --------- Co-authored-by: mmetc <92726601+mmetc@users.noreply.github.com> --- pkg/csconfig/config_paths.go | 6 ++++++ pkg/parser/unix_parser.go | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pkg/csconfig/config_paths.go b/pkg/csconfig/config_paths.go index 71e3bacda..3de05ee02 100644 --- a/pkg/csconfig/config_paths.go +++ b/pkg/csconfig/config_paths.go @@ -13,6 +13,7 @@ type ConfigurationPaths struct { HubDir string `yaml:"hub_dir,omitempty"` PluginDir string `yaml:"plugin_dir,omitempty"` NotificationDir string `yaml:"notification_dir,omitempty"` + PatternDir string `yaml:"pattern_dir,omitempty"` } func (c *Config) loadConfigurationPaths() error { @@ -33,6 +34,10 @@ func (c *Config) loadConfigurationPaths() error { c.ConfigPaths.HubIndexFile = filepath.Clean(c.ConfigPaths.HubDir + "/.index.json") } + if c.ConfigPaths.PatternDir == "" { + c.ConfigPaths.PatternDir = filepath.Join(c.ConfigPaths.ConfigDir, "patterns/") + } + var configPathsCleanup = []*string{ &c.ConfigPaths.HubDir, &c.ConfigPaths.HubIndexFile, @@ -41,6 +46,7 @@ func (c *Config) loadConfigurationPaths() error { &c.ConfigPaths.SimulationFilePath, &c.ConfigPaths.PluginDir, &c.ConfigPaths.NotificationDir, + &c.ConfigPaths.PatternDir, } for _, k := range configPathsCleanup { if *k == "" { diff --git a/pkg/parser/unix_parser.go b/pkg/parser/unix_parser.go index 617e46189..720bac3d1 100644 --- a/pkg/parser/unix_parser.go +++ b/pkg/parser/unix_parser.go @@ -98,7 +98,7 @@ func NewParsers(hub *cwhub.Hub) *Parsers { func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) { var err error - patternsDir := filepath.Join(cConfig.ConfigPaths.ConfigDir, "patterns/") + patternsDir := cConfig.ConfigPaths.PatternDir log.Infof("Loading grok library %s", patternsDir) /* load base regexps for two grok parsers */ parsers.Ctx, err = Init(map[string]interface{}{"patterns": patternsDir, From 368d22ec305c483c45ec459baafff34e8308e8de Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Mar 2024 09:12:57 +0100 Subject: [PATCH 08/22] Bump github.com/jackc/pgx/v4 from 4.14.1 to 4.18.2 (#2887) Bumps [github.com/jackc/pgx/v4](https://github.com/jackc/pgx) from 4.14.1 to 4.18.2. - [Changelog](https://github.com/jackc/pgx/blob/v4.18.2/CHANGELOG.md) - [Commits](https://github.com/jackc/pgx/compare/v4.14.1...v4.18.2) --- updated-dependencies: - dependency-name: github.com/jackc/pgx/v4 dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 18 +++++++++--------- go.sum | 36 ++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 22d52be1c..604fef762 100644 --- a/go.mod +++ b/go.mod @@ -55,7 +55,7 @@ require ( github.com/hashicorp/go-version v1.2.1 github.com/hexops/gotextdiff v1.0.3 github.com/ivanpirog/coloredcobra v1.0.1 - github.com/jackc/pgx/v4 v4.14.1 + github.com/jackc/pgx/v4 v4.18.2 github.com/jarcoal/httpmock v1.1.0 github.com/jszwec/csvutil v1.5.1 github.com/lithammer/dedent v1.1.0 @@ -81,9 +81,9 @@ require ( github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 github.com/wasilibs/go-re2 v1.3.0 github.com/xhit/go-simple-mail/v2 v2.16.0 - golang.org/x/crypto v0.17.0 + golang.org/x/crypto v0.20.0 golang.org/x/mod v0.11.0 - golang.org/x/sys v0.15.0 + golang.org/x/sys v0.17.0 golang.org/x/text v0.14.0 google.golang.org/grpc v1.56.3 google.golang.org/protobuf v1.33.0 @@ -137,12 +137,12 @@ require ( github.com/imdario/mergo v0.3.12 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect - github.com/jackc/pgconn v1.10.1 // indirect + github.com/jackc/pgconn v1.14.3 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect - github.com/jackc/pgproto3/v2 v2.2.0 // indirect - github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect - github.com/jackc/pgtype v1.9.1 // indirect + github.com/jackc/pgproto3/v2 v2.3.3 // indirect + github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect + github.com/jackc/pgtype v1.14.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect @@ -198,9 +198,9 @@ require ( github.com/zclconf/go-cty v1.8.0 // indirect go.mongodb.org/mongo-driver v1.9.4 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/net v0.19.0 // indirect + golang.org/x/net v0.21.0 // indirect golang.org/x/sync v0.6.0 // indirect - golang.org/x/term v0.15.0 // indirect + golang.org/x/term v0.17.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.8.1-0.20230428195545-5283a0178901 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect diff --git a/go.sum b/go.sum index 07bddaf89..17a83cd66 100644 --- a/go.sum +++ b/go.sum @@ -368,8 +368,8 @@ github.com/jackc/pgconn v0.0.0-20190831204454-2fabfa3c18b7/go.mod h1:ZJKsE/KZfsU github.com/jackc/pgconn v1.8.0/go.mod h1:1C2Pb36bGIP9QHGBYCjnyhqu7Rv3sGshaQUvmfGIB/o= github.com/jackc/pgconn v1.9.0/go.mod h1:YctiPyvzfU11JFxoXokUOOKQXQmDMoJL9vJzHH8/2JY= github.com/jackc/pgconn v1.9.1-0.20210724152538-d89c8390a530/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= -github.com/jackc/pgconn v1.10.1 h1:DzdIHIjG1AxGwoEEqS+mGsURyjt4enSmqzACXvVzOT8= -github.com/jackc/pgconn v1.10.1/go.mod h1:4z2w8XhRbP1hYxkpTuBjTS3ne3J48K83+u0zoyvg2pI= +github.com/jackc/pgconn v1.14.3 h1:bVoTr12EGANZz66nZPkMInAV/KHD2TxH9npjXXgiB3w= +github.com/jackc/pgconn v1.14.3/go.mod h1:RZbme4uasqzybK2RK5c65VsHxoyaml09lx3tXOcO/VM= github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= @@ -385,26 +385,26 @@ github.com/jackc/pgproto3/v2 v2.0.0-rc3/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvW github.com/jackc/pgproto3/v2 v2.0.0-rc3.0.20190831210041-4c03ce451f29/go.mod h1:ryONWYqW6dqSg1Lw6vXNMXoBJhpzvWKnT95C46ckYeM= github.com/jackc/pgproto3/v2 v2.0.6/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgproto3/v2 v2.1.1/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgproto3/v2 v2.2.0 h1:r7JypeP2D3onoQTCxWdTpCtJ4D+qpKr0TxvoyMhZ5ns= -github.com/jackc/pgproto3/v2 v2.2.0/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= -github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b h1:C8S2+VttkHFdOOCXJe+YGfa4vHYwlt4Zx+IVXQ97jYg= +github.com/jackc/pgproto3/v2 v2.3.3 h1:1HLSx5H+tXR9pW3in3zaztoEwQYRC9SQaYUHjTSUOag= +github.com/jackc/pgproto3/v2 v2.3.3/go.mod h1:WfJCnwN3HIg9Ish/j3sgWXnAfK8A9Y0bwXYU5xKaEdA= github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b/go.mod h1:vsD4gTJCa9TptPL8sPkXrLZ+hDuNrZCnj29CQpr4X1E= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/Y25WS6cokEszi5g+S0QxI/d45PkRi7Nk= +github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM= github.com/jackc/pgtype v0.0.0-20190421001408-4ed0de4755e0/go.mod h1:hdSHsc1V01CGwFsrv11mJRHWJ6aifDLfdV3aVjFF0zg= github.com/jackc/pgtype v0.0.0-20190824184912-ab885b375b90/go.mod h1:KcahbBH1nCMSo2DXpzsoWOAfFkdEtEJpPbVLq8eE+mc= github.com/jackc/pgtype v0.0.0-20190828014616-a8802b16cc59/go.mod h1:MWlu30kVJrUS8lot6TQqcg7mtthZ9T0EoIBFiJcmcyw= github.com/jackc/pgtype v1.8.1-0.20210724151600-32e20a603178/go.mod h1:C516IlIV9NKqfsMCXTdChteoXmwgUceqaLfjg2e3NlM= -github.com/jackc/pgtype v1.9.1 h1:MJc2s0MFS8C3ok1wQTdQxWuXQcB6+HwAm5x1CzW7mf0= -github.com/jackc/pgtype v1.9.1/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= +github.com/jackc/pgtype v1.14.0 h1:y+xUdabmyMkJLyApYuPj38mW+aAIqCe5uuBB51rH3Vw= +github.com/jackc/pgtype v1.14.0/go.mod h1:LUMuVrfsFfdKGLw+AFFVv6KtHOFMwRgDDzBt76IqCA4= github.com/jackc/pgx/v4 v4.0.0-20190420224344-cc3461e65d96/go.mod h1:mdxmSJJuR08CZQyj1PVQBHy9XOp5p8/SHH6a0psbY9Y= github.com/jackc/pgx/v4 v4.0.0-20190421002000-1b8f0016e912/go.mod h1:no/Y67Jkk/9WuGR0JG/JseM9irFbnEPbuWV2EELPNuM= github.com/jackc/pgx/v4 v4.0.0-pre1.0.20190824185557-6972a5742186/go.mod h1:X+GQnOEnf1dqHGpw7JmHqHc1NxDoalibchSk9/RWuDc= github.com/jackc/pgx/v4 v4.12.1-0.20210724153913-640aa07df17c/go.mod h1:1QD0+tgSXP7iUjYm9C1NxKhny7lq6ee99u/z+IHFcgs= -github.com/jackc/pgx/v4 v4.14.1 h1:71oo1KAGI6mXhLiTMn6iDFcp3e7+zon/capWjl2OEFU= -github.com/jackc/pgx/v4 v4.14.1/go.mod h1:RgDuE4Z34o7XE92RpLsvFiOEfrAUT0Xt2KxvX73W06M= +github.com/jackc/pgx/v4 v4.18.2 h1:xVpYkNR5pk5bMCZGfClbO962UIqVABcAGt7ha1s/FeU= +github.com/jackc/pgx/v4 v4.18.2/go.mod h1:Ey4Oru5tH5sB6tV7hDmfWFahwF15Eb7DNXlRKx2CkVw= github.com/jackc/puddle v0.0.0-20190413234325-e4ced69a3a2b/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= -github.com/jackc/puddle v1.2.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jarcoal/httpmock v1.1.0 h1:F47ChZj1Y2zFsCXxNkBPwNNKnAyOATcdQibk0qEdVCE= github.com/jarcoal/httpmock v1.1.0/go.mod h1:ATjnClrvW/3tijVmpL/va5Z3aAyGvqU3gCT8nX0Txik= github.com/jhump/protoreflect v1.6.0 h1:h5jfMVslIg6l29nsMs0D8Wj17RDVdNYti0vDN/PZZoE= @@ -757,8 +757,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg= +golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -791,8 +791,8 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -841,8 +841,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= +golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -850,8 +850,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= +golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From df13f4315620ab601a16a077936b0185c3a584d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 26 Mar 2024 09:13:16 +0100 Subject: [PATCH 09/22] Bump github.com/docker/docker (#2913) Bumps [github.com/docker/docker](https://github.com/docker/docker) from 24.0.7+incompatible to 24.0.9+incompatible. - [Release notes](https://github.com/docker/docker/releases) - [Commits](https://github.com/docker/docker/compare/v24.0.7...v24.0.9) --- updated-dependencies: - dependency-name: github.com/docker/docker dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 604fef762..50b88f7d4 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/crowdsecurity/machineid v1.0.2 github.com/davecgh/go-spew v1.1.1 github.com/dghubble/sling v1.3.0 - github.com/docker/docker v24.0.7+incompatible + github.com/docker/docker v24.0.9+incompatible github.com/docker/go-connections v0.4.0 github.com/fatih/color v1.15.0 github.com/fsnotify/fsnotify v1.6.0 diff --git a/go.sum b/go.sum index 17a83cd66..8f91bd31f 100644 --- a/go.sum +++ b/go.sum @@ -116,8 +116,8 @@ github.com/dghubble/sling v1.3.0 h1:pZHjCJq4zJvc6qVQ5wN1jo5oNZlNE0+8T/h0XeXBUKU= github.com/dghubble/sling v1.3.0/go.mod h1:XXShWaBWKzNLhu2OxikSNFrlsvowtz4kyRuXUG7oQKY= github.com/docker/distribution v2.8.2+incompatible h1:T3de5rq0dB1j30rp0sA2rER+m322EBzniBPB6ZIzuh8= github.com/docker/distribution v2.8.2+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= -github.com/docker/docker v24.0.7+incompatible h1:Wo6l37AuwP3JaMnZa226lzVXGA3F9Ig1seQen0cKYlM= -github.com/docker/docker v24.0.7+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v24.0.9+incompatible h1:HPGzNmwfLZWdxHqK9/II92pyi1EpYKsAqcl4G0Of9v0= +github.com/docker/docker v24.0.9+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= From be97466809af04775f82d481371f459498536fd8 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Tue, 26 Mar 2024 09:30:32 +0100 Subject: [PATCH 10/22] CI: use golangci-lint 1.57 (#2916) --- .github/workflows/go-tests-windows.yml | 2 +- .github/workflows/go-tests.yml | 2 +- .golangci.yml | 24 +++++++++++++----------- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/.github/workflows/go-tests-windows.yml b/.github/workflows/go-tests-windows.yml index 643cb9b39..6b2f1132a 100644 --- a/.github/workflows/go-tests-windows.yml +++ b/.github/workflows/go-tests-windows.yml @@ -56,7 +56,7 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v4 with: - version: v1.56 + version: v1.57 args: --issues-exit-code=1 --timeout 10m only-new-issues: false # the cache is already managed above, enabling it here diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index 918cefb26..d76315462 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -157,7 +157,7 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v4 with: - version: v1.56 + version: v1.57 args: --issues-exit-code=1 --timeout 10m only-new-issues: false # the cache is already managed above, enabling it here diff --git a/.golangci.yml b/.golangci.yml index 758327e40..df0cb67d1 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,12 +1,5 @@ # https://github.com/golangci/golangci-lint/blob/master/.golangci.reference.yml -run: - skip-dirs: - - pkg/time/rate - skip-files: - - pkg/yamlpatch/merge.go - - pkg/yamlpatch/merge_test.go - linters-settings: cyclop: # lower this after refactoring @@ -19,6 +12,10 @@ linters-settings: - prefix(github.com/crowdsecurity) - prefix(github.com/crowdsecurity/crowdsec) + gomoddirectives: + replace-allow-list: + - golang.org/x/time/rate + gocognit: # lower this after refactoring min-complexity: 145 @@ -40,7 +37,6 @@ linters-settings: statements: 122 govet: - check-shadowing: true enable: - atomicalign - deepequalerrors @@ -295,15 +291,21 @@ issues: # “Look, that’s why there’s rules, understand? So that you think before you # break ‘em.” ― Terry Pratchett + exclude-dirs: + - pkg/time/rate + + exclude-files: + - pkg/yamlpatch/merge.go + - pkg/yamlpatch/merge_test.go + + exclude-generated-strict: true + max-issues-per-linter: 0 max-same-issues: 0 exclude-rules: # Won't fix: - - path: go.mod - text: "replacement are not allowed: golang.org/x/time/rate" - # `err` is often shadowed, we may continue to do it - linters: - govet From 63bd31b471e765dd85a2d7760483782111e0fb68 Mon Sep 17 00:00:00 2001 From: "Thibault \"bui\" Koechlin" Date: Fri, 29 Mar 2024 17:57:54 +0100 Subject: [PATCH 11/22] Fix REQUEST_URI behavior + fix #2891 (#2917) * fix our behavior to comply more with modsec, REQUEST_URI should be: path+query string * fix #2891 as well * add new transforms * add transform tests --- .../modules/appsec/appsec_others_test.go | 74 +++++++ pkg/acquisition/modules/appsec/appsec_test.go | 200 ++++++++++++++++++ .../modules/appsec/appsec_win_test.go | 46 ++++ pkg/appsec/appsec_rule/modsecurity.go | 13 +- pkg/appsec/request.go | 6 +- 5 files changed, 333 insertions(+), 6 deletions(-) create mode 100644 pkg/acquisition/modules/appsec/appsec_others_test.go create mode 100644 pkg/acquisition/modules/appsec/appsec_win_test.go diff --git a/pkg/acquisition/modules/appsec/appsec_others_test.go b/pkg/acquisition/modules/appsec/appsec_others_test.go new file mode 100644 index 000000000..93edc9d9e --- /dev/null +++ b/pkg/acquisition/modules/appsec/appsec_others_test.go @@ -0,0 +1,74 @@ +//go:build !windows +// +build !windows + +package appsecacquisition + +import ( + "testing" + + "github.com/crowdsecurity/crowdsec/pkg/appsec" + "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" + "github.com/crowdsecurity/crowdsec/pkg/types" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func TestAppsecRuleTransformsOthers(t *testing.T) { + + log.SetLevel(log.TraceLevel) + tests := []appsecRuleTest{ + { + name: "normalizepath", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "equals", Value: "b/c"}, + Transform: []string{"normalizepath"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/?foo=a/../b/c", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "normalizepath #2", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "equals", Value: "b/c/"}, + Transform: []string{"normalizepath"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/?foo=a/../b/c/////././././", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + loadAppSecEngine(test, t) + }) + } +} diff --git a/pkg/acquisition/modules/appsec/appsec_test.go b/pkg/acquisition/modules/appsec/appsec_test.go index 25aea0c78..d98215bf2 100644 --- a/pkg/acquisition/modules/appsec/appsec_test.go +++ b/pkg/acquisition/modules/appsec/appsec_test.go @@ -1284,6 +1284,206 @@ func TestAppsecRuleMatches(t *testing.T) { } } +func TestAppsecRuleTransforms(t *testing.T) { + + log.SetLevel(log.TraceLevel) + tests := []appsecRuleTest{ + { + name: "Basic matching rule", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"URI"}, + Match: appsec_rule.Match{Type: "equals", Value: "/toto"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/toto", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "lowercase", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"URI"}, + Match: appsec_rule.Match{Type: "equals", Value: "/toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/TOTO", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "uppercase", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"URI"}, + Match: appsec_rule.Match{Type: "equals", Value: "/TOTO"}, + Transform: []string{"uppercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/toto", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "b64decode", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "equals", Value: "toto"}, + Transform: []string{"b64decode"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/?foo=dG90bw", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "b64decode with extra padding", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "equals", Value: "toto"}, + Transform: []string{"b64decode"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/?foo=dG90bw===", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "length", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "gte", Value: "3"}, + Transform: []string{"length"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/?foo=toto", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "urldecode", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "equals", Value: "BB/A"}, + Transform: []string{"urldecode"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/?foo=%42%42%2F%41", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "trim", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "equals", Value: "BB/A"}, + Transform: []string{"urldecode", "trim"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/?foo=%20%20%42%42%2F%41%20%20", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + loadAppSecEngine(test, t) + }) + } +} + func loadAppSecEngine(test appsecRuleTest, t *testing.T) { if testing.Verbose() { log.SetLevel(log.TraceLevel) diff --git a/pkg/acquisition/modules/appsec/appsec_win_test.go b/pkg/acquisition/modules/appsec/appsec_win_test.go new file mode 100644 index 000000000..e85d75df2 --- /dev/null +++ b/pkg/acquisition/modules/appsec/appsec_win_test.go @@ -0,0 +1,46 @@ +//go:build windows +// +build windows + +package appsecacquisition + +import ( + "testing" + + log "github.com/sirupsen/logrus" +) + +func TestAppsecRuleTransformsWindows(t *testing.T) { + + log.SetLevel(log.TraceLevel) + tests := []appsecRuleTest{ + // { + // name: "normalizepath", + // expected_load_ok: true, + // inband_rules: []appsec_rule.CustomRule{ + // { + // Name: "rule1", + // Zones: []string{"ARGS"}, + // Variables: []string{"foo"}, + // Match: appsec_rule.Match{Type: "equals", Value: "b/c"}, + // Transform: []string{"normalizepath"}, + // }, + // }, + // input_request: appsec.ParsedRequest{ + // RemoteAddr: "1.2.3.4", + // Method: "GET", + // URI: "/?foo=a/../b/c", + // }, + // output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + // require.Len(t, events, 2) + // require.Equal(t, types.APPSEC, events[0].Type) + // require.Equal(t, types.LOG, events[1].Type) + // require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + // }, + // }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + loadAppSecEngine(test, t) + }) + } +} diff --git a/pkg/appsec/appsec_rule/modsecurity.go b/pkg/appsec/appsec_rule/modsecurity.go index 0b117cd77..a269384cc 100644 --- a/pkg/appsec/appsec_rule/modsecurity.go +++ b/pkg/appsec/appsec_rule/modsecurity.go @@ -19,7 +19,8 @@ var zonesMap map[string]string = map[string]string{ "HEADERS": "REQUEST_HEADERS", "METHOD": "REQUEST_METHOD", "PROTOCOL": "REQUEST_PROTOCOL", - "URI": "REQUEST_URI", + "URI": "REQUEST_FILENAME", + "URI_FULL": "REQUEST_URI", "RAW_BODY": "REQUEST_BODY", "FILENAMES": "FILES", } @@ -28,8 +29,14 @@ var transformMap map[string]string = map[string]string{ "lowercase": "t:lowercase", "uppercase": "t:uppercase", "b64decode": "t:base64Decode", - "hexdecode": "t:hexDecode", - "length": "t:length", + //"hexdecode": "t:hexDecode", -> not supported by coraza + "length": "t:length", + "urldecode": "t:urlDecode", + "trim": "t:trim", + "normalize_path": "t:normalizePath", + "normalizepath": "t:normalizePath", + "htmlentitydecode": "t:htmlEntityDecode", + "html_entity_decode": "t:htmlEntityDecode", } var matchMap map[string]string = map[string]string{ diff --git a/pkg/appsec/request.go b/pkg/appsec/request.go index a9eb0d372..66b5d797f 100644 --- a/pkg/appsec/request.go +++ b/pkg/appsec/request.go @@ -365,11 +365,11 @@ func NewParsedRequestFromRequest(r *http.Request, logger *logrus.Entry) (ParsedR UUID: uuid.New().String(), ClientHost: clientHost, ClientIP: clientIP, - URI: parsedURL.Path, + URI: clientURI, Method: clientMethod, - Host: r.Host, + Host: clientHost, Headers: r.Header, - URL: r.URL, + URL: parsedURL, Proto: r.Proto, Body: body, Args: ParseQuery(parsedURL.RawQuery), From 26bcd0912aa432dbf525359ab0fd8426fd24745a Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Wed, 3 Apr 2024 13:34:35 +0200 Subject: [PATCH 12/22] docker: distribute geoip db in slim image (#2920) --- Dockerfile | 11 ++--------- docker/test/default.env | 2 +- docker/test/tests/test_flavors.py | 2 +- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/Dockerfile b/Dockerfile index ed1ac5e28..53a6cd045 100644 --- a/Dockerfile +++ b/Dockerfile @@ -43,11 +43,12 @@ COPY --from=build /go/bin/yq /usr/local/bin/crowdsec /usr/local/bin/cscli /usr/l COPY --from=build /etc/crowdsec /staging/etc/crowdsec COPY --from=build /go/src/crowdsec/docker/docker_start.sh / COPY --from=build /go/src/crowdsec/docker/config.yaml /staging/etc/crowdsec/config.yaml +COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec RUN yq -n '.url="http://0.0.0.0:8080"' | install -m 0600 /dev/stdin /staging/etc/crowdsec/local_api_credentials.yaml ENTRYPOINT /bin/bash /docker_start.sh -FROM slim as plugins +FROM slim as full # Due to the wizard using cp -n, we have to copy the config files directly from the source as -n does not exist in busybox cp # The files are here for reference, as users will need to mount a new version to be actually able to use notifications @@ -60,11 +61,3 @@ COPY --from=build \ /staging/etc/crowdsec/notifications/ COPY --from=build /usr/local/lib/crowdsec/plugins /usr/local/lib/crowdsec/plugins - -FROM slim as geoip - -COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec - -FROM plugins as full - -COPY --from=build /var/lib/crowdsec /staging/var/lib/crowdsec diff --git a/docker/test/default.env b/docker/test/default.env index c46fdab7f..9607c8aaa 100644 --- a/docker/test/default.env +++ b/docker/test/default.env @@ -6,7 +6,7 @@ CROWDSEC_TEST_VERSION="dev" # All of the following flavors will be tested when using the "flavor" fixture CROWDSEC_TEST_FLAVORS="full" # CROWDSEC_TEST_FLAVORS="full,slim,debian" -# CROWDSEC_TEST_FLAVORS="full,slim,debian,geoip,plugins-debian-slim,debian-geoip,debian-plugins" +# CROWDSEC_TEST_FLAVORS="full,slim,debian,debian-slim" # network to use CROWDSEC_TEST_NETWORK="net-test" diff --git a/docker/test/tests/test_flavors.py b/docker/test/tests/test_flavors.py index 223cf995c..7e78b8d68 100644 --- a/docker/test/tests/test_flavors.py +++ b/docker/test/tests/test_flavors.py @@ -42,7 +42,7 @@ def test_flavor_content(crowdsec, flavor): x = cs.cont.exec_run( 'ls -1 /usr/local/lib/crowdsec/plugins/') stdout = x.output.decode() - if 'slim' in flavor or 'geoip' in flavor: + if 'slim' in flavor: # the exact return code and full message depend # on the 'ls' implementation (busybox vs coreutils) assert x.exit_code != 0 From 912c4bca707cbe9434f5ee62f1d1b92d34395ccd Mon Sep 17 00:00:00 2001 From: "Thibault \"bui\" Koechlin" Date: Wed, 3 Apr 2024 17:49:05 +0200 Subject: [PATCH 13/22] split & reorganize tests a bit. Add tests on existing zones (#2925) --- .../modules/appsec/appsec_hooks_test.go | 714 ++++++++ ...psec_others_test.go => appsec_lnx_test.go} | 0 .../modules/appsec/appsec_remediation_test.go | 320 ++++ .../modules/appsec/appsec_rules_test.go | 733 +++++++++ pkg/acquisition/modules/appsec/appsec_test.go | 1454 ----------------- 5 files changed, 1767 insertions(+), 1454 deletions(-) create mode 100644 pkg/acquisition/modules/appsec/appsec_hooks_test.go rename pkg/acquisition/modules/appsec/{appsec_others_test.go => appsec_lnx_test.go} (100%) create mode 100644 pkg/acquisition/modules/appsec/appsec_remediation_test.go create mode 100644 pkg/acquisition/modules/appsec/appsec_rules_test.go diff --git a/pkg/acquisition/modules/appsec/appsec_hooks_test.go b/pkg/acquisition/modules/appsec/appsec_hooks_test.go new file mode 100644 index 000000000..3cb2fcfde --- /dev/null +++ b/pkg/acquisition/modules/appsec/appsec_hooks_test.go @@ -0,0 +1,714 @@ +package appsecacquisition + +import ( + "net/http" + "net/url" + "testing" + + "github.com/crowdsecurity/crowdsec/pkg/appsec" + "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/davecgh/go-spew/spew" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func TestAppsecOnMatchHooks(t *testing.T) { + tests := []appsecRuleTest{ + { + name: "no rule : check return code", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Len(t, responses, 1) + require.Equal(t, 403, responses[0].BouncerHTTPResponseCode) + require.Equal(t, 403, responses[0].UserHTTPResponseCode) + require.Equal(t, appsec.BanRemediation, responses[0].Action) + + }, + }, + { + name: "on_match: change return code", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + on_match: []appsec.Hook{ + {Filter: "IsInBand == true", Apply: []string{"SetReturnCode(413)"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Len(t, responses, 1) + require.Equal(t, 403, responses[0].BouncerHTTPResponseCode) + require.Equal(t, 413, responses[0].UserHTTPResponseCode) + require.Equal(t, appsec.BanRemediation, responses[0].Action) + }, + }, + { + name: "on_match: change action to a non standard one (log)", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + on_match: []appsec.Hook{ + {Filter: "IsInBand == true", Apply: []string{"SetRemediation('log')"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Len(t, responses, 1) + require.Equal(t, "log", responses[0].Action) + require.Equal(t, 403, responses[0].BouncerHTTPResponseCode) + require.Equal(t, 403, responses[0].UserHTTPResponseCode) + }, + }, + { + name: "on_match: change action to another standard one (allow)", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + on_match: []appsec.Hook{ + {Filter: "IsInBand == true", Apply: []string{"SetRemediation('allow')"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Len(t, responses, 1) + require.Equal(t, appsec.AllowRemediation, responses[0].Action) + }, + }, + { + name: "on_match: change action to another standard one (ban)", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + on_match: []appsec.Hook{ + {Filter: "IsInBand == true", Apply: []string{"SetRemediation('ban')"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, responses, 1) + //note: SetAction normalizes deny, ban and block to ban + require.Equal(t, appsec.BanRemediation, responses[0].Action) + }, + }, + { + name: "on_match: change action to another standard one (captcha)", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + on_match: []appsec.Hook{ + {Filter: "IsInBand == true", Apply: []string{"SetRemediation('captcha')"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, responses, 1) + //note: SetAction normalizes deny, ban and block to ban + require.Equal(t, appsec.CaptchaRemediation, responses[0].Action) + }, + }, + { + name: "on_match: change action to a non standard one", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + on_match: []appsec.Hook{ + {Filter: "IsInBand == true", Apply: []string{"SetRemediation('foobar')"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Len(t, responses, 1) + require.Equal(t, "foobar", responses[0].Action) + }, + }, + { + name: "on_match: cancel alert", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule42", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + on_match: []appsec.Hook{ + {Filter: "IsInBand == true && LogInfo('XX -> %s', evt.Appsec.MatchedRules.GetName())", Apply: []string{"CancelAlert()"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 1) + require.Equal(t, types.LOG, events[0].Type) + require.Len(t, responses, 1) + require.Equal(t, appsec.BanRemediation, responses[0].Action) + }, + }, + { + name: "on_match: cancel event", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule42", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + on_match: []appsec.Hook{ + {Filter: "IsInBand == true", Apply: []string{"CancelEvent()"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 1) + require.Equal(t, types.APPSEC, events[0].Type) + require.Len(t, responses, 1) + require.Equal(t, appsec.BanRemediation, responses[0].Action) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + loadAppSecEngine(test, t) + }) + } +} + +func TestAppsecPreEvalHooks(t *testing.T) { + + tests := []appsecRuleTest{ + { + name: "Basic on_load hook to disable inband rule", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + pre_eval: []appsec.Hook{ + {Filter: "1 == 1", Apply: []string{"RemoveInBandRuleByName('rule1')"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Empty(t, events) + require.Len(t, responses, 1) + require.False(t, responses[0].InBandInterrupt) + require.False(t, responses[0].OutOfBandInterrupt) + }, + }, + { + name: "Basic on_load fails to disable rule", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + pre_eval: []appsec.Hook{ + {Filter: "1 ==2", Apply: []string{"RemoveInBandRuleByName('rule1')"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + + require.Equal(t, types.LOG, events[1].Type) + require.True(t, events[1].Appsec.HasInBandMatches) + require.Len(t, events[1].Appsec.MatchedRules, 1) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + + require.Len(t, responses, 1) + require.True(t, responses[0].InBandInterrupt) + + }, + }, + { + name: "on_load : disable inband by tag", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rulez", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + pre_eval: []appsec.Hook{ + {Apply: []string{"RemoveInBandRuleByTag('crowdsec-rulez')"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Empty(t, events) + require.Len(t, responses, 1) + require.False(t, responses[0].InBandInterrupt) + require.False(t, responses[0].OutOfBandInterrupt) + }, + }, + { + name: "on_load : disable inband by ID", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rulez", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + pre_eval: []appsec.Hook{ + {Apply: []string{"RemoveInBandRuleByID(1516470898)"}}, //rule ID is generated at runtime. If you change rule, it will break the test (: + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Empty(t, events) + require.Len(t, responses, 1) + require.False(t, responses[0].InBandInterrupt) + require.False(t, responses[0].OutOfBandInterrupt) + }, + }, + { + name: "on_load : disable inband by name", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rulez", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + pre_eval: []appsec.Hook{ + {Apply: []string{"RemoveInBandRuleByName('rulez')"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Empty(t, events) + require.Len(t, responses, 1) + require.False(t, responses[0].InBandInterrupt) + require.False(t, responses[0].OutOfBandInterrupt) + }, + }, + { + name: "on_load : outofband default behavior", + expected_load_ok: true, + outofband_rules: []appsec_rule.CustomRule{ + { + Name: "rulez", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 1) + require.Equal(t, types.LOG, events[0].Type) + require.True(t, events[0].Appsec.HasOutBandMatches) + require.False(t, events[0].Appsec.HasInBandMatches) + require.Len(t, events[0].Appsec.MatchedRules, 1) + require.Equal(t, "rulez", events[0].Appsec.MatchedRules[0]["msg"]) + //maybe surprising, but response won't mention OOB event, as it's sent as soon as the inband phase is over. + require.Len(t, responses, 1) + require.False(t, responses[0].InBandInterrupt) + require.False(t, responses[0].OutOfBandInterrupt) + }, + }, + { + name: "on_load : set remediation by tag", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rulez", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + pre_eval: []appsec.Hook{ + {Apply: []string{"SetRemediationByTag('crowdsec-rulez', 'foobar')"}}, //rule ID is generated at runtime. If you change rule, it will break the test (: + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Len(t, responses, 1) + require.Equal(t, "foobar", responses[0].Action) + }, + }, + { + name: "on_load : set remediation by name", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rulez", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + pre_eval: []appsec.Hook{ + {Apply: []string{"SetRemediationByName('rulez', 'foobar')"}}, //rule ID is generated at runtime. If you change rule, it will break the test (: + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Len(t, responses, 1) + require.Equal(t, "foobar", responses[0].Action) + }, + }, + { + name: "on_load : set remediation by ID", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rulez", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + pre_eval: []appsec.Hook{ + {Apply: []string{"SetRemediationByID(1516470898, 'foobar')"}}, //rule ID is generated at runtime. If you change rule, it will break the test (: + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Len(t, responses, 1) + require.Equal(t, "foobar", responses[0].Action) + require.Equal(t, "foobar", appsecResponse.Action) + require.Equal(t, http.StatusForbidden, appsecResponse.HTTPStatus) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + loadAppSecEngine(test, t) + }) + } +} + +func TestAppsecRemediationConfigHooks(t *testing.T) { + + tests := []appsecRuleTest{ + { + name: "Basic matching rule", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Equal(t, appsec.BanRemediation, responses[0].Action) + require.Equal(t, http.StatusForbidden, statusCode) + require.Equal(t, appsec.BanRemediation, appsecResponse.Action) + require.Equal(t, http.StatusForbidden, appsecResponse.HTTPStatus) + }, + }, + { + name: "SetRemediation", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + on_match: []appsec.Hook{{Apply: []string{"SetRemediation('captcha')"}}}, //rule ID is generated at runtime. If you change rule, it will break the test (: + + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Equal(t, appsec.CaptchaRemediation, responses[0].Action) + require.Equal(t, http.StatusForbidden, statusCode) + require.Equal(t, appsec.CaptchaRemediation, appsecResponse.Action) + require.Equal(t, http.StatusForbidden, appsecResponse.HTTPStatus) + }, + }, + { + name: "SetRemediation", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + on_match: []appsec.Hook{{Apply: []string{"SetReturnCode(418)"}}}, //rule ID is generated at runtime. If you change rule, it will break the test (: + + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Equal(t, appsec.BanRemediation, responses[0].Action) + require.Equal(t, http.StatusForbidden, statusCode) + require.Equal(t, appsec.BanRemediation, appsecResponse.Action) + require.Equal(t, http.StatusTeapot, appsecResponse.HTTPStatus) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + loadAppSecEngine(test, t) + }) + } +} +func TestOnMatchRemediationHooks(t *testing.T) { + tests := []appsecRuleTest{ + { + name: "set remediation to allow with on_match hook", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule42", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + on_match: []appsec.Hook{ + {Filter: "IsInBand == true", Apply: []string{"SetRemediation('allow')"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) + require.Equal(t, http.StatusOK, appsecResponse.HTTPStatus) + }, + }, + { + name: "set remediation to captcha + custom user code with on_match hook", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule42", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + DefaultRemediation: appsec.AllowRemediation, + on_match: []appsec.Hook{ + {Filter: "IsInBand == true", Apply: []string{"SetRemediation('captcha')", "SetReturnCode(418)"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + spew.Dump(responses) + spew.Dump(appsecResponse) + + log.Errorf("http status : %d", statusCode) + require.Equal(t, appsec.CaptchaRemediation, appsecResponse.Action) + require.Equal(t, http.StatusTeapot, appsecResponse.HTTPStatus) + require.Equal(t, http.StatusForbidden, statusCode) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + loadAppSecEngine(test, t) + }) + } +} diff --git a/pkg/acquisition/modules/appsec/appsec_others_test.go b/pkg/acquisition/modules/appsec/appsec_lnx_test.go similarity index 100% rename from pkg/acquisition/modules/appsec/appsec_others_test.go rename to pkg/acquisition/modules/appsec/appsec_lnx_test.go diff --git a/pkg/acquisition/modules/appsec/appsec_remediation_test.go b/pkg/acquisition/modules/appsec/appsec_remediation_test.go new file mode 100644 index 000000000..a7f117389 --- /dev/null +++ b/pkg/acquisition/modules/appsec/appsec_remediation_test.go @@ -0,0 +1,320 @@ +package appsecacquisition + +import ( + "net/http" + "net/url" + "testing" + + "github.com/crowdsecurity/crowdsec/pkg/appsec" + "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" + "github.com/crowdsecurity/crowdsec/pkg/types" + "github.com/stretchr/testify/require" +) + +func TestAppsecDefaultPassRemediation(t *testing.T) { + + tests := []appsecRuleTest{ + { + name: "Basic non-matching rule", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/", + Args: url.Values{"foo": []string{"tutu"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Equal(t, appsec.AllowRemediation, responses[0].Action) + require.Equal(t, http.StatusOK, statusCode) + require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) + require.Equal(t, http.StatusOK, appsecResponse.HTTPStatus) + }, + }, + { + name: "DefaultPassAction: pass", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/", + Args: url.Values{"foo": []string{"tutu"}}, + }, + DefaultPassAction: "allow", + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Equal(t, appsec.AllowRemediation, responses[0].Action) + require.Equal(t, http.StatusOK, statusCode) + require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) + require.Equal(t, http.StatusOK, appsecResponse.HTTPStatus) + }, + }, + { + name: "DefaultPassAction: captcha", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/", + Args: url.Values{"foo": []string{"tutu"}}, + }, + DefaultPassAction: "captcha", + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Equal(t, appsec.CaptchaRemediation, responses[0].Action) + require.Equal(t, http.StatusOK, statusCode) //@tko: body is captcha, but as it's 200, captcha won't be showed to user + require.Equal(t, appsec.CaptchaRemediation, appsecResponse.Action) + require.Equal(t, http.StatusOK, appsecResponse.HTTPStatus) + }, + }, + { + name: "DefaultPassHTTPCode: 200", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/", + Args: url.Values{"foo": []string{"tutu"}}, + }, + UserPassedHTTPCode: 200, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Equal(t, appsec.AllowRemediation, responses[0].Action) + require.Equal(t, http.StatusOK, statusCode) + require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) + require.Equal(t, http.StatusOK, appsecResponse.HTTPStatus) + }, + }, + { + name: "DefaultPassHTTPCode: 200", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/", + Args: url.Values{"foo": []string{"tutu"}}, + }, + UserPassedHTTPCode: 418, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Equal(t, appsec.AllowRemediation, responses[0].Action) + require.Equal(t, http.StatusOK, statusCode) + require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) + require.Equal(t, http.StatusTeapot, appsecResponse.HTTPStatus) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + loadAppSecEngine(test, t) + }) + } +} + +func TestAppsecDefaultRemediation(t *testing.T) { + + tests := []appsecRuleTest{ + { + name: "Basic matching rule", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Equal(t, appsec.BanRemediation, responses[0].Action) + require.Equal(t, http.StatusForbidden, statusCode) + require.Equal(t, appsec.BanRemediation, appsecResponse.Action) + require.Equal(t, http.StatusForbidden, appsecResponse.HTTPStatus) + }, + }, + { + name: "default remediation to ban (default)", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule42", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + DefaultRemediation: "ban", + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Equal(t, appsec.BanRemediation, responses[0].Action) + require.Equal(t, http.StatusForbidden, statusCode) + require.Equal(t, appsec.BanRemediation, appsecResponse.Action) + require.Equal(t, http.StatusForbidden, appsecResponse.HTTPStatus) + }, + }, + { + name: "default remediation to allow", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule42", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + DefaultRemediation: "allow", + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Equal(t, appsec.AllowRemediation, responses[0].Action) + require.Equal(t, http.StatusOK, statusCode) + require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) + require.Equal(t, http.StatusOK, appsecResponse.HTTPStatus) + }, + }, + { + name: "default remediation to captcha", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule42", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + DefaultRemediation: "captcha", + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Equal(t, appsec.CaptchaRemediation, responses[0].Action) + require.Equal(t, http.StatusForbidden, statusCode) + require.Equal(t, appsec.CaptchaRemediation, appsecResponse.Action) + require.Equal(t, http.StatusForbidden, appsecResponse.HTTPStatus) + }, + }, + { + name: "custom user HTTP code", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule42", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + UserBlockedHTTPCode: 418, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Equal(t, appsec.BanRemediation, responses[0].Action) + require.Equal(t, http.StatusForbidden, statusCode) + require.Equal(t, appsec.BanRemediation, appsecResponse.Action) + require.Equal(t, http.StatusTeapot, appsecResponse.HTTPStatus) + }, + }, + { + name: "custom remediation + HTTP code", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule42", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + UserBlockedHTTPCode: 418, + DefaultRemediation: "foobar", + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Equal(t, "foobar", responses[0].Action) + require.Equal(t, http.StatusForbidden, statusCode) + require.Equal(t, "foobar", appsecResponse.Action) + require.Equal(t, http.StatusTeapot, appsecResponse.HTTPStatus) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + loadAppSecEngine(test, t) + }) + } +} diff --git a/pkg/acquisition/modules/appsec/appsec_rules_test.go b/pkg/acquisition/modules/appsec/appsec_rules_test.go new file mode 100644 index 000000000..3c48c50fa --- /dev/null +++ b/pkg/acquisition/modules/appsec/appsec_rules_test.go @@ -0,0 +1,733 @@ +package appsecacquisition + +import ( + "net/http" + "net/url" + "testing" + + "github.com/crowdsecurity/crowdsec/pkg/appsec" + "github.com/crowdsecurity/crowdsec/pkg/appsec/appsec_rule" + "github.com/crowdsecurity/crowdsec/pkg/types" + log "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func TestAppsecRuleMatches(t *testing.T) { + + tests := []appsecRuleTest{ + { + name: "Basic matching rule", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + + require.Equal(t, types.LOG, events[1].Type) + require.True(t, events[1].Appsec.HasInBandMatches) + require.Len(t, events[1].Appsec.MatchedRules, 1) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + + require.Len(t, responses, 1) + require.True(t, responses[0].InBandInterrupt) + }, + }, + { + name: "Basic non-matching rule", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"tutu"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Empty(t, events) + require.Len(t, responses, 1) + require.False(t, responses[0].InBandInterrupt) + require.False(t, responses[0].OutOfBandInterrupt) + }, + }, + { + name: "default remediation to allow", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule42", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + DefaultRemediation: "allow", + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Equal(t, appsec.AllowRemediation, responses[0].Action) + require.Equal(t, http.StatusOK, statusCode) + require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) + require.Equal(t, http.StatusOK, appsecResponse.HTTPStatus) + }, + }, + { + name: "default remediation to captcha", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule42", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + DefaultRemediation: "captcha", + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Equal(t, appsec.CaptchaRemediation, responses[0].Action) + require.Equal(t, http.StatusForbidden, statusCode) + require.Equal(t, appsec.CaptchaRemediation, appsecResponse.Action) + require.Equal(t, http.StatusForbidden, appsecResponse.HTTPStatus) + }, + }, + { + name: "no default remediation / custom user HTTP code", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule42", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"toto"}}, + }, + UserBlockedHTTPCode: 418, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Equal(t, appsec.BanRemediation, responses[0].Action) + require.Equal(t, http.StatusForbidden, statusCode) + require.Equal(t, appsec.BanRemediation, appsecResponse.Action) + require.Equal(t, http.StatusTeapot, appsecResponse.HTTPStatus) + }, + }, + { + name: "no match but try to set remediation to captcha with on_match hook", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule42", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + on_match: []appsec.Hook{ + {Filter: "IsInBand == true", Apply: []string{"SetRemediation('captcha')"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"bla"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Empty(t, events) + require.Equal(t, http.StatusOK, statusCode) + require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) + }, + }, + { + name: "no match but try to set user HTTP code with on_match hook", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule42", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + on_match: []appsec.Hook{ + {Filter: "IsInBand == true", Apply: []string{"SetReturnCode(418)"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"bla"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Empty(t, events) + require.Equal(t, http.StatusOK, statusCode) + require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) + }, + }, + { + name: "no match but try to set remediation with pre_eval hook", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule42", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, + Transform: []string{"lowercase"}, + }, + }, + pre_eval: []appsec.Hook{ + {Filter: "IsInBand == true", Apply: []string{"SetRemediationByName('rule42', 'captcha')"}}, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/urllll", + Args: url.Values{"foo": []string{"bla"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Empty(t, events) + require.Equal(t, http.StatusOK, statusCode) + require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + loadAppSecEngine(test, t) + }) + } +} + +func TestAppsecRuleTransforms(t *testing.T) { + + log.SetLevel(log.TraceLevel) + tests := []appsecRuleTest{ + { + name: "Basic matching rule", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"URI"}, + Match: appsec_rule.Match{Type: "equals", Value: "/toto"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/toto", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "lowercase", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"URI"}, + Match: appsec_rule.Match{Type: "equals", Value: "/toto"}, + Transform: []string{"lowercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/TOTO", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "uppercase", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"URI"}, + Match: appsec_rule.Match{Type: "equals", Value: "/TOTO"}, + Transform: []string{"uppercase"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/toto", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "b64decode", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "equals", Value: "toto"}, + Transform: []string{"b64decode"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/?foo=dG90bw", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "b64decode with extra padding", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "equals", Value: "toto"}, + Transform: []string{"b64decode"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/?foo=dG90bw===", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "length", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "gte", Value: "3"}, + Transform: []string{"length"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/?foo=toto", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "urldecode", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "equals", Value: "BB/A"}, + Transform: []string{"urldecode"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/?foo=%42%42%2F%41", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "trim", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Variables: []string{"foo"}, + Match: appsec_rule.Match{Type: "equals", Value: "BB/A"}, + Transform: []string{"urldecode", "trim"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/?foo=%20%20%42%42%2F%41%20%20", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + loadAppSecEngine(test, t) + }) + } +} + +func TestAppsecRuleZones(t *testing.T) { + + log.SetLevel(log.TraceLevel) + tests := []appsecRuleTest{ + { + name: "rule: ARGS", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS"}, + Match: appsec_rule.Match{Type: "equals", Value: "toto"}, + }, + { + Name: "rule2", + Zones: []string{"ARGS"}, + Match: appsec_rule.Match{Type: "equals", Value: "foobar"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/foobar?something=toto&foobar=smth", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "rule: ARGS_NAMES", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"ARGS_NAMES"}, + Match: appsec_rule.Match{Type: "equals", Value: "toto"}, + }, + { + Name: "rule2", + Zones: []string{"ARGS_NAMES"}, + Match: appsec_rule.Match{Type: "equals", Value: "foobar"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/foobar?something=toto&foobar=smth", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule2", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "rule: BODY_ARGS", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"BODY_ARGS"}, + Match: appsec_rule.Match{Type: "equals", Value: "toto"}, + }, + { + Name: "rule2", + Zones: []string{"BODY_ARGS"}, + Match: appsec_rule.Match{Type: "equals", Value: "foobar"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/", + Body: []byte("smth=toto&foobar=other"), + Headers: http.Header{"Content-Type": []string{"application/x-www-form-urlencoded"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "rule: BODY_ARGS_NAMES", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"BODY_ARGS_NAMES"}, + Match: appsec_rule.Match{Type: "equals", Value: "toto"}, + }, + { + Name: "rule2", + Zones: []string{"BODY_ARGS_NAMES"}, + Match: appsec_rule.Match{Type: "equals", Value: "foobar"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/", + Body: []byte("smth=toto&foobar=other"), + Headers: http.Header{"Content-Type": []string{"application/x-www-form-urlencoded"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule2", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "rule: HEADERS", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"HEADERS"}, + Match: appsec_rule.Match{Type: "equals", Value: "toto"}, + }, + { + Name: "rule2", + Zones: []string{"HEADERS"}, + Match: appsec_rule.Match{Type: "equals", Value: "foobar"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/", + Headers: http.Header{"foobar": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "rule: HEADERS_NAMES", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"HEADERS_NAMES"}, + Match: appsec_rule.Match{Type: "equals", Value: "toto"}, + }, + { + Name: "rule2", + Zones: []string{"HEADERS_NAMES"}, + Match: appsec_rule.Match{Type: "equals", Value: "foobar"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/", + Headers: http.Header{"foobar": []string{"toto"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule2", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "rule: METHOD", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"METHOD"}, + Match: appsec_rule.Match{Type: "equals", Value: "GET"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "rule: PROTOCOL", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"PROTOCOL"}, + Match: appsec_rule.Match{Type: "contains", Value: "3.1"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/", + Proto: "HTTP/3.1", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "rule: URI", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"URI"}, + Match: appsec_rule.Match{Type: "equals", Value: "/foobar"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/foobar", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "rule: URI_FULL", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"URI_FULL"}, + Match: appsec_rule.Match{Type: "equals", Value: "/foobar?a=b"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/foobar?a=b", + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + { + name: "rule: RAW_BODY", + expected_load_ok: true, + inband_rules: []appsec_rule.CustomRule{ + { + Name: "rule1", + Zones: []string{"RAW_BODY"}, + Match: appsec_rule.Match{Type: "equals", Value: "foobar=42421"}, + }, + }, + input_request: appsec.ParsedRequest{ + RemoteAddr: "1.2.3.4", + Method: "GET", + URI: "/", + Body: []byte("foobar=42421"), + Headers: http.Header{"Content-Type": []string{"application/x-www-form-urlencoded"}}, + }, + output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { + require.Len(t, events, 2) + require.Equal(t, types.APPSEC, events[0].Type) + require.Equal(t, types.LOG, events[1].Type) + require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + loadAppSecEngine(test, t) + }) + } +} diff --git a/pkg/acquisition/modules/appsec/appsec_test.go b/pkg/acquisition/modules/appsec/appsec_test.go index d98215bf2..5fe4cfe23 100644 --- a/pkg/acquisition/modules/appsec/appsec_test.go +++ b/pkg/acquisition/modules/appsec/appsec_test.go @@ -1,8 +1,6 @@ package appsecacquisition import ( - "net/http" - "net/url" "testing" "time" @@ -12,15 +10,8 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/google/uuid" log "github.com/sirupsen/logrus" - "github.com/stretchr/testify/require" ) -/* -Missing tests (wip): - - GenerateResponse - - evt.Appsec and it's subobjects and methods -*/ - type appsecRuleTest struct { name string expected_load_ok bool @@ -39,1451 +30,6 @@ type appsecRuleTest struct { output_asserts func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) } -func TestAppsecOnMatchHooks(t *testing.T) { - tests := []appsecRuleTest{ - { - name: "no rule : check return code", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 2) - require.Equal(t, types.APPSEC, events[0].Type) - require.Equal(t, types.LOG, events[1].Type) - require.Len(t, responses, 1) - require.Equal(t, 403, responses[0].BouncerHTTPResponseCode) - require.Equal(t, 403, responses[0].UserHTTPResponseCode) - require.Equal(t, appsec.BanRemediation, responses[0].Action) - - }, - }, - { - name: "on_match: change return code", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - on_match: []appsec.Hook{ - {Filter: "IsInBand == true", Apply: []string{"SetReturnCode(413)"}}, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 2) - require.Equal(t, types.APPSEC, events[0].Type) - require.Equal(t, types.LOG, events[1].Type) - require.Len(t, responses, 1) - require.Equal(t, 403, responses[0].BouncerHTTPResponseCode) - require.Equal(t, 413, responses[0].UserHTTPResponseCode) - require.Equal(t, appsec.BanRemediation, responses[0].Action) - }, - }, - { - name: "on_match: change action to a non standard one (log)", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - on_match: []appsec.Hook{ - {Filter: "IsInBand == true", Apply: []string{"SetRemediation('log')"}}, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 2) - require.Equal(t, types.APPSEC, events[0].Type) - require.Equal(t, types.LOG, events[1].Type) - require.Len(t, responses, 1) - require.Equal(t, "log", responses[0].Action) - }, - }, - { - name: "on_match: change action to another standard one (allow)", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - on_match: []appsec.Hook{ - {Filter: "IsInBand == true", Apply: []string{"SetRemediation('allow')"}}, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 2) - require.Equal(t, types.APPSEC, events[0].Type) - require.Equal(t, types.LOG, events[1].Type) - require.Len(t, responses, 1) - require.Equal(t, appsec.AllowRemediation, responses[0].Action) - }, - }, - { - name: "on_match: change action to another standard one (ban)", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - on_match: []appsec.Hook{ - {Filter: "IsInBand == true", Apply: []string{"SetRemediation('ban')"}}, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, responses, 1) - //note: SetAction normalizes deny, ban and block to ban - require.Equal(t, appsec.BanRemediation, responses[0].Action) - }, - }, - { - name: "on_match: change action to another standard one (captcha)", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - on_match: []appsec.Hook{ - {Filter: "IsInBand == true", Apply: []string{"SetRemediation('captcha')"}}, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, responses, 1) - //note: SetAction normalizes deny, ban and block to ban - require.Equal(t, appsec.CaptchaRemediation, responses[0].Action) - }, - }, - { - name: "on_match: change action to a non standard one", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - on_match: []appsec.Hook{ - {Filter: "IsInBand == true", Apply: []string{"SetRemediation('foobar')"}}, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 2) - require.Equal(t, types.APPSEC, events[0].Type) - require.Equal(t, types.LOG, events[1].Type) - require.Len(t, responses, 1) - require.Equal(t, "foobar", responses[0].Action) - }, - }, - { - name: "on_match: cancel alert", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule42", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - on_match: []appsec.Hook{ - {Filter: "IsInBand == true && LogInfo('XX -> %s', evt.Appsec.MatchedRules.GetName())", Apply: []string{"CancelAlert()"}}, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 1) - require.Equal(t, types.LOG, events[0].Type) - require.Len(t, responses, 1) - require.Equal(t, appsec.BanRemediation, responses[0].Action) - }, - }, - { - name: "on_match: cancel event", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule42", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - on_match: []appsec.Hook{ - {Filter: "IsInBand == true", Apply: []string{"CancelEvent()"}}, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 1) - require.Equal(t, types.APPSEC, events[0].Type) - require.Len(t, responses, 1) - require.Equal(t, appsec.BanRemediation, responses[0].Action) - }, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - loadAppSecEngine(test, t) - }) - } -} - -func TestAppsecPreEvalHooks(t *testing.T) { - /* - [x] basic working hook - [x] basic failing hook - [ ] test the "OnSuccess" feature - [ ] test multiple competing hooks - [ ] test the variety of helpers - */ - tests := []appsecRuleTest{ - { - name: "Basic on_load hook to disable inband rule", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - pre_eval: []appsec.Hook{ - {Filter: "1 == 1", Apply: []string{"RemoveInBandRuleByName('rule1')"}}, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Empty(t, events) - require.Len(t, responses, 1) - require.False(t, responses[0].InBandInterrupt) - require.False(t, responses[0].OutOfBandInterrupt) - }, - }, - { - name: "Basic on_load fails to disable rule", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - pre_eval: []appsec.Hook{ - {Filter: "1 ==2", Apply: []string{"RemoveInBandRuleByName('rule1')"}}, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 2) - require.Equal(t, types.APPSEC, events[0].Type) - - require.Equal(t, types.LOG, events[1].Type) - require.True(t, events[1].Appsec.HasInBandMatches) - require.Len(t, events[1].Appsec.MatchedRules, 1) - require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) - - require.Len(t, responses, 1) - require.True(t, responses[0].InBandInterrupt) - - }, - }, - { - name: "on_load : disable inband by tag", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rulez", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - pre_eval: []appsec.Hook{ - {Apply: []string{"RemoveInBandRuleByTag('crowdsec-rulez')"}}, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Empty(t, events) - require.Len(t, responses, 1) - require.False(t, responses[0].InBandInterrupt) - require.False(t, responses[0].OutOfBandInterrupt) - }, - }, - { - name: "on_load : disable inband by ID", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rulez", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - pre_eval: []appsec.Hook{ - {Apply: []string{"RemoveInBandRuleByID(1516470898)"}}, //rule ID is generated at runtime. If you change rule, it will break the test (: - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Empty(t, events) - require.Len(t, responses, 1) - require.False(t, responses[0].InBandInterrupt) - require.False(t, responses[0].OutOfBandInterrupt) - }, - }, - { - name: "on_load : disable inband by name", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rulez", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - pre_eval: []appsec.Hook{ - {Apply: []string{"RemoveInBandRuleByName('rulez')"}}, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Empty(t, events) - require.Len(t, responses, 1) - require.False(t, responses[0].InBandInterrupt) - require.False(t, responses[0].OutOfBandInterrupt) - }, - }, - { - name: "on_load : outofband default behavior", - expected_load_ok: true, - outofband_rules: []appsec_rule.CustomRule{ - { - Name: "rulez", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 1) - require.Equal(t, types.LOG, events[0].Type) - require.True(t, events[0].Appsec.HasOutBandMatches) - require.False(t, events[0].Appsec.HasInBandMatches) - require.Len(t, events[0].Appsec.MatchedRules, 1) - require.Equal(t, "rulez", events[0].Appsec.MatchedRules[0]["msg"]) - //maybe surprising, but response won't mention OOB event, as it's sent as soon as the inband phase is over. - require.Len(t, responses, 1) - require.False(t, responses[0].InBandInterrupt) - require.False(t, responses[0].OutOfBandInterrupt) - }, - }, - { - name: "on_load : set remediation by tag", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rulez", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - pre_eval: []appsec.Hook{ - {Apply: []string{"SetRemediationByTag('crowdsec-rulez', 'foobar')"}}, //rule ID is generated at runtime. If you change rule, it will break the test (: - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 2) - require.Len(t, responses, 1) - require.Equal(t, "foobar", responses[0].Action) - }, - }, - { - name: "on_load : set remediation by name", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rulez", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - pre_eval: []appsec.Hook{ - {Apply: []string{"SetRemediationByName('rulez', 'foobar')"}}, //rule ID is generated at runtime. If you change rule, it will break the test (: - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 2) - require.Len(t, responses, 1) - require.Equal(t, "foobar", responses[0].Action) - }, - }, - { - name: "on_load : set remediation by ID", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rulez", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - pre_eval: []appsec.Hook{ - {Apply: []string{"SetRemediationByID(1516470898, 'foobar')"}}, //rule ID is generated at runtime. If you change rule, it will break the test (: - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 2) - require.Len(t, responses, 1) - require.Equal(t, "foobar", responses[0].Action) - require.Equal(t, "foobar", appsecResponse.Action) - require.Equal(t, http.StatusForbidden, appsecResponse.HTTPStatus) - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - loadAppSecEngine(test, t) - }) - } -} - -func TestAppsecRemediationConfigHooks(t *testing.T) { - - tests := []appsecRuleTest{ - { - name: "Basic matching rule", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Equal(t, appsec.BanRemediation, responses[0].Action) - require.Equal(t, http.StatusForbidden, statusCode) - require.Equal(t, appsec.BanRemediation, appsecResponse.Action) - require.Equal(t, http.StatusForbidden, appsecResponse.HTTPStatus) - }, - }, - { - name: "SetRemediation", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - on_match: []appsec.Hook{{Apply: []string{"SetRemediation('captcha')"}}}, //rule ID is generated at runtime. If you change rule, it will break the test (: - - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Equal(t, appsec.CaptchaRemediation, responses[0].Action) - require.Equal(t, http.StatusForbidden, statusCode) - require.Equal(t, appsec.CaptchaRemediation, appsecResponse.Action) - require.Equal(t, http.StatusForbidden, appsecResponse.HTTPStatus) - }, - }, - { - name: "SetRemediation", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - on_match: []appsec.Hook{{Apply: []string{"SetReturnCode(418)"}}}, //rule ID is generated at runtime. If you change rule, it will break the test (: - - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Equal(t, appsec.BanRemediation, responses[0].Action) - require.Equal(t, http.StatusForbidden, statusCode) - require.Equal(t, appsec.BanRemediation, appsecResponse.Action) - require.Equal(t, http.StatusTeapot, appsecResponse.HTTPStatus) - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - loadAppSecEngine(test, t) - }) - } -} -func TestOnMatchRemediationHooks(t *testing.T) { - tests := []appsecRuleTest{ - { - name: "set remediation to allow with on_match hook", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule42", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - on_match: []appsec.Hook{ - {Filter: "IsInBand == true", Apply: []string{"SetRemediation('allow')"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) - require.Equal(t, http.StatusOK, appsecResponse.HTTPStatus) - }, - }, - { - name: "set remediation to captcha + custom user code with on_match hook", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule42", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - DefaultRemediation: appsec.AllowRemediation, - on_match: []appsec.Hook{ - {Filter: "IsInBand == true", Apply: []string{"SetRemediation('captcha')", "SetReturnCode(418)"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - spew.Dump(responses) - spew.Dump(appsecResponse) - - log.Errorf("http status : %d", statusCode) - require.Equal(t, appsec.CaptchaRemediation, appsecResponse.Action) - require.Equal(t, http.StatusTeapot, appsecResponse.HTTPStatus) - require.Equal(t, http.StatusForbidden, statusCode) - }, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - loadAppSecEngine(test, t) - }) - } -} - -func TestAppsecDefaultPassRemediation(t *testing.T) { - - tests := []appsecRuleTest{ - { - name: "Basic non-matching rule", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/", - Args: url.Values{"foo": []string{"tutu"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Equal(t, appsec.AllowRemediation, responses[0].Action) - require.Equal(t, http.StatusOK, statusCode) - require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) - require.Equal(t, http.StatusOK, appsecResponse.HTTPStatus) - }, - }, - { - name: "DefaultPassAction: pass", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/", - Args: url.Values{"foo": []string{"tutu"}}, - }, - DefaultPassAction: "allow", - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Equal(t, appsec.AllowRemediation, responses[0].Action) - require.Equal(t, http.StatusOK, statusCode) - require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) - require.Equal(t, http.StatusOK, appsecResponse.HTTPStatus) - }, - }, - { - name: "DefaultPassAction: captcha", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/", - Args: url.Values{"foo": []string{"tutu"}}, - }, - DefaultPassAction: "captcha", - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Equal(t, appsec.CaptchaRemediation, responses[0].Action) - require.Equal(t, http.StatusOK, statusCode) //@tko: body is captcha, but as it's 200, captcha won't be showed to user - require.Equal(t, appsec.CaptchaRemediation, appsecResponse.Action) - require.Equal(t, http.StatusOK, appsecResponse.HTTPStatus) - }, - }, - { - name: "DefaultPassHTTPCode: 200", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/", - Args: url.Values{"foo": []string{"tutu"}}, - }, - UserPassedHTTPCode: 200, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Equal(t, appsec.AllowRemediation, responses[0].Action) - require.Equal(t, http.StatusOK, statusCode) - require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) - require.Equal(t, http.StatusOK, appsecResponse.HTTPStatus) - }, - }, - { - name: "DefaultPassHTTPCode: 200", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/", - Args: url.Values{"foo": []string{"tutu"}}, - }, - UserPassedHTTPCode: 418, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Equal(t, appsec.AllowRemediation, responses[0].Action) - require.Equal(t, http.StatusOK, statusCode) - require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) - require.Equal(t, http.StatusTeapot, appsecResponse.HTTPStatus) - }, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - loadAppSecEngine(test, t) - }) - } -} - -func TestAppsecDefaultRemediation(t *testing.T) { - - tests := []appsecRuleTest{ - { - name: "Basic matching rule", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Equal(t, appsec.BanRemediation, responses[0].Action) - require.Equal(t, http.StatusForbidden, statusCode) - require.Equal(t, appsec.BanRemediation, appsecResponse.Action) - require.Equal(t, http.StatusForbidden, appsecResponse.HTTPStatus) - }, - }, - { - name: "default remediation to ban (default)", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule42", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - DefaultRemediation: "ban", - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Equal(t, appsec.BanRemediation, responses[0].Action) - require.Equal(t, http.StatusForbidden, statusCode) - require.Equal(t, appsec.BanRemediation, appsecResponse.Action) - require.Equal(t, http.StatusForbidden, appsecResponse.HTTPStatus) - }, - }, - { - name: "default remediation to allow", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule42", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - DefaultRemediation: "allow", - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Equal(t, appsec.AllowRemediation, responses[0].Action) - require.Equal(t, http.StatusOK, statusCode) - require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) - require.Equal(t, http.StatusOK, appsecResponse.HTTPStatus) - }, - }, - { - name: "default remediation to captcha", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule42", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - DefaultRemediation: "captcha", - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Equal(t, appsec.CaptchaRemediation, responses[0].Action) - require.Equal(t, http.StatusForbidden, statusCode) - require.Equal(t, appsec.CaptchaRemediation, appsecResponse.Action) - require.Equal(t, http.StatusForbidden, appsecResponse.HTTPStatus) - }, - }, - { - name: "custom user HTTP code", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule42", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - UserBlockedHTTPCode: 418, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Equal(t, appsec.BanRemediation, responses[0].Action) - require.Equal(t, http.StatusForbidden, statusCode) - require.Equal(t, appsec.BanRemediation, appsecResponse.Action) - require.Equal(t, http.StatusTeapot, appsecResponse.HTTPStatus) - }, - }, - { - name: "custom remediation + HTTP code", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule42", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - UserBlockedHTTPCode: 418, - DefaultRemediation: "foobar", - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Equal(t, "foobar", responses[0].Action) - require.Equal(t, http.StatusForbidden, statusCode) - require.Equal(t, "foobar", appsecResponse.Action) - require.Equal(t, http.StatusTeapot, appsecResponse.HTTPStatus) - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - loadAppSecEngine(test, t) - }) - } -} - -func TestAppsecRuleMatches(t *testing.T) { - - /* - [x] basic matching rule - [x] basic non-matching rule - [ ] test the transformation - [ ] ? - */ - tests := []appsecRuleTest{ - { - name: "Basic matching rule", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 2) - require.Equal(t, types.APPSEC, events[0].Type) - - require.Equal(t, types.LOG, events[1].Type) - require.True(t, events[1].Appsec.HasInBandMatches) - require.Len(t, events[1].Appsec.MatchedRules, 1) - require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) - - require.Len(t, responses, 1) - require.True(t, responses[0].InBandInterrupt) - }, - }, - { - name: "Basic non-matching rule", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"tutu"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Empty(t, events) - require.Len(t, responses, 1) - require.False(t, responses[0].InBandInterrupt) - require.False(t, responses[0].OutOfBandInterrupt) - }, - }, - { - name: "default remediation to allow", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule42", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - DefaultRemediation: "allow", - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Equal(t, appsec.AllowRemediation, responses[0].Action) - require.Equal(t, http.StatusOK, statusCode) - require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) - require.Equal(t, http.StatusOK, appsecResponse.HTTPStatus) - }, - }, - { - name: "default remediation to captcha", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule42", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - DefaultRemediation: "captcha", - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Equal(t, appsec.CaptchaRemediation, responses[0].Action) - require.Equal(t, http.StatusForbidden, statusCode) - require.Equal(t, appsec.CaptchaRemediation, appsecResponse.Action) - require.Equal(t, http.StatusForbidden, appsecResponse.HTTPStatus) - }, - }, - { - name: "no default remediation / custom user HTTP code", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule42", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"toto"}}, - }, - UserBlockedHTTPCode: 418, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Equal(t, appsec.BanRemediation, responses[0].Action) - require.Equal(t, http.StatusForbidden, statusCode) - require.Equal(t, appsec.BanRemediation, appsecResponse.Action) - require.Equal(t, http.StatusTeapot, appsecResponse.HTTPStatus) - }, - }, - { - name: "no match but try to set remediation to captcha with on_match hook", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule42", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - on_match: []appsec.Hook{ - {Filter: "IsInBand == true", Apply: []string{"SetRemediation('captcha')"}}, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"bla"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Empty(t, events) - require.Equal(t, http.StatusOK, statusCode) - require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) - }, - }, - { - name: "no match but try to set user HTTP code with on_match hook", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule42", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - on_match: []appsec.Hook{ - {Filter: "IsInBand == true", Apply: []string{"SetReturnCode(418)"}}, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"bla"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Empty(t, events) - require.Equal(t, http.StatusOK, statusCode) - require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) - }, - }, - { - name: "no match but try to set remediation with pre_eval hook", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule42", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "regex", Value: "^toto"}, - Transform: []string{"lowercase"}, - }, - }, - pre_eval: []appsec.Hook{ - {Filter: "IsInBand == true", Apply: []string{"SetRemediationByName('rule42', 'captcha')"}}, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/urllll", - Args: url.Values{"foo": []string{"bla"}}, - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Empty(t, events) - require.Equal(t, http.StatusOK, statusCode) - require.Equal(t, appsec.AllowRemediation, appsecResponse.Action) - }, - }, - } - - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - loadAppSecEngine(test, t) - }) - } -} - -func TestAppsecRuleTransforms(t *testing.T) { - - log.SetLevel(log.TraceLevel) - tests := []appsecRuleTest{ - { - name: "Basic matching rule", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"URI"}, - Match: appsec_rule.Match{Type: "equals", Value: "/toto"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/toto", - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 2) - require.Equal(t, types.APPSEC, events[0].Type) - require.Equal(t, types.LOG, events[1].Type) - require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) - }, - }, - { - name: "lowercase", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"URI"}, - Match: appsec_rule.Match{Type: "equals", Value: "/toto"}, - Transform: []string{"lowercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/TOTO", - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 2) - require.Equal(t, types.APPSEC, events[0].Type) - require.Equal(t, types.LOG, events[1].Type) - require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) - }, - }, - { - name: "uppercase", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"URI"}, - Match: appsec_rule.Match{Type: "equals", Value: "/TOTO"}, - Transform: []string{"uppercase"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/toto", - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 2) - require.Equal(t, types.APPSEC, events[0].Type) - require.Equal(t, types.LOG, events[1].Type) - require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) - }, - }, - { - name: "b64decode", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "equals", Value: "toto"}, - Transform: []string{"b64decode"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/?foo=dG90bw", - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 2) - require.Equal(t, types.APPSEC, events[0].Type) - require.Equal(t, types.LOG, events[1].Type) - require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) - }, - }, - { - name: "b64decode with extra padding", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "equals", Value: "toto"}, - Transform: []string{"b64decode"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/?foo=dG90bw===", - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 2) - require.Equal(t, types.APPSEC, events[0].Type) - require.Equal(t, types.LOG, events[1].Type) - require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) - }, - }, - { - name: "length", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "gte", Value: "3"}, - Transform: []string{"length"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/?foo=toto", - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 2) - require.Equal(t, types.APPSEC, events[0].Type) - require.Equal(t, types.LOG, events[1].Type) - require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) - }, - }, - { - name: "urldecode", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "equals", Value: "BB/A"}, - Transform: []string{"urldecode"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/?foo=%42%42%2F%41", - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 2) - require.Equal(t, types.APPSEC, events[0].Type) - require.Equal(t, types.LOG, events[1].Type) - require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) - }, - }, - { - name: "trim", - expected_load_ok: true, - inband_rules: []appsec_rule.CustomRule{ - { - Name: "rule1", - Zones: []string{"ARGS"}, - Variables: []string{"foo"}, - Match: appsec_rule.Match{Type: "equals", Value: "BB/A"}, - Transform: []string{"urldecode", "trim"}, - }, - }, - input_request: appsec.ParsedRequest{ - RemoteAddr: "1.2.3.4", - Method: "GET", - URI: "/?foo=%20%20%42%42%2F%41%20%20", - }, - output_asserts: func(events []types.Event, responses []appsec.AppsecTempResponse, appsecResponse appsec.BodyResponse, statusCode int) { - require.Len(t, events, 2) - require.Equal(t, types.APPSEC, events[0].Type) - require.Equal(t, types.LOG, events[1].Type) - require.Equal(t, "rule1", events[1].Appsec.MatchedRules[0]["msg"]) - }, - }, - } - for _, test := range tests { - t.Run(test.name, func(t *testing.T) { - loadAppSecEngine(test, t) - }) - } -} - func loadAppSecEngine(test appsecRuleTest, t *testing.T) { if testing.Verbose() { log.SetLevel(log.TraceLevel) From 2682f801dfe90a85a7bf4d3ec2a51136280a2dac Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Fri, 5 Apr 2024 14:57:33 +0200 Subject: [PATCH 14/22] windows: fix data file update (remove before rename) (#2930) --- pkg/cwhub/dataset.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pkg/cwhub/dataset.go b/pkg/cwhub/dataset.go index 97fd9c5a0..921361e3f 100644 --- a/pkg/cwhub/dataset.go +++ b/pkg/cwhub/dataset.go @@ -4,9 +4,11 @@ import ( "errors" "fmt" "io" + "io/fs" "net/http" "os" "path/filepath" + "runtime" "time" "github.com/sirupsen/logrus" @@ -65,6 +67,18 @@ func downloadFile(url string, destPath string) error { // TODO: use a better way to communicate this fmt.Printf("updated %s\n", filepath.Base(destPath)) + if runtime.GOOS == "windows" { + // On Windows, rename will fail if the destination file already exists + // so we remove it first. + err = os.Remove(destPath) + switch { + case errors.Is(err, fs.ErrNotExist): + break + case err != nil: + return err + } + } + if err = os.Rename(tmpFileName, destPath); err != nil { return err } From 990dd5e08e6efaa608910fdc258958c9db279518 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Fri, 5 Apr 2024 15:11:11 +0200 Subject: [PATCH 15/22] use go 1.21.9; update dependencies (#2931) --- .github/workflows/bats-hub.yml | 2 +- .github/workflows/bats-mysql.yml | 2 +- .github/workflows/bats-postgres.yml | 2 +- .github/workflows/bats-sqlite-coverage.yml | 2 +- .github/workflows/ci-windows-build-msi.yml | 2 +- .github/workflows/codeql-analysis.yml | 2 +- .github/workflows/go-tests-windows.yml | 2 +- .github/workflows/go-tests.yml | 2 +- .github/workflows/publish-tarball-release.yml | 2 +- Dockerfile | 2 +- Dockerfile.debian | 2 +- azure-pipelines.yml | 2 +- go.mod | 10 +++++----- go.sum | 16 ++++++++-------- 14 files changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/bats-hub.yml b/.github/workflows/bats-hub.yml index bd84389b0..4e977201c 100644 --- a/.github/workflows/bats-hub.yml +++ b/.github/workflows/bats-hub.yml @@ -33,7 +33,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v5 with: - go-version: "1.21.8" + go-version: "1.21.9" - name: "Install bats dependencies" env: diff --git a/.github/workflows/bats-mysql.yml b/.github/workflows/bats-mysql.yml index cc90961bf..9e320b1b3 100644 --- a/.github/workflows/bats-mysql.yml +++ b/.github/workflows/bats-mysql.yml @@ -36,7 +36,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v5 with: - go-version: "1.21.8" + go-version: "1.21.9" - name: "Install bats dependencies" env: diff --git a/.github/workflows/bats-postgres.yml b/.github/workflows/bats-postgres.yml index 18cbb50a1..2cd09b494 100644 --- a/.github/workflows/bats-postgres.yml +++ b/.github/workflows/bats-postgres.yml @@ -45,7 +45,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v5 with: - go-version: "1.21.8" + go-version: "1.21.9" - name: "Install bats dependencies" env: diff --git a/.github/workflows/bats-sqlite-coverage.yml b/.github/workflows/bats-sqlite-coverage.yml index 6780727e9..d56d69f28 100644 --- a/.github/workflows/bats-sqlite-coverage.yml +++ b/.github/workflows/bats-sqlite-coverage.yml @@ -28,7 +28,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v5 with: - go-version: "1.21.8" + go-version: "1.21.9" - name: "Install bats dependencies" env: diff --git a/.github/workflows/ci-windows-build-msi.yml b/.github/workflows/ci-windows-build-msi.yml index ae3a47577..278426d77 100644 --- a/.github/workflows/ci-windows-build-msi.yml +++ b/.github/workflows/ci-windows-build-msi.yml @@ -35,7 +35,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v5 with: - go-version: "1.21.8" + go-version: "1.21.9" - name: Build run: make windows_installer BUILD_RE2_WASM=1 diff --git a/.github/workflows/codeql-analysis.yml b/.github/workflows/codeql-analysis.yml index 68c95ed64..1f27a8c7c 100644 --- a/.github/workflows/codeql-analysis.yml +++ b/.github/workflows/codeql-analysis.yml @@ -52,7 +52,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v5 with: - go-version: "1.21.8" + go-version: "1.21.9" cache-dependency-path: "**/go.sum" # Initializes the CodeQL tools for scanning. diff --git a/.github/workflows/go-tests-windows.yml b/.github/workflows/go-tests-windows.yml index 6b2f1132a..781f2a4a9 100644 --- a/.github/workflows/go-tests-windows.yml +++ b/.github/workflows/go-tests-windows.yml @@ -34,7 +34,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v5 with: - go-version: "1.21.8" + go-version: "1.21.9" - name: Build run: | diff --git a/.github/workflows/go-tests.yml b/.github/workflows/go-tests.yml index d76315462..67f73d81a 100644 --- a/.github/workflows/go-tests.yml +++ b/.github/workflows/go-tests.yml @@ -126,7 +126,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v5 with: - go-version: "1.21.8" + go-version: "1.21.9" - name: Create localstack streams run: | diff --git a/.github/workflows/publish-tarball-release.yml b/.github/workflows/publish-tarball-release.yml index 6fd582da8..6cdf111a4 100644 --- a/.github/workflows/publish-tarball-release.yml +++ b/.github/workflows/publish-tarball-release.yml @@ -25,7 +25,7 @@ jobs: - name: "Set up Go" uses: actions/setup-go@v5 with: - go-version: "1.21.8" + go-version: "1.21.9" - name: Build the binaries run: | diff --git a/Dockerfile b/Dockerfile index 53a6cd045..1e311bfa8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ # vim: set ft=dockerfile: -FROM golang:1.21.8-alpine3.18 AS build +FROM golang:1.21.9-alpine3.18 AS build ARG BUILD_VERSION diff --git a/Dockerfile.debian b/Dockerfile.debian index fd40bd475..ee0fa32bb 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -1,5 +1,5 @@ # vim: set ft=dockerfile: -FROM golang:1.21.8-bookworm AS build +FROM golang:1.21.9-bookworm AS build ARG BUILD_VERSION diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f90af6f17..b662a809a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -21,7 +21,7 @@ stages: - task: GoTool@0 displayName: "Install Go" inputs: - version: '1.21.8' + version: '1.21.9' - pwsh: | choco install -y make diff --git a/go.mod b/go.mod index 50b88f7d4..04f34e648 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/buger/jsonparser v1.1.1 github.com/c-robinson/iplib v1.0.3 github.com/cespare/xxhash/v2 v2.2.0 + github.com/corazawaf/libinjection-go v0.1.2 github.com/crowdsecurity/coraza/v3 v3.0.0-20240108124027-a62b8d8e5607 github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 github.com/crowdsecurity/go-cs-lib v0.0.6 @@ -81,9 +82,9 @@ require ( github.com/umahmood/haversine v0.0.0-20151105152445-808ab04add26 github.com/wasilibs/go-re2 v1.3.0 github.com/xhit/go-simple-mail/v2 v2.16.0 - golang.org/x/crypto v0.20.0 + golang.org/x/crypto v0.22.0 golang.org/x/mod v0.11.0 - golang.org/x/sys v0.17.0 + golang.org/x/sys v0.19.0 golang.org/x/text v0.14.0 google.golang.org/grpc v1.56.3 google.golang.org/protobuf v1.33.0 @@ -104,7 +105,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bytedance/sonic v1.9.1 // indirect github.com/chenzhuoyu/base64x v0.0.0-20221115062448-fe3a3abad311 // indirect - github.com/corazawaf/libinjection-go v0.1.2 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.3 // indirect github.com/creack/pty v1.1.18 // indirect @@ -198,9 +198,9 @@ require ( github.com/zclconf/go-cty v1.8.0 // indirect go.mongodb.org/mongo-driver v1.9.4 // indirect golang.org/x/arch v0.3.0 // indirect - golang.org/x/net v0.21.0 // indirect + golang.org/x/net v0.24.0 // indirect golang.org/x/sync v0.6.0 // indirect - golang.org/x/term v0.17.0 // indirect + golang.org/x/term v0.19.0 // indirect golang.org/x/time v0.3.0 // indirect golang.org/x/tools v0.8.1-0.20230428195545-5283a0178901 // indirect golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect diff --git a/go.sum b/go.sum index 8f91bd31f..29e23f02a 100644 --- a/go.sum +++ b/go.sum @@ -757,8 +757,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.3.0/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= -golang.org/x/crypto v0.20.0 h1:jmAMJJZXr5KiCw05dfYK9QnqaqKLYXijU23lsEdcQqg= -golang.org/x/crypto v0.20.0/go.mod h1:Xwo95rrVNIoSMx9wa1JroENMToLWn3RNVrTBpLHgZPQ= +golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= +golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= @@ -791,8 +791,8 @@ golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= -golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= -golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= +golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -841,8 +841,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.17.0 h1:25cE3gD+tdBA7lp7QfhuV+rJiE9YXTcS3VG1SqssI/Y= -golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= +golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -850,8 +850,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= -golang.org/x/term v0.17.0 h1:mkTF7LCd6WGJNL3K1Ad7kwxNfYAW6a8a8QqtMblp/4U= -golang.org/x/term v0.17.0/go.mod h1:lLRBjIVuehSbZlaOtGMbcMncT+aqLLLmKrsjNrUguwk= +golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= +golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= From 0e8a1c681b0c72dc45509aa14f6c5a6c9df83ab0 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 8 Apr 2024 14:53:12 +0200 Subject: [PATCH 16/22] docker: pre-download all hub items and data, opt-in hub update/upgrade (#2933) * docker: pre-download all hub items and data, opt-in hub update/upgrade * docker/bars: don't purge anything before pre-downloading hub * Docker: README update --- Dockerfile | 1 + docker/README.md | 11 +++++++++-- docker/docker_start.sh | 5 ++--- docker/preload-hub-items | 22 ++++++++++++++++++++++ test/bin/preload-hub-items | 19 ++----------------- 5 files changed, 36 insertions(+), 22 deletions(-) create mode 100755 docker/preload-hub-items diff --git a/Dockerfile b/Dockerfile index 1e311bfa8..d2b01ed77 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \ ./wizard.sh --docker-mode && \ cd - >/dev/null && \ cscli hub update && \ + ./docker/preload-hub-items && \ cscli collections install crowdsecurity/linux && \ cscli parsers install crowdsecurity/whitelists diff --git a/docker/README.md b/docker/README.md index 5e39838a1..2fea57a61 100644 --- a/docker/README.md +++ b/docker/README.md @@ -134,7 +134,6 @@ labels: type: apache2 ``` - ## Recommended configuration ### Volumes @@ -146,6 +145,14 @@ to avoid losing credentials and decision data in case of container destruction a * Acquisition: `/etc/crowdsec/acquis.d` and/or `/etc/crowdsec.acquis.yaml` (yes, they can be nested in `/etc/crowdsec`) * Database when using SQLite (default): `/var/lib/crowdsec/data` +### Hub updates + +To ensure you have the latest version of the collections, scenarios, parsers, etc., you can set the variable `DO_HUB_UPGRADE` to true. +This will perform an update/upgrade of the hub every time the container is started. + +Be aware that if your container is misbehaving and caught in a restart loop, the CrowdSec hub may ban your IP for some time and your containers +will run with the version of the hub that is cached in the container's image. If you enable `DO_HUB_UPGRADE`, do it when your infrastructure is running +correctly and make sure you have some monitoring in place. ## Start a Crowdsec instance @@ -316,7 +323,7 @@ config.yaml) each time the container is run. | `BOUNCERS_ALLOWED_OU` | bouncer-ou | OU values allowed for bouncers, separated by comma | | | | | | __Hub management__ | | | -| `NO_HUB_UPGRADE` | false | Skip hub update / upgrade when the container starts | +| `DO_HUB_UPGRADE` | false | Force hub update / upgrade when the container starts. If for some reason the container restarts too often, it may lead to a temporary ban from hub updates. | | `COLLECTIONS` | | Collections to install, separated by space: `-e COLLECTIONS="crowdsecurity/linux crowdsecurity/apache2"` | | `PARSERS` | | Parsers to install, separated by space | | `SCENARIOS` | | Scenarios to install, separated by space | diff --git a/docker/docker_start.sh b/docker/docker_start.sh index dd96184cc..26c5b0eee 100755 --- a/docker/docker_start.sh +++ b/docker/docker_start.sh @@ -304,9 +304,8 @@ conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)' ## Install hub items -cscli hub update || true - -if isfalse "$NO_HUB_UPGRADE"; then +if istrue "$DO_HUB_UPGRADE"; then + cscli hub update || true cscli hub upgrade || true fi diff --git a/docker/preload-hub-items b/docker/preload-hub-items new file mode 100755 index 000000000..d02b09485 --- /dev/null +++ b/docker/preload-hub-items @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +set -eu + +# pre-download everything but don't install anything + +echo "Pre-downloading Hub content..." + +types=$(cscli hub types -o raw) + +for itemtype in $types; do + ALL_ITEMS=$(cscli "$itemtype" list -a -o json | itemtype="$itemtype" yq '.[env(itemtype)][] | .name') + if [[ -n "${ALL_ITEMS}" ]]; then + #shellcheck disable=SC2086 + cscli "$itemtype" install \ + $ALL_ITEMS \ + --download-only \ + --error + fi +done + +echo " done." diff --git a/test/bin/preload-hub-items b/test/bin/preload-hub-items index 14e9cff99..ddf7fecba 100755 --- a/test/bin/preload-hub-items +++ b/test/bin/preload-hub-items @@ -9,20 +9,12 @@ THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) # pre-download everything but don't install anything -echo -n "Purging existing hub..." +echo "Pre-downloading Hub content..." types=$("$CSCLI" hub types -o raw) for itemtype in $types; do - "$CSCLI" "${itemtype}" delete --all --error --purge --force -done - -echo " done." - -echo -n "Pre-downloading Hub content..." - -for itemtype in $types; do - ALL_ITEMS=$("$CSCLI" "$itemtype" list -a -o json | jq --arg itemtype "$itemtype" -r '.[$itemtype][].name') + ALL_ITEMS=$("$CSCLI" "$itemtype" list -a -o json | itemtype="$itemtype" yq '.[env(itemtype)][] | .name') if [[ -n "${ALL_ITEMS}" ]]; then #shellcheck disable=SC2086 "$CSCLI" "$itemtype" install \ @@ -32,11 +24,4 @@ for itemtype in $types; do fi done -# XXX: download-only works only for collections, not for parsers, scenarios, postoverflows. -# so we have to delete the links manually, and leave the downloaded files in place - -for itemtype in $types; do - "$CSCLI" "$itemtype" delete --all --error -done - echo " done." From 2291a232cb7188862bd1976615a19835e0f94ad2 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 8 Apr 2024 15:00:45 +0200 Subject: [PATCH 17/22] docker: pre-download hub items (debian image) (#2934) --- Dockerfile.debian | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile.debian b/Dockerfile.debian index ee0fa32bb..0d094f8df 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -30,6 +30,7 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \ ./wizard.sh --docker-mode && \ cd - >/dev/null && \ cscli hub update && \ + ./docker/preload-hub-items && \ cscli collections install crowdsecurity/linux && \ cscli parsers install crowdsecurity/whitelists From 0746e0c091b8426791a4a495b25a186efd899c9d Mon Sep 17 00:00:00 2001 From: AlteredCoder <64792091+AlteredCoder@users.noreply.github.com> Date: Thu, 11 Apr 2024 11:23:19 +0200 Subject: [PATCH 18/22] Rename bouncers to Remediation component in openAPI (#2936) * Rename bouncers to Remediation component in openAPI --- pkg/models/localapi_swagger.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/models/localapi_swagger.yaml b/pkg/models/localapi_swagger.yaml index 66132e5e3..d167da9b6 100644 --- a/pkg/models/localapi_swagger.yaml +++ b/pkg/models/localapi_swagger.yaml @@ -26,10 +26,10 @@ produces: paths: /decisions/stream: get: - description: Returns a list of new/expired decisions. Intended for bouncers that need to "stream" decisions + description: Returns a list of new/expired decisions. Intended for remediation component that need to "stream" decisions summary: getDecisionsStream tags: - - bouncers + - Remediation component operationId: getDecisionsStream deprecated: false produces: @@ -39,7 +39,7 @@ paths: in: query required: false type: boolean - description: 'If true, means that the bouncers is starting and a full list must be provided' + description: 'If true, means that the remediation component is starting and a full list must be provided' - name: scopes in: query required: false @@ -73,10 +73,10 @@ paths: security: - APIKeyAuthorizer: [] head: - description: Returns a list of new/expired decisions. Intended for bouncers that need to "stream" decisions + description: Returns a list of new/expired decisions. Intended for remediation component that need to "stream" decisions summary: GetDecisionsStream tags: - - bouncers + - Remediation component operationId: headDecisionsStream deprecated: false produces: @@ -100,7 +100,7 @@ paths: description: Returns information about existing decisions summary: getDecisions tags: - - bouncers + - Remediation component operationId: getDecisions deprecated: false produces: @@ -164,7 +164,7 @@ paths: description: Returns information about existing decisions summary: GetDecisions tags: - - bouncers + - Remediation component operationId: headDecisions deprecated: false produces: @@ -1008,7 +1008,7 @@ definitions: title: "error response" description: "error response return by the API" tags: - - name: bouncers + - name: Remediation component description: 'Operations about decisions : bans, captcha, rate-limit etc.' - name: watchers description: 'Operations about watchers : cscli & crowdsec' From c6e40191ddf7a87b8c0d946c36083c084677e5f1 Mon Sep 17 00:00:00 2001 From: blotus Date: Thu, 18 Apr 2024 15:33:51 +0200 Subject: [PATCH 19/22] =?UTF-8?q?Revert=20"docker:=20pre-download=20all=20?= =?UTF-8?q?hub=20items=20and=20data,=20opt-in=20hub=20updat=E2=80=A6=20(#2?= =?UTF-8?q?947)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .github/workflows/docker-tests.yml | 14 +++++++------- Dockerfile | 1 - Dockerfile.debian | 1 - docker/README.md | 11 ++--------- docker/docker_start.sh | 5 +++-- docker/preload-hub-items | 22 ---------------------- test/bin/preload-hub-items | 19 +++++++++++++++++-- 7 files changed, 29 insertions(+), 44 deletions(-) delete mode 100755 docker/preload-hub-items diff --git a/.github/workflows/docker-tests.yml b/.github/workflows/docker-tests.yml index d3ae4f90d..3e87d3ba4 100644 --- a/.github/workflows/docker-tests.yml +++ b/.github/workflows/docker-tests.yml @@ -59,15 +59,15 @@ jobs: cd docker/test python -m pip install --upgrade pipenv wheel - - name: "Cache virtualenvs" - id: cache-pipenv - uses: actions/cache@v4 - with: - path: ~/.local/share/virtualenvs - key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }} + #- name: "Cache virtualenvs" + # id: cache-pipenv + # uses: actions/cache@v4 + # with: + # path: ~/.local/share/virtualenvs + # key: ${{ runner.os }}-pipenv-${{ hashFiles('**/Pipfile.lock') }} - name: "Install dependencies" - if: steps.cache-pipenv.outputs.cache-hit != 'true' + #if: steps.cache-pipenv.outputs.cache-hit != 'true' run: | cd docker/test pipenv install --deploy diff --git a/Dockerfile b/Dockerfile index d2b01ed77..1e311bfa8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,6 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \ ./wizard.sh --docker-mode && \ cd - >/dev/null && \ cscli hub update && \ - ./docker/preload-hub-items && \ cscli collections install crowdsecurity/linux && \ cscli parsers install crowdsecurity/whitelists diff --git a/Dockerfile.debian b/Dockerfile.debian index 0d094f8df..ee0fa32bb 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -30,7 +30,6 @@ RUN make clean release DOCKER_BUILD=1 BUILD_STATIC=1 && \ ./wizard.sh --docker-mode && \ cd - >/dev/null && \ cscli hub update && \ - ./docker/preload-hub-items && \ cscli collections install crowdsecurity/linux && \ cscli parsers install crowdsecurity/whitelists diff --git a/docker/README.md b/docker/README.md index 2fea57a61..5e39838a1 100644 --- a/docker/README.md +++ b/docker/README.md @@ -134,6 +134,7 @@ labels: type: apache2 ``` + ## Recommended configuration ### Volumes @@ -145,14 +146,6 @@ to avoid losing credentials and decision data in case of container destruction a * Acquisition: `/etc/crowdsec/acquis.d` and/or `/etc/crowdsec.acquis.yaml` (yes, they can be nested in `/etc/crowdsec`) * Database when using SQLite (default): `/var/lib/crowdsec/data` -### Hub updates - -To ensure you have the latest version of the collections, scenarios, parsers, etc., you can set the variable `DO_HUB_UPGRADE` to true. -This will perform an update/upgrade of the hub every time the container is started. - -Be aware that if your container is misbehaving and caught in a restart loop, the CrowdSec hub may ban your IP for some time and your containers -will run with the version of the hub that is cached in the container's image. If you enable `DO_HUB_UPGRADE`, do it when your infrastructure is running -correctly and make sure you have some monitoring in place. ## Start a Crowdsec instance @@ -323,7 +316,7 @@ config.yaml) each time the container is run. | `BOUNCERS_ALLOWED_OU` | bouncer-ou | OU values allowed for bouncers, separated by comma | | | | | | __Hub management__ | | | -| `DO_HUB_UPGRADE` | false | Force hub update / upgrade when the container starts. If for some reason the container restarts too often, it may lead to a temporary ban from hub updates. | +| `NO_HUB_UPGRADE` | false | Skip hub update / upgrade when the container starts | | `COLLECTIONS` | | Collections to install, separated by space: `-e COLLECTIONS="crowdsecurity/linux crowdsecurity/apache2"` | | `PARSERS` | | Parsers to install, separated by space | | `SCENARIOS` | | Scenarios to install, separated by space | diff --git a/docker/docker_start.sh b/docker/docker_start.sh index 26c5b0eee..dd96184cc 100755 --- a/docker/docker_start.sh +++ b/docker/docker_start.sh @@ -304,8 +304,9 @@ conf_set_if "$PLUGIN_DIR" '.config_paths.plugin_dir = strenv(PLUGIN_DIR)' ## Install hub items -if istrue "$DO_HUB_UPGRADE"; then - cscli hub update || true +cscli hub update || true + +if isfalse "$NO_HUB_UPGRADE"; then cscli hub upgrade || true fi diff --git a/docker/preload-hub-items b/docker/preload-hub-items deleted file mode 100755 index d02b09485..000000000 --- a/docker/preload-hub-items +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -# pre-download everything but don't install anything - -echo "Pre-downloading Hub content..." - -types=$(cscli hub types -o raw) - -for itemtype in $types; do - ALL_ITEMS=$(cscli "$itemtype" list -a -o json | itemtype="$itemtype" yq '.[env(itemtype)][] | .name') - if [[ -n "${ALL_ITEMS}" ]]; then - #shellcheck disable=SC2086 - cscli "$itemtype" install \ - $ALL_ITEMS \ - --download-only \ - --error - fi -done - -echo " done." diff --git a/test/bin/preload-hub-items b/test/bin/preload-hub-items index ddf7fecba..14e9cff99 100755 --- a/test/bin/preload-hub-items +++ b/test/bin/preload-hub-items @@ -9,12 +9,20 @@ THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) # pre-download everything but don't install anything -echo "Pre-downloading Hub content..." +echo -n "Purging existing hub..." types=$("$CSCLI" hub types -o raw) for itemtype in $types; do - ALL_ITEMS=$("$CSCLI" "$itemtype" list -a -o json | itemtype="$itemtype" yq '.[env(itemtype)][] | .name') + "$CSCLI" "${itemtype}" delete --all --error --purge --force +done + +echo " done." + +echo -n "Pre-downloading Hub content..." + +for itemtype in $types; do + ALL_ITEMS=$("$CSCLI" "$itemtype" list -a -o json | jq --arg itemtype "$itemtype" -r '.[$itemtype][].name') if [[ -n "${ALL_ITEMS}" ]]; then #shellcheck disable=SC2086 "$CSCLI" "$itemtype" install \ @@ -24,4 +32,11 @@ for itemtype in $types; do fi done +# XXX: download-only works only for collections, not for parsers, scenarios, postoverflows. +# so we have to delete the links manually, and leave the downloaded files in place + +for itemtype in $types; do + "$CSCLI" "$itemtype" delete --all --error +done + echo " done." From 95f27677e4c27d66cbf387c58c77a58491c5b378 Mon Sep 17 00:00:00 2001 From: Laurence Jones Date: Mon, 22 Apr 2024 14:18:34 +0100 Subject: [PATCH 20/22] enhance: add refactoring to governance (#2955) --- .github/governance.yml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/governance.yml b/.github/governance.yml index c57fd3626..0457a24c7 100644 --- a/.github/governance.yml +++ b/.github/governance.yml @@ -42,7 +42,7 @@ issue: 3. Check [Releases](https://github.com/crowdsecurity/crowdsec/releases/latest) to make sure your agent is on the latest version. - prefix: kind - list: ['feature', 'bug', 'packaging', 'enhancement'] + list: ['feature', 'bug', 'packaging', 'enhancement', 'refactoring'] multiple: false author_association: author: true @@ -54,6 +54,7 @@ issue: @$AUTHOR: There are no 'kind' label on this issue. You need a 'kind' label to start the triage process. * `/kind feature` * `/kind enhancement` + * `/kind refactoring` * `/kind bug` * `/kind packaging` @@ -65,12 +66,13 @@ pull_request: labels: - prefix: kind multiple: false - list: [ 'feature', 'enhancement', 'fix', 'chore', 'dependencies'] + list: [ 'feature', 'enhancement', 'fix', 'chore', 'dependencies', 'refactoring'] needs: comment: | @$AUTHOR: There are no 'kind' label on this PR. You need a 'kind' label to generate the release automatically. * `/kind feature` * `/kind enhancement` + * `/kind refactoring` * `/kind fix` * `/kind chore` * `/kind dependencies` From 630cbf0c70658dff679bb1b95e7b54ba43dd4c0a Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 22 Apr 2024 17:18:11 +0200 Subject: [PATCH 21/22] update linter list and descriptions (#2951) --- .golangci.yml | 77 +++++++++++++++++++++++++++++---------------------- 1 file changed, 44 insertions(+), 33 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index df0cb67d1..f38fa337a 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -147,37 +147,40 @@ linters: # # DEPRECATED by golangi-lint # - - deadcode # The owner seems to have abandoned the linter. Replaced by unused. - - exhaustivestruct # The owner seems to have abandoned the linter. Replaced by exhaustruct. - - golint # Golint differs from gofmt. Gofmt reformats Go source code, whereas golint prints out style mistakes - - ifshort # Checks that your code uses short syntax for if-statements whenever possible - - interfacer # Linter that suggests narrower interface types - - maligned # Tool to detect Go structs that would take less memory if their fields were sorted - - nosnakecase # nosnakecase is a linter that detects snake case of variable naming and function name. - - scopelint # Scopelint checks for unpinned variables in go programs - - structcheck # The owner seems to have abandoned the linter. Replaced by unused. - - varcheck # The owner seems to have abandoned the linter. Replaced by unused. + - deadcode + - exhaustivestruct + - golint + - ifshort + - interfacer + - maligned + - nosnakecase + - scopelint + - structcheck + - varcheck # # Enabled # # - asasalint # check for pass []any as any in variadic func(...any) - # - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers + # - asciicheck # checks that all code identifiers does not have non-ASCII symbols in the name # - bidichk # Checks for dangerous unicode character sequences # - bodyclose # checks whether HTTP response body is closed successfully + # - copyloopvar # copyloopvar is a linter detects places where loop variables are copied # - cyclop # checks function and package cyclomatic complexity # - decorder # check declaration order and count of types, constants, variables and functions # - depguard # Go linter that checks if package imports are in a list of acceptable packages # - dupword # checks for duplicate words in the source code # - durationcheck # check for two durations multiplied together - # - errcheck # Errcheck is a program for checking for unchecked errors in go programs. These unchecked errors can be critical bugs in some cases + # - errcheck # errcheck is a program for checking for unchecked errors in Go code. These unchecked errors can be critical bugs in some cases # - errorlint # errorlint is a linter for that can be used to find code that will cause problems with the error wrapping scheme introduced in Go 1.13. # - execinquery # execinquery is a linter about query string checker in Query function which reads your Go src files and warning it finds # - exportloopref # checks for pointers to enclosing loop variables # - funlen # Tool for detection of long functions # - ginkgolinter # enforces standards of using ginkgo and gomega + # - gocheckcompilerdirectives # Checks that go compiler directive comments (//go:) are valid. # - gochecknoinits # Checks that no init functions are present in Go code + # - gochecksumtype # Run exhaustiveness checks on Go "sum types" # - gocognit # Computes and checks the cognitive complexity of functions # - gocritic # Provides diagnostics that check for bugs, performance and style issues. # - gocyclo # Computes and checks the cyclomatic complexity of functions @@ -185,48 +188,56 @@ linters: # - gomoddirectives # Manage the use of 'replace', 'retract', and 'excludes' directives in go.mod. # - gomodguard # Allow and block list linter for direct Go module dependencies. This is different from depguard where there are different block types for example version constraints and module recommendations. # - goprintffuncname # Checks that printf-like functions are named with `f` at the end - # - gosimple # (megacheck): Linter for Go source code that specializes in simplifying a code - # - govet # (vet, vetshadow): Vet examines Go source code and reports suspicious constructs, such as Printf calls whose arguments do not align with the format string - # - grouper # An analyzer to analyze expression groups. + # - gosimple # (megacheck): Linter for Go source code that specializes in simplifying code + # - gosmopolitan # Report certain i18n/l10n anti-patterns in your Go codebase + # - govet # (vet, vetshadow): Vet examines Go source code and reports suspicious constructs. It is roughly the same as 'go vet' and uses its passes. + # - grouper # Analyze expression groups. # - importas # Enforces consistent import aliases # - ineffassign # Detects when assignments to existing variables are not used # - interfacebloat # A linter that checks the number of methods inside an interface. + # - intrange # intrange is a linter to find places where for loops could make use of an integer range. # - lll # Reports long lines + # - loggercheck # (logrlint): Checks key value pairs for common logger libraries (kitlog,klog,logr,zap). # - logrlint # Check logr arguments. # - maintidx # maintidx measures the maintainability index of each function. # - makezero # Finds slice declarations with non-zero initial length - # - misspell # Finds commonly misspelled English words in comments - # - nakedret # Finds naked returns in functions greater than a specified function length + # - mirror # reports wrong mirror patterns of bytes/strings usage + # - misspell # Finds commonly misspelled English words + # - nakedret # Checks that functions with naked returns are not longer than a maximum size (can be zero). # - nestif # Reports deeply nested if statements # - nilerr # Finds the code that returns nil even if it checks that the error is not nil. # - nolintlint # Reports ill-formed or insufficient nolint directives # - nonamedreturns # Reports all named returns # - nosprintfhostport # Checks for misuse of Sprintf to construct a host with port in a URL. + # - perfsprint # Checks that fmt.Sprintf can be replaced with a faster alternative. # - predeclared # find code that shadows one of Go's predeclared identifiers # - reassign # Checks that package variables are not reassigned - # - rowserrcheck # checks whether Err of rows is checked successfully - # - sqlclosecheck # Checks that sql.Rows and sql.Stmt are closed. - # - staticcheck # (megacheck): Staticcheck is a go vet on steroids, applying a ton of static analysis checks - # - testableexamples # linter checks if examples are testable (have an expected output) + # - rowserrcheck # checks whether Rows.Err of rows is checked successfully + # - sloglint # ensure consistent code style when using log/slog + # - spancheck # Checks for mistakes with OpenTelemetry/Census spans. + # - sqlclosecheck # Checks that sql.Rows, sql.Stmt, sqlx.NamedStmt, pgx.Query are closed. + # - staticcheck # (megacheck): It's a set of rules from staticcheck. It's not the same thing as the staticcheck binary. The author of staticcheck doesn't support or approve the use of staticcheck as a library inside golangci-lint. # - tenv # tenv is analyzer that detects using os.Setenv instead of t.Setenv since Go1.17 + # - testableexamples # linter checks if examples are testable (have an expected output) + # - testifylint # Checks usage of github.com/stretchr/testify. # - tparallel # tparallel detects inappropriate usage of t.Parallel() method in your Go test codes - # - typecheck # Like the front-end of a Go compiler, parses and type-checks Go code # - unconvert # Remove unnecessary type conversions # - unused # (megacheck): Checks Go code for unused constants, variables, functions and types # - usestdlibvars # A linter that detect the possibility to use variables/constants from the Go standard library. - # - wastedassign # wastedassign finds wasted assignment statements. + # - wastedassign # Finds wasted assignment statements + # - zerologlint # Detects the wrong usage of `zerolog` that a user forgets to dispatch with `Send` or `Msg` # # Recommended? (easy) # - dogsled # Checks assignments with too many blank identifiers (e.g. x, _, _, _, := f()) - - errchkjson # Checks types passed to the json encoding functions. Reports unsupported types and optionally reports occations, where the check for the returned error can be omitted. + - errchkjson # Checks types passed to the json encoding functions. Reports unsupported types and reports occations, where the check for the returned error can be omitted. - exhaustive # check exhaustiveness of enum switch statements - gci # Gci control golang package import order and make it always deterministic. - godot # Check if comments end in a period - gofmt # Gofmt checks whether code was gofmt-ed. By default this tool runs with -s option to check for code simplification - - goimports # In addition to fixing imports, goimports also formats your code in the same style as gofmt. + - goimports # Check import statements are formatted according to the 'goimport' command. Reformat imports in autofix mode. - gosec # (gas): Inspects source code for security problems - inamedparam # reports interfaces with unnamed method parameters - musttag # enforce field tags in (un)marshaled structs @@ -234,7 +245,7 @@ linters: - protogetter # Reports direct reads from proto message fields when getters should be used - revive # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint. - tagalign # check that struct tags are well aligned - - thelper # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers + - thelper # thelper detects tests helpers which is not start with t.Helper() method. - wrapcheck # Checks that errors returned from external packages are wrapped # @@ -242,12 +253,12 @@ linters: # - containedctx # containedctx is a linter that detects struct contained context.Context field - - contextcheck # check the function whether use a non-inherited context + - contextcheck # check whether the function uses a non-inherited context - errname # Checks that sentinel errors are prefixed with the `Err` and error types are suffixed with the `Error`. - gomnd # An analyzer to detect magic numbers. - ireturn # Accept Interfaces, Return Concrete Types - nilnil # Checks that there is no simultaneous return of `nil` error and an invalid value. - - noctx # noctx finds sending http request without context.Context + - noctx # Finds sending http request without context.Context - unparam # Reports unused function parameters # @@ -256,8 +267,8 @@ linters: - gofumpt # Gofumpt checks whether code was gofumpt-ed. - nlreturn # nlreturn checks for a new line before return and branch statements to increase code clarity - - whitespace # Tool for detection of leading and trailing whitespace - - wsl # Whitespace Linter - Forces you to use empty lines! + - whitespace # Whitespace is a linter that checks for unnecessary newlines at the start and end of functions, if, for, etc. + - wsl # add or remove empty lines # # Well intended, but not ready for this @@ -265,8 +276,8 @@ linters: - dupl # Tool for code clone detection - forcetypeassert # finds forced type assertions - godox # Tool for detection of FIXME, TODO and other comment keywords - - goerr113 # Golang linter to check the errors handling expressions - - paralleltest # paralleltest detects missing usage of t.Parallel() method in your Go test + - goerr113 # Go linter to check the errors handling expressions + - paralleltest # Detects missing usage of t.Parallel() method in your Go test - testpackage # linter that makes you use a separate _test package # @@ -274,7 +285,7 @@ linters: # - exhaustruct # Checks if all structure fields are initialized - forbidigo # Forbids identifiers - - gochecknoglobals # check that no global variables exist + - gochecknoglobals # Check that no global variables exist. - goconst # Finds repeated strings that could be replaced by a constant - stylecheck # Stylecheck is a replacement for golint - tagliatelle # Checks the struct tags. From fb393f1c57cd72298d6684884bbb7419365a798e Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Mon, 22 Apr 2024 17:19:00 +0200 Subject: [PATCH 22/22] tests: bump yq, cfssl (#2952) --- Dockerfile | 2 +- Dockerfile.debian | 2 +- test/bats.mk | 10 +++++----- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1e311bfa8..69de0f9df 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN apk add --no-cache git g++ gcc libc-dev make bash gettext binutils-gold core cd re2-${RE2_VERSION} && \ make install && \ echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \ - go install github.com/mikefarah/yq/v4@v4.40.4 + go install github.com/mikefarah/yq/v4@v4.43.1 COPY . . diff --git a/Dockerfile.debian b/Dockerfile.debian index ee0fa32bb..9bcb517bb 100644 --- a/Dockerfile.debian +++ b/Dockerfile.debian @@ -21,7 +21,7 @@ RUN apt-get update && \ make && \ make install && \ echo "githubciXXXXXXXXXXXXXXXXXXXXXXXX" > /etc/machine-id && \ - go install github.com/mikefarah/yq/v4@v4.40.4 + go install github.com/mikefarah/yq/v4@v4.43.1 COPY . . diff --git a/test/bats.mk b/test/bats.mk index 0cc5deb9b..8f507cb65 100644 --- a/test/bats.mk +++ b/test/bats.mk @@ -66,11 +66,11 @@ bats-check-requirements: ## Check dependencies for functional tests @$(TEST_DIR)/bin/check-requirements bats-update-tools: ## Install/update tools required for functional tests - # yq v4.40.4 - GOBIN=$(TEST_DIR)/tools go install github.com/mikefarah/yq/v4@1c3d55106075bd37df197b4bc03cb4a413fdb903 - # cfssl v1.6.4 - GOBIN=$(TEST_DIR)/tools go install github.com/cloudflare/cfssl/cmd/cfssl@b4d0d877cac528f63db39dfb62d5c96cd3a32a0b - GOBIN=$(TEST_DIR)/tools go install github.com/cloudflare/cfssl/cmd/cfssljson@b4d0d877cac528f63db39dfb62d5c96cd3a32a0b + # yq v4.43.1 + GOBIN=$(TEST_DIR)/tools go install github.com/mikefarah/yq/v4@c35ec752e38ea0c096d3c44e13cfc0797ac394d8 + # cfssl v1.6.5 + GOBIN=$(TEST_DIR)/tools go install github.com/cloudflare/cfssl/cmd/cfssl@96259aa29c9cc9b2f4e04bad7d4bc152e5405dda + GOBIN=$(TEST_DIR)/tools go install github.com/cloudflare/cfssl/cmd/cfssljson@96259aa29c9cc9b2f4e04bad7d4bc152e5405dda # Build and installs crowdsec in a local directory. Rebuilds if already exists. bats-build: bats-environment ## Build binaries for functional tests