beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'master' into coraza_poc_acquis

Browse source
This commit is contained in:
Sebastien Blot 2023-11-29 22:02:51 +01:00
commit 4b7b138be7
No known key found for this signature in database
GPG key ID: DFC2902F40449F6A
31 changed files with 861 additions and 564 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
.github/workflows/go-tests.yml vendored
View file

@ -142,12 +142,13 @@ jobs:
go install github.com/kyoh86/richgo@v0.3.10
set -o pipefail
make build BUILD_STATIC=1
make go-acc | richgo testfilter
make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
- name: Run tests again, dynamic
run: |
make clean build
make go-acc | richgo testfilter
set -o pipefail
make go-acc | sed 's/ *coverage:.*of statements in.*//' | richgo testfilter
- name: Upload unit coverage to Codecov
uses: codecov/codecov-action@v3

3
Makefile
View file

@ -232,8 +232,7 @@ test: testenv goversion
# run the tests with localstack and coverage
.PHONY: go-acc
go-acc: testenv goversion
go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS) | \
sed 's/ *coverage:.*of statements in.*//'
go-acc ./... -o coverage.out --ignore database,notifications,protobufs,cwversion,cstest,models -- $(LD_OPTS)
# mock AWS services
.PHONY: localstack

8
cmd/crowdsec-cli/item_suggest.go
View file

@ -12,10 +12,10 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
const MaxDistance = 7
// suggestNearestMessage returns a message with the most similar item name, if one is found
func suggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string {
const maxDistance = 7
// SuggestNearestMessage returns a message with the most similar item name, if one is found
func SuggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) string {
score := 100
nearest := ""
@ -29,7 +29,7 @@ func SuggestNearestMessage(hub *cwhub.Hub, itemType string, itemName string) str
msg := fmt.Sprintf("can't find '%s' in %s", itemName, itemType)
if score < MaxDistance {
if score < maxDistance {
msg += fmt.Sprintf(", did you mean '%s'?", nearest)
}

5
cmd/crowdsec-cli/itemcommands.go
View file

@ -274,7 +274,7 @@ func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string)
for _, name := range args {
item := hub.GetItem(it.name, name)
if item == nil {
msg := SuggestNearestMessage(hub, it.name, name)
msg := suggestNearestMessage(hub, it.name, name)
if !ignoreError {
return fmt.Errorf(msg)
}
@ -379,6 +379,7 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e
return err
}
if didRemove {
log.Infof("Removed %s", item.Name)
removed++
}
}
@ -421,6 +422,8 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e
removed++
}
}
log.Infof("Removed %d %s", removed, it.name)
if removed > 0 {
log.Infof(ReloadMessage())
}

223
cmd/crowdsec-cli/notifications.go
View file

@ -18,15 +18,19 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/tomb.v2"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/go-cs-lib/ptr"
"github.com/crowdsecurity/go-cs-lib/version"
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/csplugin"
"github.com/crowdsecurity/crowdsec/pkg/csprofiles"
"github.com/crowdsecurity/crowdsec/pkg/types"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/models"
)
type NotificationsCfg struct {
@ -61,11 +65,12 @@ func NewNotificationsCmd() *cobra.Command {
cmdNotifications.AddCommand(NewNotificationsListCmd())
cmdNotifications.AddCommand(NewNotificationsInspectCmd())
cmdNotifications.AddCommand(NewNotificationsReinjectCmd())
cmdNotifications.AddCommand(NewNotificationsTestCmd())
return cmdNotifications
}
func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
func getPluginConfigs() (map[string]csplugin.PluginConfig, error) {
pcfgs := map[string]csplugin.PluginConfig{}
wf := func(path string, info fs.FileInfo, err error) error {
if info == nil {
@ -78,6 +83,7 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
return fmt.Errorf("loading notifification plugin configuration with %s: %w", name, err)
}
for _, t := range ts {
csplugin.SetRequiredFields(&t)
pcfgs[t.Name] = t
}
}
@ -87,8 +93,15 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
if err := filepath.Walk(csConfig.ConfigPaths.NotificationDir, wf); err != nil {
return nil, fmt.Errorf("while loading notifification plugin configuration: %w", err)
}
return pcfgs, nil
}
func getProfilesConfigs() (map[string]NotificationsCfg, error) {
// A bit of a tricky stuf now: reconcile profiles and notification plugins
pcfgs, err := getPluginConfigs()
if err != nil {
return nil, err
}
ncfgs := map[string]NotificationsCfg{}
profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
if err != nil {
@ -131,13 +144,13 @@ func getNotificationsConfiguration() (map[string]NotificationsCfg, error) {
func NewNotificationsListCmd() *cobra.Command {
var cmdNotificationsList = &cobra.Command{
Use: "list",
Short: "List active notifications plugins",
Long: `List active notifications plugins`,
Short: "list active notifications plugins",
Long: `list active notifications plugins`,
Example: `cscli notifications list`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, arg []string) error {
ncfgs, err := getNotificationsConfiguration()
ncfgs, err := getProfilesConfigs()
if err != nil {
return fmt.Errorf("can't build profiles configuration: %w", err)
}
@ -183,25 +196,21 @@ func NewNotificationsInspectCmd() *cobra.Command {
Example: `cscli notifications inspect <plugin_name>`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, arg []string) error {
var (
cfg NotificationsCfg
ok bool
)
pluginName := arg[0]
if pluginName == "" {
PreRunE: func(cmd *cobra.Command, args []string) error {
if args[0] == "" {
return fmt.Errorf("please provide a plugin name to inspect")
}
ncfgs, err := getNotificationsConfiguration()
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
ncfgs, err := getProfilesConfigs()
if err != nil {
return fmt.Errorf("can't build profiles configuration: %w", err)
}
if cfg, ok = ncfgs[pluginName]; !ok {
return fmt.Errorf("plugin '%s' does not exist or is not active", pluginName)
cfg, ok := ncfgs[args[0]]
if !ok {
return fmt.Errorf("plugin '%s' does not exist or is not active", args[0])
}
if csConfig.Cscli.Output == "human" || csConfig.Cscli.Output == "raw" {
fmt.Printf(" - %15s: %15s\n", "Type", cfg.Config.Type)
fmt.Printf(" - %15s: %15s\n", "Name", cfg.Config.Name)
@ -224,75 +233,125 @@ func NewNotificationsInspectCmd() *cobra.Command {
return cmdNotificationsInspect
}
func NewNotificationsTestCmd() *cobra.Command {
var (
pluginBroker csplugin.PluginBroker
pluginTomb tomb.Tomb
alertOverride string
)
var cmdNotificationsTest = &cobra.Command{
Use: "test [plugin name]",
Short: "send a generic test alert to notification plugin",
Long: `send a generic test alert to a notification plugin to test configuration even if is not active`,
Example: `cscli notifications test [plugin_name]`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
PreRunE: func(cmd *cobra.Command, args []string) error {
pconfigs, err := getPluginConfigs()
if err != nil {
return fmt.Errorf("can't build profiles configuration: %w", err)
}
cfg, ok := pconfigs[args[0]]
if !ok {
return fmt.Errorf("plugin name: '%s' does not exist", args[0])
}
//Create a single profile with plugin name as notification name
return pluginBroker.Init(csConfig.PluginConfig, []*csconfig.ProfileCfg{
{
Notifications: []string{
cfg.Name,
},
},
}, csConfig.ConfigPaths)
},
RunE: func(cmd *cobra.Command, args []string) error {
pluginTomb.Go(func() error {
pluginBroker.Run(&pluginTomb)
return nil
})
alert := &models.Alert{
Capacity: ptr.Of(int32(0)),
Decisions: []*models.Decision{{
Duration: ptr.Of("4h"),
Scope: ptr.Of("Ip"),
Value: ptr.Of("10.10.10.10"),
Type: ptr.Of("ban"),
Scenario: ptr.Of("test alert"),
Origin: ptr.Of(types.CscliOrigin),
}},
Events: []*models.Event{},
EventsCount: ptr.Of(int32(1)),
Leakspeed: ptr.Of("0"),
Message: ptr.Of("test alert"),
ScenarioHash: ptr.Of(""),
Scenario: ptr.Of("test alert"),
ScenarioVersion: ptr.Of(""),
Simulated: ptr.Of(false),
Source: &models.Source{
AsName: "",
AsNumber: "",
Cn: "",
IP: "10.10.10.10",
Range: "",
Scope: ptr.Of("Ip"),
Value: ptr.Of("10.10.10.10"),
},
StartAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
StopAt: ptr.Of(time.Now().UTC().Format(time.RFC3339)),
CreatedAt: time.Now().UTC().Format(time.RFC3339),
}
if err := yaml.Unmarshal([]byte(alertOverride), alert); err != nil {
return fmt.Errorf("failed to unmarshal alert override: %w", err)
}
pluginBroker.PluginChannel <- csplugin.ProfileAlert{
ProfileID: uint(0),
Alert: alert,
}
//time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
pluginTomb.Kill(fmt.Errorf("terminating"))
pluginTomb.Wait()
return nil
},
}
cmdNotificationsTest.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the generic alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
return cmdNotificationsTest
}
func NewNotificationsReinjectCmd() *cobra.Command {
var remediation bool
var alertOverride string
var alert *models.Alert
var cmdNotificationsReinject = &cobra.Command{
Use: "reinject",
Short: "reinject alert into notifications system",
Long: `Reinject alert into notifications system`,
Short: "reinject an alert into profiles to trigger notifications",
Long: `reinject an alert into profiles to be evaluated by the filter and sent to matched notifications plugins`,
Example: `
cscli notifications reinject <alert_id>
cscli notifications reinject <alert_id> --remediation
cscli notifications reinject <alert_id> -a '{"remediation": false,"scenario":"notification/test"}'
cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"notification/test"}'
`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
PreRunE: func(cmd *cobra.Command, args []string) error {
var err error
alert, err = FetchAlertFromArgString(args[0])
if err != nil {
return err
}
return nil
},
RunE: func(cmd *cobra.Command, args []string) error {
var (
pluginBroker csplugin.PluginBroker
pluginTomb tomb.Tomb
)
if len(args) != 1 {
printHelp(cmd)
return fmt.Errorf("wrong number of argument: there should be one argument")
}
//first: get the alert
id, err := strconv.Atoi(args[0])
if err != nil {
return fmt.Errorf("bad alert id %s", args[0])
}
if err := csConfig.LoadAPIClient(); err != nil {
return fmt.Errorf("loading api client: %w", err)
}
if csConfig.API.Client == nil {
return fmt.Errorf("missing configuration on 'api_client:'")
}
if csConfig.API.Client.Credentials == nil {
return fmt.Errorf("missing API credentials in '%s'", csConfig.API.Client.CredentialsFilePath)
}
apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
if err != nil {
return fmt.Errorf("error parsing the URL of the API: %w", err)
}
client, err := apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
Password: strfmt.Password(csConfig.API.Client.Credentials.Password),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
})
if err != nil {
return fmt.Errorf("error creating the client for the API: %w", err)
}
alert, _, err := client.Alerts.GetByID(context.Background(), id)
if err != nil {
return fmt.Errorf("can't find alert with id %s: %w", args[0], err)
}
if alertOverride != "" {
if err = json.Unmarshal([]byte(alertOverride), alert); err != nil {
if err := json.Unmarshal([]byte(alertOverride), alert); err != nil {
return fmt.Errorf("can't unmarshal data in the alert flag: %w", err)
}
}
if !remediation {
alert.Remediation = true
}
// second we start plugins
err = pluginBroker.Init(csConfig.PluginConfig, csConfig.API.Server.Profiles, csConfig.ConfigPaths)
err := pluginBroker.Init(csConfig.PluginConfig, csConfig.API.Server.Profiles, csConfig.ConfigPaths)
if err != nil {
return fmt.Errorf("can't initialize plugins: %w", err)
}
@ -302,8 +361,6 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
return nil
})
//third: get the profile(s), and process the whole stuff
profiles, err := csprofiles.NewProfile(csConfig.API.Server.Profiles)
if err != nil {
return fmt.Errorf("cannot extract profiles from configuration: %w", err)
@ -338,15 +395,39 @@ cscli notifications reinject <alert_id> -a '{"remediation": true,"scenario":"not
break
}
}
// time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
//time.Sleep(2 * time.Second) // There's no mechanism to ensure notification has been sent
pluginTomb.Kill(fmt.Errorf("terminating"))
pluginTomb.Wait()
return nil
},
}
cmdNotificationsReinject.Flags().BoolVarP(&remediation, "remediation", "r", false, "Set Alert.Remediation to false in the reinjected alert (see your profile filter configuration)")
cmdNotificationsReinject.Flags().StringVarP(&alertOverride, "alert", "a", "", "JSON string used to override alert fields in the reinjected alert (see crowdsec/pkg/models/alert.go in the source tree for the full definition of the object)")
return cmdNotificationsReinject
}
func FetchAlertFromArgString(toParse string) (*models.Alert, error) {
id, err := strconv.Atoi(toParse)
if err != nil {
return nil, fmt.Errorf("bad alert id %s", toParse)
}
apiURL, err := url.Parse(csConfig.API.Client.Credentials.URL)
if err != nil {
return nil, fmt.Errorf("error parsing the URL of the API: %w", err)
}
client, err := apiclient.NewClient(&apiclient.Config{
MachineID: csConfig.API.Client.Credentials.Login,
Password: strfmt.Password(csConfig.API.Client.Credentials.Password),
UserAgent: fmt.Sprintf("crowdsec/%s", version.String()),
URL: apiURL,
VersionPrefix: "v1",
})
if err != nil {
return nil, fmt.Errorf("error creating the client for the API: %w", err)
}
alert, _, err := client.Alerts.GetByID(context.Background(), id)
if err != nil {
return nil, fmt.Errorf("can't find alert with id %d: %w", id, err)
}
return alert, nil
}

5
cmd/crowdsec-cli/utils.go
View file

@ -13,14 +13,12 @@ import (
)
func printHelp(cmd *cobra.Command) {
err := cmd.Help()
if err != nil {
if err := cmd.Help(); err != nil {
log.Fatalf("unable to print help(): %s", err)
}
}
func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
/*if a range is provided, change the scope*/
if *ipRange != "" {
_, _, err := net.ParseCIDR(*ipRange)
@ -50,7 +48,6 @@ func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *
}
func getDBClient() (*database.Client, error) {
var err error
if err := csConfig.LoadAPIServer(); err != nil || csConfig.DisableAPI {
return nil, err
}

15
cmd/crowdsec/output.go
View file

@ -146,13 +146,6 @@ LOOP:
}
break LOOP
case event := <-overflow:
//if the Alert is nil, it's to signal bucket is ready for GC, don't track this
if dumpStates && event.Overflow.Alert != nil {
if bucketOverflows == nil {
bucketOverflows = make([]types.Event, 0)
}
bucketOverflows = append(bucketOverflows, event)
}
/*if alert is empty and mapKey is present, the overflow is just to cleanup bucket*/
if event.Overflow.Alert == nil && event.Overflow.Mapkey != "" {
buckets.Bucket_map.Delete(event.Overflow.Mapkey)
@ -164,6 +157,14 @@ LOOP:
return fmt.Errorf("postoverflow failed : %s", err)
}
log.Printf("%s", *event.Overflow.Alert.Message)
//if the Alert is nil, it's to signal bucket is ready for GC, don't track this
//dump after postoveflow processing to avoid missing whitelist info
if dumpStates && event.Overflow.Alert != nil {
if bucketOverflows == nil {
bucketOverflows = make([]types.Event, 0)
}
bucketOverflows = append(bucketOverflows, event)
}
if event.Overflow.Whitelisted {
log.Printf("[%s] is whitelisted, skip.", *event.Overflow.Alert.Message)
continue

29
pkg/apiserver/apic.go
View file

@ -618,12 +618,23 @@ func (a *apic) PullTop(forcePull bool) error {
}
// update blocklists
if err := a.UpdateBlocklists(data.Links, add_counters); err != nil {
if err := a.UpdateBlocklists(data.Links, add_counters, forcePull); err != nil {
return fmt.Errorf("while updating blocklists: %w", err)
}
return nil
}
// we receive a link to a blocklist, we pull the content of the blocklist and we create one alert
func (a *apic) PullBlocklist(blocklist *modelscapi.BlocklistLink, forcePull bool) error {
add_counters, _ := makeAddAndDeleteCounters()
if err := a.UpdateBlocklists(&modelscapi.GetDecisionsStreamResponseLinks{
Blocklists: []*modelscapi.BlocklistLink{blocklist},
}, add_counters, forcePull); err != nil {
return fmt.Errorf("while pulling blocklist: %w", err)
}
return nil
}
// if decisions is whitelisted: return representation of the whitelist ip or cidr
// if not whitelisted: empty string
func (a *apic) whitelistedBy(decision *models.Decision) string {
@ -710,7 +721,7 @@ func (a *apic) ShouldForcePullBlocklist(blocklist *modelscapi.BlocklistLink) (bo
return false, nil
}
func (a *apic) updateBlocklist(client *apiclient.ApiClient, blocklist *modelscapi.BlocklistLink, add_counters map[string]map[string]int) error {
func (a *apic) updateBlocklist(client *apiclient.ApiClient, blocklist *modelscapi.BlocklistLink, add_counters map[string]map[string]int, forcePull bool) error {
if blocklist.Scope == nil {
log.Warningf("blocklist has no scope")
return nil
@ -719,12 +730,16 @@ func (a *apic) updateBlocklist(client *apiclient.ApiClient, blocklist *modelscap
log.Warningf("blocklist has no duration")
return nil
}
forcePull, err := a.ShouldForcePullBlocklist(blocklist)
if err != nil {
return fmt.Errorf("while checking if we should force pull blocklist %s: %w", *blocklist.Name, err)
if !forcePull {
_forcePull, err := a.ShouldForcePullBlocklist(blocklist)
if err != nil {
return fmt.Errorf("while checking if we should force pull blocklist %s: %w", *blocklist.Name, err)
}
forcePull = _forcePull
}
blocklistConfigItemName := fmt.Sprintf("blocklist:%s:last_pull", *blocklist.Name)
var lastPullTimestamp *string
var err error
if !forcePull {
lastPullTimestamp, err = a.dbClient.GetConfigItem(blocklistConfigItemName)
if err != nil {
@ -764,7 +779,7 @@ func (a *apic) updateBlocklist(client *apiclient.ApiClient, blocklist *modelscap
return nil
}
func (a *apic) UpdateBlocklists(links *modelscapi.GetDecisionsStreamResponseLinks, add_counters map[string]map[string]int) error {
func (a *apic) UpdateBlocklists(links *modelscapi.GetDecisionsStreamResponseLinks, add_counters map[string]map[string]int, forcePull bool) error {
if links == nil {
return nil
}
@ -778,7 +793,7 @@ func (a *apic) UpdateBlocklists(links *modelscapi.GetDecisionsStreamResponseLink
return fmt.Errorf("while creating default client: %w", err)
}
for _, blocklist := range links.Blocklists {
if err := a.updateBlocklist(defaultClient, blocklist, add_counters); err != nil {
if err := a.updateBlocklist(defaultClient, blocklist, add_counters, forcePull); err != nil {
return err
}
}

10
pkg/apiserver/apic_metrics.go
View file

@ -26,8 +26,8 @@ func (a *apic) GetMetrics() (*models.Metrics, error) {
machinesInfo[i] = &models.MetricsAgentInfo{
Version: machine.Version,
Name: machine.MachineId,
LastUpdate: machine.UpdatedAt.String(),
LastPush: ptr.OrEmpty(machine.LastPush).String(),
LastUpdate: machine.UpdatedAt.Format(time.RFC3339),
LastPush: ptr.OrEmpty(machine.LastPush).Format(time.RFC3339),
}
}
@ -43,7 +43,7 @@ func (a *apic) GetMetrics() (*models.Metrics, error) {
Version: bouncer.Version,
CustomName: bouncer.Name,
Name: bouncer.Type,
LastPull: bouncer.LastPull.String(),
LastPull: bouncer.LastPull.Format(time.RFC3339),
}
}
@ -81,7 +81,7 @@ func (a *apic) SendMetrics(stop chan (bool)) {
const checkInt = 20 * time.Second
// intervals must always be > 0
metInts := []time.Duration{1*time.Millisecond, a.metricsIntervalFirst, a.metricsInterval}
metInts := []time.Duration{1 * time.Millisecond, a.metricsIntervalFirst, a.metricsInterval}
log.Infof("Start sending metrics to CrowdSec Central API (interval: %s once, then %s)",
metInts[1].Round(time.Second), metInts[2])
@ -123,7 +123,7 @@ func (a *apic) SendMetrics(stop chan (bool)) {
reloadMachineIDs()
if !slices.Equal(oldIDs, machineIDs) {
log.Infof("capi metrics: machines changed, immediate send")
metTicker.Reset(1*time.Millisecond)
metTicker.Reset(1 * time.Millisecond)
}
case <-metTicker.C:
metTicker.Stop()

49
pkg/apiserver/apic_test.go
View file

@ -309,30 +309,30 @@ func TestAPICGetMetrics(t *testing.T) {
Bouncers: []*models.MetricsBouncerInfo{
{
CustomName: "1",
LastPull: time.Time{}.String(),
LastPull: time.Time{}.Format(time.RFC3339),
}, {
CustomName: "2",
LastPull: time.Time{}.String(),
LastPull: time.Time{}.Format(time.RFC3339),
}, {
CustomName: "3",
LastPull: time.Time{}.String(),
LastPull: time.Time{}.Format(time.RFC3339),
},
},
Machines: []*models.MetricsAgentInfo{
{
Name: "a",
LastPush: time.Time{}.String(),
LastUpdate: time.Time{}.String(),
LastPush: time.Time{}.Format(time.RFC3339),
LastUpdate: time.Time{}.Format(time.RFC3339),
},
{
Name: "b",
LastPush: time.Time{}.String(),
LastUpdate: time.Time{}.String(),
LastPush: time.Time{}.Format(time.RFC3339),
LastUpdate: time.Time{}.Format(time.RFC3339),
},
{
Name: "c",
LastPush: time.Time{}.String(),
LastUpdate: time.Time{}.String(),
LastPush: time.Time{}.Format(time.RFC3339),
LastUpdate: time.Time{}.Format(time.RFC3339),
},
},
},
@ -973,6 +973,37 @@ func TestAPICPullTopBLCacheForceCall(t *testing.T) {
require.NoError(t, err)
}
func TestAPICPullBlocklistCall(t *testing.T) {
api := getAPIC(t)
httpmock.Activate()
defer httpmock.DeactivateAndReset()
httpmock.RegisterResponder("GET", "http://api.crowdsec.net/blocklist1", func(req *http.Request) (*http.Response, error) {
assert.Equal(t, "", req.Header.Get("If-Modified-Since"))
return httpmock.NewStringResponse(200, "1.2.3.4"), nil
})
url, err := url.ParseRequestURI("http://api.crowdsec.net/")
require.NoError(t, err)
apic, err := apiclient.NewDefaultClient(
url,
"/api",
fmt.Sprintf("crowdsec/%s", version.String()),
nil,
)
require.NoError(t, err)
api.apiClient = apic
err = api.PullBlocklist(&modelscapi.BlocklistLink{
URL: ptr.Of("http://api.crowdsec.net/blocklist1"),
Name: ptr.Of("blocklist1"),
Scope: ptr.Of("Ip"),
Remediation: ptr.Of("ban"),
Duration: ptr.Of("24h"),
}, true)
require.NoError(t, err)
}
func TestAPICPush(t *testing.T) {
tests := []struct {
name string

48
pkg/apiserver/papi_cmd.go
View file

@ -11,6 +11,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/apiclient"
"github.com/crowdsecurity/crowdsec/pkg/models"
"github.com/crowdsecurity/crowdsec/pkg/modelscapi"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
@ -19,6 +20,23 @@ type deleteDecisions struct {
Decisions []string `json:"decisions"`
}
type blocklistLink struct {
// blocklist name
Name string `json:"name"`
// blocklist url
Url string `json:"url"`
// blocklist remediation
Remediation string `json:"remediation"`
// blocklist scope
Scope string `json:"scope,omitempty"`
// blocklist duration
Duration string `json:"duration,omitempty"`
}
type forcePull struct {
Blocklist *blocklistLink `json:"blocklist,omitempty"`
}
func DecisionCmd(message *Message, p *Papi, sync bool) error {
switch message.Header.OperationCmd {
case "delete":
@ -144,11 +162,35 @@ func ManagementCmd(message *Message, p *Papi, sync bool) error {
log.Infof("Received reauth command from PAPI, resetting token")
p.apiClient.GetClient().Transport.(*apiclient.JWTTransport).ResetToken()
case "force_pull":
log.Infof("Received force_pull command from PAPI, pulling community and 3rd-party blocklists")
err := p.apic.PullTop(true)
data, err := json.Marshal(message.Data)
if err != nil {
return fmt.Errorf("failed to force pull operation: %s", err)
return err
}
forcePullMsg := forcePull{}
if err := json.Unmarshal(data, &forcePullMsg); err != nil {
return fmt.Errorf("message for '%s' contains bad data format: %s", message.Header.OperationType, err)
}
if forcePullMsg.Blocklist == nil {
log.Infof("Received force_pull command from PAPI, pulling community and 3rd-party blocklists")
err = p.apic.PullTop(true)
if err != nil {
return fmt.Errorf("failed to force pull operation: %s", err)
}
} else {
log.Infof("Received force_pull command from PAPI, pulling blocklist %s", forcePullMsg.Blocklist.Name)
err = p.apic.PullBlocklist(&modelscapi.BlocklistLink{
Name: &forcePullMsg.Blocklist.Name,
URL: &forcePullMsg.Blocklist.Url,
Remediation: &forcePullMsg.Blocklist.Remediation,
Scope: &forcePullMsg.Blocklist.Scope,
Duration: &forcePullMsg.Blocklist.Duration,
}, true)
if err != nil {
return fmt.Errorf("failed to force pull operation: %s", err)
}
}
default:
return fmt.Errorf("unknown command '%s' for operation type '%s'", message.Header.OperationCmd, message.Header.OperationType)
}

4
pkg/csplugin/broker.go
View file

@ -192,7 +192,7 @@ func (pb *PluginBroker) loadConfig(path string) error {
return err
}
for _, pluginConfig := range pluginConfigs {
setRequiredFields(&pluginConfig)
SetRequiredFields(&pluginConfig)
if _, ok := pb.pluginConfigByName[pluginConfig.Name]; ok {
log.Warningf("notification '%s' is defined multiple times", pluginConfig.Name)
}
@ -376,7 +376,7 @@ func ParsePluginConfigFile(path string) ([]PluginConfig, error) {
return parsedConfigs, nil
}
func setRequiredFields(pluginCfg *PluginConfig) {
func SetRequiredFields(pluginCfg *PluginConfig) {
if pluginCfg.MaxRetry == 0 {
pluginCfg.MaxRetry++
}

8
pkg/cwhub/cwhub.go
View file

@ -4,6 +4,7 @@ import (
"fmt"
"net/http"
"path/filepath"
"sort"
"strings"
"time"
)
@ -30,3 +31,10 @@ func safePath(dir, filePath string) (string, error) {
return absFilePath, nil
}
// SortItemSlice sorts a slice of items by name, case insensitive.
func SortItemSlice(items []*Item) {
sort.Slice(items, func(i, j int) bool {
return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name)
})
}

63
pkg/cwhub/dataset.go
View file

@ -6,9 +6,10 @@ import (
"io"
"net/http"
"os"
"time"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/pkg/types"
)
@ -51,6 +52,62 @@ func downloadFile(url string, destPath string) error {
return nil
}
// needsUpdate checks if a data file has to be downloaded (or updated).
// if the local file doesn't exist, update.
// if the remote is newer than the local file, update.
// if the remote has no modification date, but local file has been modified > a week ago, update.
func needsUpdate(destPath string, url string) bool {
fileInfo, err := os.Stat(destPath)
switch {
case os.IsNotExist(err):
return true
case err != nil:
log.Errorf("while getting %s: %s", destPath, err)
return true
}
resp, err := hubClient.Head(url)
if err != nil {
log.Errorf("while getting %s: %s", url, err)
// Head failed, Get would likely fail too -> no update
return false
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
log.Errorf("bad http code %d for %s", resp.StatusCode, url)
return false
}
// update if local file is older than this
shelfLife := 7 * 24 * time.Hour
lastModify := fileInfo.ModTime()
localIsOld := lastModify.Add(shelfLife).Before(time.Now())
remoteLastModified := resp.Header.Get("Last-Modified")
if remoteLastModified == "" {
if localIsOld {
log.Infof("no last modified date for %s, but local file is older than %s", url, shelfLife)
}
return localIsOld
}
lastAvailable, err := time.Parse(time.RFC1123, remoteLastModified)
if err != nil {
log.Warningf("while parsing last modified date for %s: %s", url, err)
return localIsOld
}
if lastModify.Before(lastAvailable) {
log.Infof("new version available, updating %s", destPath)
return true
}
return false
}
// downloadDataSet downloads all the data files for an item.
func downloadDataSet(dataFolder string, force bool, reader io.Reader) error {
dec := yaml.NewDecoder(reader)
@ -72,9 +129,7 @@ func downloadDataSet(dataFolder string, force bool, reader io.Reader) error {
return err
}
if _, err := os.Stat(destPath); os.IsNotExist(err) || force {
log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath)
if force || needsUpdate(destPath, dataS.SourceURL) {
if err := downloadFile(dataS.SourceURL, destPath); err != nil {
return fmt.Errorf("while getting data: %w", err)
}

190
pkg/cwhub/enable.go
View file

@ -1,190 +0,0 @@
package cwhub
// Enable/disable items already downloaded
import (
"fmt"
"os"
"path/filepath"
log "github.com/sirupsen/logrus"
)
// installPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local
// (eg. /etc/crowdsec/collections/xyz.yaml).
// Raises an error if the path goes outside of the install dir.
func (i *Item) installPath() (string, error) {
p := i.Type
if i.Stage != "" {
p = filepath.Join(p, i.Stage)
}
return safePath(i.hub.local.InstallDir, filepath.Join(p, i.FileName))
}
// downloadPath returns the location of the actual config file in the hub
// (eg. /etc/crowdsec/hub/collections/author/xyz.yaml).
// Raises an error if the path goes outside of the hub dir.
func (i *Item) downloadPath() (string, error) {
ret, err := safePath(i.hub.local.HubDir, i.RemotePath)
if err != nil {
return "", err
}
return ret, nil
}
// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir.
func (i *Item) createInstallLink() error {
dest, err := i.installPath()
if err != nil {
return err
}
destDir := filepath.Dir(dest)
if err = os.MkdirAll(destDir, os.ModePerm); err != nil {
return fmt.Errorf("while creating %s: %w", destDir, err)
}
if _, err = os.Lstat(dest); !os.IsNotExist(err) {
log.Infof("%s already exists.", dest)
return nil
}
src, err := i.downloadPath()
if err != nil {
return err
}
if err = os.Symlink(src, dest); err != nil {
return fmt.Errorf("while creating symlink from %s to %s: %w", src, dest, err)
}
return nil
}
// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items.
func (i *Item) enable() error {
if i.State.Installed {
if i.State.Tainted {
return fmt.Errorf("%s is tainted, won't enable unless --force", i.Name)
}
if i.IsLocal() {
return fmt.Errorf("%s is local, won't enable", i.Name)
}
// if it's a collection, check sub-items even if the collection file itself is up-to-date
if i.State.UpToDate && !i.HasSubItems() {
log.Tracef("%s is installed and up-to-date, skip.", i.Name)
return nil
}
}
for _, sub := range i.SubItems() {
if err := sub.enable(); err != nil {
return fmt.Errorf("while installing %s: %w", sub.Name, err)
}
}
if err := i.createInstallLink(); err != nil {
return err
}
log.Infof("Enabled %s: %s", i.Type, i.Name)
i.State.Installed = true
return nil
}
// purge removes the actual config file that was downloaded.
func (i *Item) purge() error {
if !i.State.Downloaded {
log.Infof("removing %s: not downloaded -- no need to remove", i.Name)
return nil
}
src, err := i.downloadPath()
if err != nil {
return err
}
if err := os.Remove(src); err != nil {
if os.IsNotExist(err) {
log.Debugf("%s doesn't exist, no need to remove", src)
return nil
}
return fmt.Errorf("while removing file: %w", err)
}
i.State.Downloaded = false
log.Infof("Removed source file [%s]: %s", i.Name, src)
return nil
}
// removeInstallLink removes the symlink to the downloaded content.
func (i *Item) removeInstallLink() error {
syml, err := i.installPath()
if err != nil {
return err
}
stat, err := os.Lstat(syml)
if err != nil {
return err
}
// if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ...
if stat.Mode()&os.ModeSymlink == 0 {
log.Warningf("%s (%s) isn't a symlink, can't disable", i.Name, syml)
return fmt.Errorf("%s isn't managed by hub", i.Name)
}
hubpath, err := os.Readlink(syml)
if err != nil {
return fmt.Errorf("while reading symlink: %w", err)
}
src, err := i.downloadPath()
if err != nil {
return err
}
if hubpath != src {
log.Warningf("%s (%s) isn't a symlink to %s", i.Name, syml, src)
return fmt.Errorf("%s isn't managed by hub", i.Name)
}
if err := os.Remove(syml); err != nil {
return fmt.Errorf("while removing symlink: %w", err)
}
log.Infof("Removed symlink [%s]: %s", i.Name, syml)
return nil
}
// disable removes the install link, and optionally the downloaded content.
func (i *Item) disable(purge bool, force bool) error {
err := i.removeInstallLink()
if os.IsNotExist(err) {
if !purge && !force {
link, _ := i.installPath()
return fmt.Errorf("link %s does not exist (override with --force or --purge)", link)
}
} else if err != nil {
return err
}
i.State.Installed = false
if purge {
if err := i.purge(); err != nil {
return err
}
}
return nil
}

79
pkg/cwhub/hub.go
View file

@ -159,3 +159,82 @@ func (h *Hub) updateIndex() error {
return nil
}
// GetItemMap returns the map of items for a given type.
func (h *Hub) GetItemMap(itemType string) map[string]*Item {
return h.Items[itemType]
}
// GetItem returns an item from hub based on its type and full name (author/name).
func (h *Hub) GetItem(itemType string, itemName string) *Item {
return h.GetItemMap(itemType)[itemName]
}
// GetItemNames returns a slice of (full) item names for a given type
// (eg. for collections: crowdsecurity/apache2 crowdsecurity/nginx).
func (h *Hub) GetItemNames(itemType string) []string {
m := h.GetItemMap(itemType)
if m == nil {
return nil
}
names := make([]string, 0, len(m))
for k := range m {
names = append(names, k)
}
return names
}
// GetAllItems returns a slice of all the items of a given type, installed or not.
func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
items, ok := h.Items[itemType]
if !ok {
return nil, fmt.Errorf("no %s in the hub index", itemType)
}
ret := make([]*Item, len(items))
idx := 0
for _, item := range items {
ret[idx] = item
idx++
}
return ret, nil
}
// GetInstalledItems returns a slice of the installed items of a given type.
func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
items, ok := h.Items[itemType]
if !ok {
return nil, fmt.Errorf("no %s in the hub index", itemType)
}
retItems := make([]*Item, 0)
for _, item := range items {
if item.State.Installed {
retItems = append(retItems, item)
}
}
return retItems, nil
}
// GetInstalledItemNames returns the names of the installed items of a given type.
func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) {
items, err := h.GetInstalledItems(itemType)
if err != nil {
return nil, err
}
retStr := make([]string, len(items))
for idx, it := range items {
retStr[idx] = it.Name
}
return retStr, nil
}

161
pkg/cwhub/items.go → pkg/cwhub/item.go
View file

@ -3,8 +3,7 @@ package cwhub
import (
"encoding/json"
"fmt"
"sort"
"strings"
"path/filepath"
"github.com/Masterminds/semver/v3"
"github.com/enescakir/emoji"
@ -70,9 +69,9 @@ type Item struct {
Author string `json:"author,omitempty" yaml:"author,omitempty"`
References []string `json:"references,omitempty" yaml:"references,omitempty"`
RemotePath string `json:"path,omitempty" yaml:"remote_path,omitempty"` // path relative to the base URL eg. /parsers/stage/author/file.yaml
Version string `json:"version,omitempty" yaml:"version,omitempty"` // the last available version
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // all the known versions
RemotePath string `json:"path,omitempty" yaml:"path,omitempty"` // path relative to the base URL eg. /parsers/stage/author/file.yaml
Version string `json:"version,omitempty" yaml:"version,omitempty"` // the last available version
Versions map[string]ItemVersion `json:"versions,omitempty" yaml:"-"` // all the known versions
// if it's a collection, it can have sub items
Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"`
@ -83,6 +82,30 @@ type Item struct {
WaapRules []string `json:"waap-rules,omitempty" yaml:"waap-rules,omitempty"`
}
// installPath returns the location of the symlink to the item in the hub, or the path of the item itself if it's local
// (eg. /etc/crowdsec/collections/xyz.yaml).
// Raises an error if the path goes outside of the install dir.
func (i *Item) installPath() (string, error) {
p := i.Type
if i.Stage != "" {
p = filepath.Join(p, i.Stage)
}
return safePath(i.hub.local.InstallDir, filepath.Join(p, i.FileName))
}
// downloadPath returns the location of the actual config file in the hub
// (eg. /etc/crowdsec/hub/collections/author/xyz.yaml).
// Raises an error if the path goes outside of the hub dir.
func (i *Item) downloadPath() (string, error) {
ret, err := safePath(i.hub.local.HubDir, i.RemotePath)
if err != nil {
return "", err
}
return ret, nil
}
// HasSubItems returns true if items of this type can have sub-items. Currently only collections.
func (i *Item) HasSubItems() bool {
return i.Type == COLLECTIONS
@ -259,6 +282,48 @@ func (i *Item) Ancestors() []*Item {
return ret
}
// descendants returns a list of all (direct or indirect) dependencies of the item.
func (i *Item) descendants() ([]*Item, error) {
var collectSubItems func(item *Item, visited map[*Item]bool, result *[]*Item) error
collectSubItems = func(item *Item, visited map[*Item]bool, result *[]*Item) error {
if item == nil {
return nil
}
if visited[item] {
return nil
}
visited[item] = true
for _, subItem := range item.SubItems() {
if subItem == i {
return fmt.Errorf("circular dependency detected: %s depends on %s", item.Name, i.Name)
}
*result = append(*result, subItem)
err := collectSubItems(subItem, visited, result)
if err != nil {
return err
}
}
return nil
}
ret := []*Item{}
visited := map[*Item]bool{}
err := collectSubItems(i, visited, &ret)
if err != nil {
return nil, err
}
return ret, nil
}
// InstallStatus returns the status of the item as a string and an emoji
// (eg. "enabled,update-available" and emoji.Warning).
func (i *Item) InstallStatus() (string, emoji.Emoji) {
@ -329,89 +394,3 @@ func (i *Item) versionStatus() int {
func (i *Item) validPath(dirName, fileName string) bool {
return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml")
}
// GetItemMap returns the map of items for a given type.
func (h *Hub) GetItemMap(itemType string) map[string]*Item {
return h.Items[itemType]
}
// GetItem returns an item from hub based on its type and full name (author/name).
func (h *Hub) GetItem(itemType string, itemName string) *Item {
return h.GetItemMap(itemType)[itemName]
}
// GetItemNames returns a slice of (full) item names for a given type
// (eg. for collections: crowdsecurity/apache2 crowdsecurity/nginx).
func (h *Hub) GetItemNames(itemType string) []string {
m := h.GetItemMap(itemType)
if m == nil {
return nil
}
names := make([]string, 0, len(m))
for k := range m {
names = append(names, k)
}
return names
}
// GetAllItems returns a slice of all the items of a given type, installed or not.
func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
items, ok := h.Items[itemType]
if !ok {
return nil, fmt.Errorf("no %s in the hub index", itemType)
}
ret := make([]*Item, len(items))
idx := 0
for _, item := range items {
ret[idx] = item
idx++
}
return ret, nil
}
// GetInstalledItems returns a slice of the installed items of a given type.
func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
items, ok := h.Items[itemType]
if !ok {
return nil, fmt.Errorf("no %s in the hub index", itemType)
}
retItems := make([]*Item, 0)
for _, item := range items {
if item.State.Installed {
retItems = append(retItems, item)
}
}
return retItems, nil
}
// GetInstalledItemNames returns the names of the installed items of a given type.
func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) {
items, err := h.GetInstalledItems(itemType)
if err != nil {
return nil, err
}
retStr := make([]string, len(items))
for idx, it := range items {
retStr[idx] = it.Name
}
return retStr, nil
}
// SortItemSlice sorts a slice of items by name, case insensitive.
func SortItemSlice(items []*Item) {
sort.Slice(items, func(i, j int) bool {
return strings.ToLower(items[i].Name) < strings.ToLower(items[j].Name)
})
}

0
pkg/cwhub/items_test.go → pkg/cwhub/item_test.go
View file

70
pkg/cwhub/iteminstall.go Normal file
View file

@ -0,0 +1,70 @@
package cwhub
import (
"fmt"
log "github.com/sirupsen/logrus"
)
// enable enables the item by creating a symlink to the downloaded content, and also enables sub-items.
func (i *Item) enable() error {
if i.State.Installed {
if i.State.Tainted {
return fmt.Errorf("%s is tainted, won't enable unless --force", i.Name)
}
if i.IsLocal() {
return fmt.Errorf("%s is local, won't enable", i.Name)
}
// if it's a collection, check sub-items even if the collection file itself is up-to-date
if i.State.UpToDate && !i.HasSubItems() {
log.Tracef("%s is installed and up-to-date, skip.", i.Name)
return nil
}
}
for _, sub := range i.SubItems() {
if err := sub.enable(); err != nil {
return fmt.Errorf("while installing %s: %w", sub.Name, err)
}
}
if err := i.createInstallLink(); err != nil {
return err
}
log.Infof("Enabled %s: %s", i.Type, i.Name)
i.State.Installed = true
return nil
}
// Install installs the item from the hub, downloading it if needed.
func (i *Item) Install(force bool, downloadOnly bool) error {
if downloadOnly && i.State.Downloaded && i.State.UpToDate {
log.Infof("%s is already downloaded and up-to-date", i.Name)
if !force {
return nil
}
}
filePath, err := i.downloadLatest(force, true)
if err != nil {
return fmt.Errorf("while downloading %s: %w", i.Name, err)
}
if downloadOnly {
log.Infof("Downloaded %s to %s", i.Name, filePath)
return nil
}
if err := i.enable(); err != nil {
return fmt.Errorf("while enabling %s: %w", i.Name, err)
}
log.Infof("Enabled %s", i.Name)
return nil
}

4
pkg/cwhub/enable_test.go → pkg/cwhub/iteminstall_test.go
View file

@ -63,7 +63,7 @@ func testDisable(hub *Hub, t *testing.T, item *Item) {
assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name)
// Remove
err := item.disable(false, false)
_, err := item.disable(false, false)
require.NoError(t, err, "failed to disable %s", item.Name)
// Local sync and check status
@ -76,7 +76,7 @@ func testDisable(hub *Hub, t *testing.T, item *Item) {
assert.True(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should still be downloaded", item.Name)
// Purge
err = item.disable(true, false)
_, err = item.disable(true, false)
require.NoError(t, err, "failed to purge %s", item.Name)
// Local sync and check status

80
pkg/cwhub/itemlink.go Normal file
View file

@ -0,0 +1,80 @@
package cwhub
import (
"fmt"
"os"
"path/filepath"
log "github.com/sirupsen/logrus"
)
// createInstallLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir.
func (i *Item) createInstallLink() error {
dest, err := i.installPath()
if err != nil {
return err
}
destDir := filepath.Dir(dest)
if err = os.MkdirAll(destDir, os.ModePerm); err != nil {
return fmt.Errorf("while creating %s: %w", destDir, err)
}
if _, err = os.Lstat(dest); !os.IsNotExist(err) {
log.Infof("%s already exists.", dest)
return nil
}
src, err := i.downloadPath()
if err != nil {
return err
}
if err = os.Symlink(src, dest); err != nil {
return fmt.Errorf("while creating symlink from %s to %s: %w", src, dest, err)
}
return nil
}
// removeInstallLink removes the symlink to the downloaded content.
func (i *Item) removeInstallLink() error {
syml, err := i.installPath()
if err != nil {
return err
}
stat, err := os.Lstat(syml)
if err != nil {
return err
}
// if it's managed by hub, it's a symlink to csconfig.GConfig.hub.HubDir / ...
if stat.Mode()&os.ModeSymlink == 0 {
log.Warningf("%s (%s) isn't a symlink, can't disable", i.Name, syml)
return fmt.Errorf("%s isn't managed by hub", i.Name)
}
hubpath, err := os.Readlink(syml)
if err != nil {
return fmt.Errorf("while reading symlink: %w", err)
}
src, err := i.downloadPath()
if err != nil {
return err
}
if hubpath != src {
log.Warningf("%s (%s) isn't a symlink to %s", i.Name, syml, src)
return fmt.Errorf("%s isn't managed by hub", i.Name)
}
if err := os.Remove(syml); err != nil {
return fmt.Errorf("while removing symlink: %w", err)
}
log.Infof("Removed symlink [%s]: %s", i.Name, syml)
return nil
}

139
pkg/cwhub/itemremove.go Normal file
View file

@ -0,0 +1,139 @@
package cwhub
import (
"fmt"
"os"
log "github.com/sirupsen/logrus"
"slices"
)
// purge removes the actual config file that was downloaded.
func (i *Item) purge() (bool, error) {
if !i.State.Downloaded {
log.Debugf("removing %s: not downloaded -- no need to remove", i.Name)
return false, nil
}
src, err := i.downloadPath()
if err != nil {
return false, err
}
if err := os.Remove(src); err != nil {
if os.IsNotExist(err) {
log.Debugf("%s doesn't exist, no need to remove", src)
return false, nil
}
return false, fmt.Errorf("while removing file: %w", err)
}
i.State.Downloaded = false
log.Infof("Removed source file [%s]: %s", i.Name, src)
return true, nil
}
// disable removes the install link, and optionally the downloaded content.
func (i *Item) disable(purge bool, force bool) (bool, error) {
didRemove := true
err := i.removeInstallLink()
if os.IsNotExist(err) {
if !purge && !force {
link, _ := i.installPath()
return false, fmt.Errorf("link %s does not exist (override with --force or --purge)", link)
}
didRemove = false
} else if err != nil {
return false, err
}
i.State.Installed = false
didPurge := false
if purge {
if didPurge, err = i.purge(); err != nil {
return didRemove, err
}
}
ret := didRemove || didPurge
return ret, nil
}
// Remove disables the item, optionally removing the downloaded content.
func (i *Item) Remove(purge bool, force bool) (bool, error) {
if i.IsLocal() {
log.Warningf("%s is a local item, please delete manually", i.Name)
return false, nil
}
if i.State.Tainted && !force {
return false, fmt.Errorf("%s is tainted, use '--force' to remove", i.Name)
}
if !i.State.Installed && !purge {
log.Infof("removing %s: not installed -- no need to remove", i.Name)
return false, nil
}
removed := false
descendants, err := i.descendants()
if err != nil {
return false, err
}
ancestors := i.Ancestors()
for _, sub := range i.SubItems() {
if !sub.State.Installed {
continue
}
// if the sub depends on a collection that is not a direct or indirect dependency
// of the current item, it is not removed
for _, subParent := range sub.Ancestors() {
if !purge && !subParent.State.Installed {
continue
}
// the ancestor that would block the removal of the sub item is also an ancestor
// of the item we are removing, so we don't want false warnings
// (e.g. crowdsecurity/sshd-logs was not removed because it also belongs to crowdsecurity/linux,
// while we are removing crowdsecurity/sshd)
if slices.Contains(ancestors, subParent) {
continue
}
// the sub-item belongs to the item we are removing, but we already knew that
if subParent == i {
continue
}
if !slices.Contains(descendants, subParent) {
log.Infof("%s was not removed because it also belongs to %s", sub.Name, subParent.Name)
continue
}
}
subRemoved, err := sub.Remove(purge, force)
if err != nil {
return false, fmt.Errorf("unable to disable %s: %w", i.Name, err)
}
removed = removed || subRemoved
}
didDisable, err := i.disable(purge, force)
if err != nil {
return false, fmt.Errorf("while removing %s: %w", i.Name, err)
}
removed = removed || didDisable
return removed, nil
}

154
pkg/cwhub/helpers.go → pkg/cwhub/itemupgrade.go
View file

@ -14,156 +14,17 @@ import (
"github.com/enescakir/emoji"
log "github.com/sirupsen/logrus"
"slices"
)
// Install installs the item from the hub, downloading it if needed.
func (i *Item) Install(force bool, downloadOnly bool) error {
if downloadOnly && i.State.Downloaded && i.State.UpToDate {
log.Infof("%s is already downloaded and up-to-date", i.Name)
if !force {
return nil
}
}
filePath, err := i.downloadLatest(force, true)
if err != nil {
return fmt.Errorf("while downloading %s: %w", i.Name, err)
}
if downloadOnly {
log.Infof("Downloaded %s to %s", i.Name, filePath)
return nil
}
if err := i.enable(); err != nil {
return fmt.Errorf("while enabling %s: %w", i.Name, err)
}
log.Infof("Enabled %s", i.Name)
return nil
}
// descendants returns a list of all (direct or indirect) dependencies of the item.
func (i *Item) descendants() ([]*Item, error) {
var collectSubItems func(item *Item, visited map[*Item]bool, result *[]*Item) error
collectSubItems = func(item *Item, visited map[*Item]bool, result *[]*Item) error {
if item == nil {
return nil
}
if visited[item] {
return nil
}
visited[item] = true
for _, subItem := range item.SubItems() {
if subItem == i {
return fmt.Errorf("circular dependency detected: %s depends on %s", item.Name, i.Name)
}
*result = append(*result, subItem)
err := collectSubItems(subItem, visited, result)
if err != nil {
return err
}
}
return nil
}
ret := []*Item{}
visited := map[*Item]bool{}
err := collectSubItems(i, visited, &ret)
if err != nil {
return nil, err
}
return ret, nil
}
// Remove disables the item, optionally removing the downloaded content.
func (i *Item) Remove(purge bool, force bool) (bool, error) {
if i.IsLocal() {
return false, fmt.Errorf("%s isn't managed by hub. Please delete manually", i.Name)
}
if i.State.Tainted && !force {
return false, fmt.Errorf("%s is tainted, use '--force' to remove", i.Name)
}
if !i.State.Installed && !purge {
log.Infof("removing %s: not installed -- no need to remove", i.Name)
return false, nil
}
removed := false
descendants, err := i.descendants()
if err != nil {
return false, err
}
ancestors := i.Ancestors()
for _, sub := range i.SubItems() {
if !sub.State.Installed {
continue
}
// if the sub depends on a collection that is not a direct or indirect dependency
// of the current item, it is not removed
for _, subParent := range sub.Ancestors() {
if !purge && !subParent.State.Installed {
continue
}
// the ancestor that would block the removal of the sub item is also an ancestor
// of the item we are removing, so we don't want false warnings
// (e.g. crowdsecurity/sshd-logs was not removed because it also belongs to crowdsecurity/linux,
// while we are removing crowdsecurity/sshd)
if slices.Contains(ancestors, subParent) {
continue
}
// the sub-item belongs to the item we are removing, but we already knew that
if subParent == i {
continue
}
if !slices.Contains(descendants, subParent) {
log.Infof("%s was not removed because it also belongs to %s", sub.Name, subParent.Name)
continue
}
}
subRemoved, err := sub.Remove(purge, force)
if err != nil {
return false, fmt.Errorf("unable to disable %s: %w", i.Name, err)
}
removed = removed || subRemoved
}
if err = i.disable(purge, force); err != nil {
return false, fmt.Errorf("while removing %s: %w", i.Name, err)
}
removed = true
return removed, nil
}
// Upgrade downloads and applies the last version of the item from the hub.
func (i *Item) Upgrade(force bool) (bool, error) {
updated := false
if i.IsLocal() {
log.Infof("not upgrading %s: local item", i.Name)
return false, nil
}
if !i.State.Downloaded {
return false, fmt.Errorf("can't upgrade %s: not installed", i.Name)
}
@ -192,8 +53,6 @@ func (i *Item) Upgrade(force bool) (bool, error) {
if !i.State.UpToDate {
if i.State.Tainted {
log.Warningf("%v %s is tainted, --force to overwrite", emoji.Warning, i.Name)
} else if i.IsLocal() {
log.Infof("%v %s is local", emoji.Prohibited, i.Name)
}
} else {
// a check on stdout is used while scripting to know if the hub has been upgraded
@ -296,6 +155,9 @@ func (i *Item) fetch() ([]byte, error) {
// download downloads the item from the hub and writes it to the hub directory.
func (i *Item) download(overwrite bool) (string, error) {
if i.IsLocal() {
return "", fmt.Errorf("%s is local, can't download", i.Name)
}
// if user didn't --force, don't overwrite local, tainted, up-to-date files
if !overwrite {
if i.State.Tainted {

0
pkg/cwhub/helpers_test.go → pkg/cwhub/itemupgrade_test.go
View file

2
pkg/cwhub/sync.go
View file

@ -30,7 +30,7 @@ func linkTarget(path string) (string, error) {
_, err = os.Lstat(hubpath)
if os.IsNotExist(err) {
log.Infof("link target does not exist: %s -> %s", path, hubpath)
log.Warningf("link target does not exist: %s -> %s", path, hubpath)
return "", nil
}

7
test/bats/20_hub.bats
View file

@ -119,3 +119,10 @@ teardown() {
# this is used by the cron script to know if the hub was updated
assert_output --partial "updated crowdsecurity/syslog-logs"
}
@test "cscli hub upgrade (with local items)" {
mkdir -p "$CONFIG_DIR/collections"
touch "$CONFIG_DIR/collections/foo.yaml"
rune -0 cscli hub upgrade
assert_stderr --partial "not upgrading foo.yaml: local item"
}

7
test/bats/20_hub_collections.bats
View file

@ -208,7 +208,7 @@ teardown() {
assert_line 'type: collections'
assert_line 'name: crowdsecurity/sshd'
assert_line 'author: crowdsecurity'
assert_line 'remote_path: collections/crowdsecurity/sshd.yaml'
assert_line 'path: collections/crowdsecurity/sshd.yaml'
assert_line 'installed: false'
refute_line --partial 'Current metrics:'
@ -226,7 +226,7 @@ teardown() {
assert_line 'type: collections'
assert_line 'name: crowdsecurity/sshd'
assert_line 'author: crowdsecurity'
assert_line 'remote_path: collections/crowdsecurity/sshd.yaml'
assert_line 'path: collections/crowdsecurity/sshd.yaml'
assert_line 'installed: false'
refute_line --partial 'Current metrics:'
@ -275,8 +275,9 @@ teardown() {
rune -0 cscli collections remove crowdsecurity/sshd
assert_stderr --partial 'removing crowdsecurity/sshd: not installed -- no need to remove'
rune -0 cscli collections remove crowdsecurity/sshd --purge
rune -0 cscli collections remove crowdsecurity/sshd --purge --debug
assert_stderr --partial 'removing crowdsecurity/sshd: not downloaded -- no need to remove'
refute_stderr --partial 'Removed source file [crowdsecurity/sshd]'
# install, then remove, check files
rune -0 cscli collections install crowdsecurity/sshd

36
test/bats/20_hub_items.bats
View file

@ -99,7 +99,7 @@ teardown() {
rune -0 jq -r '.path' <(output)
rune -0 rm "$HUB_DIR/$(output)"
rune -0 cscli parsers remove crowdsecurity/syslog-logs --purge
rune -0 cscli parsers remove crowdsecurity/syslog-logs --purge --debug
assert_stderr --partial "removing crowdsecurity/syslog-logs: not downloaded -- no need to remove"
rune -0 cscli parsers remove crowdsecurity/linux --all --error --purge --force
@ -147,3 +147,37 @@ teardown() {
rune -0 cscli collections inspect hi-its-me -o json
rune -0 jq -e '[.installed,.local]==[true,true]' <(output)
}
@test "a local item cannot be downloaded by cscli" {
rune -0 mkdir -p "$CONFIG_DIR/collections"
rune -0 touch "$CONFIG_DIR/collections/foobar.yaml"
rune -1 cscli collections install foobar.yaml
assert_stderr --partial "failed to download item: foobar.yaml is local, can't download"
rune -1 cscli collections install foobar.yaml --force
assert_stderr --partial "failed to download item: foobar.yaml is local, can't download"
}
@test "a local item cannot be removed by cscli" {
rune -0 mkdir -p "$CONFIG_DIR/collections"
rune -0 touch "$CONFIG_DIR/collections/foobar.yaml"
rune -0 cscli collections remove foobar.yaml
assert_stderr --partial "foobar.yaml is a local item, please delete manually"
rune -0 cscli collections remove foobar.yaml --purge
assert_stderr --partial "foobar.yaml is a local item, please delete manually"
rune -0 cscli collections remove foobar.yaml --force
assert_stderr --partial "foobar.yaml is a local item, please delete manually"
rune -0 cscli collections remove --all
assert_stderr --partial "foobar.yaml is a local item, please delete manually"
rune -0 cscli collections remove --all --purge
assert_stderr --partial "foobar.yaml is a local item, please delete manually"
}
@test "a dangling link is reported with a warning" {
rune -0 mkdir -p "$CONFIG_DIR/collections"
rune -0 ln -s /this/does/not/exist.yaml "$CONFIG_DIR/collections/foobar.yaml"
rune -0 cscli hub list
assert_stderr --partial "link target does not exist: $CONFIG_DIR/collections/foobar.yaml -> /this/does/not/exist.yaml"
rune -0 cscli hub list -o json
rune -0 jq '.collections' <(output)
assert_json '[]'
}

7
test/bats/20_hub_parsers.bats
View file

@ -209,7 +209,7 @@ teardown() {
assert_line 'stage: s01-parse'
assert_line 'name: crowdsecurity/sshd-logs'
assert_line 'author: crowdsecurity'
assert_line 'remote_path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml'
assert_line 'path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml'
assert_line 'installed: false'
refute_line --partial 'Current metrics:'
@ -228,7 +228,7 @@ teardown() {
assert_line 'name: crowdsecurity/sshd-logs'
assert_line 'stage: s01-parse'
assert_line 'author: crowdsecurity'
assert_line 'remote_path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml'
assert_line 'path: parsers/s01-parse/crowdsecurity/sshd-logs.yaml'
assert_line 'installed: false'
refute_line --partial 'Current metrics:'
@ -277,8 +277,9 @@ teardown() {
rune -0 cscli parsers remove crowdsecurity/whitelists
assert_stderr --partial "removing crowdsecurity/whitelists: not installed -- no need to remove"
rune -0 cscli parsers remove crowdsecurity/whitelists --purge
rune -0 cscli parsers remove crowdsecurity/whitelists --purge --debug
assert_stderr --partial 'removing crowdsecurity/whitelists: not downloaded -- no need to remove'
refute_stderr --partial 'Removed source file [crowdsecurity/whitelists]'
# install, then remove, check files
rune -0 cscli parsers install crowdsecurity/whitelists

7
test/bats/20_hub_postoverflows.bats
View file

@ -209,7 +209,7 @@ teardown() {
assert_line 'stage: s00-enrich'
assert_line 'name: crowdsecurity/rdns'
assert_line 'author: crowdsecurity'
assert_line 'remote_path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml'
assert_line 'path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml'
assert_line 'installed: false'
refute_line --partial 'Current metrics:'
@ -228,7 +228,7 @@ teardown() {
assert_line 'name: crowdsecurity/rdns'
assert_line 'stage: s00-enrich'
assert_line 'author: crowdsecurity'
assert_line 'remote_path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml'
assert_line 'path: postoverflows/s00-enrich/crowdsecurity/rdns.yaml'
assert_line 'installed: false'
refute_line --partial 'Current metrics:'
@ -277,8 +277,9 @@ teardown() {
rune -0 cscli postoverflows remove crowdsecurity/rdns
assert_stderr --partial 'removing crowdsecurity/rdns: not installed -- no need to remove'
rune -0 cscli postoverflows remove crowdsecurity/rdns --purge
rune -0 cscli postoverflows remove crowdsecurity/rdns --purge --debug
assert_stderr --partial 'removing crowdsecurity/rdns: not downloaded -- no need to remove'
refute_stderr --partial 'Removed source file [crowdsecurity/rdns]'
# install, then remove, check files
rune -0 cscli postoverflows install crowdsecurity/rdns

7
test/bats/20_hub_scenarios.bats
View file

@ -209,7 +209,7 @@ teardown() {
assert_line 'type: scenarios'
assert_line 'name: crowdsecurity/ssh-bf'
assert_line 'author: crowdsecurity'
assert_line 'remote_path: scenarios/crowdsecurity/ssh-bf.yaml'
assert_line 'path: scenarios/crowdsecurity/ssh-bf.yaml'
assert_line 'installed: false'
refute_line --partial 'Current metrics:'
@ -227,7 +227,7 @@ teardown() {
assert_line 'type: scenarios'
assert_line 'name: crowdsecurity/ssh-bf'
assert_line 'author: crowdsecurity'
assert_line 'remote_path: scenarios/crowdsecurity/ssh-bf.yaml'
assert_line 'path: scenarios/crowdsecurity/ssh-bf.yaml'
assert_line 'installed: false'
refute_line --partial 'Current metrics:'
@ -276,8 +276,9 @@ teardown() {
rune -0 cscli scenarios remove crowdsecurity/ssh-bf
assert_stderr --partial "removing crowdsecurity/ssh-bf: not installed -- no need to remove"
rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge
rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge --debug
assert_stderr --partial 'removing crowdsecurity/ssh-bf: not downloaded -- no need to remove'
refute_stderr --partial 'Removed source file [crowdsecurity/ssh-bf]'
# install, then remove, check files
rune -0 cscli scenarios install crowdsecurity/ssh-bf