From 88e4f7c15755321230adb8c190ac39cf77b3dc22 Mon Sep 17 00:00:00 2001 From: mmetc <92726601+mmetc@users.noreply.github.com> Date: Thu, 19 Oct 2023 12:04:29 +0200 Subject: [PATCH] Refact pkg/csconfig, pkg/cwhub (#2555) * csconfig: drop redundant hub information on *Cfg structs * rename validItemFileName() -> item.validPath() * Methods on hub object * updated tests to reduce need of csconfig.Config or global state --- cmd/crowdsec-cli/capi.go | 5 +- cmd/crowdsec-cli/collections.go | 31 +++- cmd/crowdsec-cli/config_backup.go | 13 +- cmd/crowdsec-cli/config_restore.go | 27 ++-- cmd/crowdsec-cli/config_show.go | 1 - cmd/crowdsec-cli/console.go | 5 +- cmd/crowdsec-cli/hub.go | 29 ++-- cmd/crowdsec-cli/item_metrics.go | 6 +- cmd/crowdsec-cli/item_suggest.go | 18 ++- cmd/crowdsec-cli/items.go | 32 ++-- cmd/crowdsec-cli/lapi.go | 11 +- cmd/crowdsec-cli/parsers.go | 29 +++- cmd/crowdsec-cli/postoverflows.go | 29 +++- cmd/crowdsec-cli/require/require.go | 11 +- cmd/crowdsec-cli/scenarios.go | 29 +++- cmd/crowdsec-cli/simulation.go | 5 +- cmd/crowdsec-cli/support.go | 13 +- cmd/crowdsec-cli/utils_table.go | 4 +- cmd/crowdsec/crowdsec.go | 6 +- cmd/crowdsec/main.go | 6 +- cmd/crowdsec/output.go | 9 +- pkg/csconfig/config.go | 2 +- pkg/csconfig/crowdsec_service.go | 11 +- pkg/csconfig/crowdsec_service_test.go | 15 -- pkg/csconfig/cscli.go | 11 +- pkg/csconfig/cscli_test.go | 4 - pkg/csconfig/hub.go | 3 + pkg/cwhub/cwhub.go | 44 ++--- pkg/cwhub/cwhub_test.go | 225 +++++++++++++------------- pkg/cwhub/download.go | 72 +++++---- pkg/cwhub/download_test.go | 18 ++- pkg/cwhub/helpers.go | 39 +++-- pkg/cwhub/helpers_test.go | 166 +++++++++---------- pkg/cwhub/{hubindex.go => hub.go} | 40 +++-- pkg/cwhub/install.go | 36 ++--- pkg/cwhub/loader.go | 118 +++++++------- pkg/hubtest/hubtest.go | 4 +- pkg/hubtest/hubtest_item.go | 18 +-- pkg/leakybucket/buckets_test.go | 32 +++- pkg/leakybucket/manager_load.go | 11 +- pkg/leakybucket/tests/hub/index.json | 1 + pkg/parser/unix_parser.go | 14 +- pkg/setup/install.go | 11 +- 43 files changed, 669 insertions(+), 545 deletions(-) rename pkg/cwhub/{hubindex.go => hub.go} (76%) create mode 100644 pkg/leakybucket/tests/hub/index.json diff --git a/cmd/crowdsec-cli/capi.go b/cmd/crowdsec-cli/capi.go index 0e0f217aa..3f94d286c 100644 --- a/cmd/crowdsec-cli/capi.go +++ b/cmd/crowdsec-cli/capi.go @@ -151,11 +151,12 @@ func NewCapiStatusCmd() *cobra.Command { return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err) } - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig) + if err != nil { return err } - scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS) if err != nil { return fmt.Errorf("failed to get scenarios: %w", err) } diff --git a/cmd/crowdsec-cli/collections.go b/cmd/crowdsec-cli/collections.go index 158b80823..494cedab2 100644 --- a/cmd/crowdsec-cli/collections.go +++ b/cmd/crowdsec-cli/collections.go @@ -25,7 +25,7 @@ cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables Aliases: []string{"collection"}, DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { + if _, err := require.Hub(csConfig); err != nil { return err } @@ -66,8 +66,13 @@ func runCollectionsInstall(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + for _, name := range args { - t := cwhub.GetItem(cwhub.COLLECTIONS, name) + t := hub.GetItem(cwhub.COLLECTIONS, name) if t == nil { nearestItem, score := GetDistance(cwhub.COLLECTIONS, name) Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError) @@ -75,7 +80,7 @@ func runCollectionsInstall(cmd *cobra.Command, args []string) error { continue } - if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, force, downloadOnly); err != nil { + if err := hub.InstallItem(name, cwhub.COLLECTIONS, force, downloadOnly); err != nil { if !ignoreError { return fmt.Errorf("error while installing '%s': %w", name, err) } @@ -126,8 +131,13 @@ func runCollectionsRemove(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + if all { - err := cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, force) + err := hub.RemoveMany(cwhub.COLLECTIONS, "", all, purge, force) if err != nil { return err } @@ -141,7 +151,7 @@ func runCollectionsRemove(cmd *cobra.Command, args []string) error { for _, name := range args { if !force { - item := cwhub.GetItem(cwhub.COLLECTIONS, name) + item := hub.GetItem(cwhub.COLLECTIONS, name) if item == nil { // XXX: this should be in GetItem? return fmt.Errorf("can't find '%s' in %s", name, cwhub.COLLECTIONS) @@ -153,7 +163,7 @@ func runCollectionsRemove(cmd *cobra.Command, args []string) error { } } - err := cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, force) + err := hub.RemoveMany(cwhub.COLLECTIONS, name, all, purge, force) if err != nil { return err } @@ -197,8 +207,13 @@ func runCollectionsUpgrade(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + if all { - if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.COLLECTIONS, "", force); err != nil { return err } return nil @@ -209,7 +224,7 @@ func runCollectionsUpgrade(cmd *cobra.Command, args []string) error { } for _, name := range args { - if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, force); err != nil { + if err := hub.UpgradeConfig(cwhub.COLLECTIONS, name, force); err != nil { return err } } diff --git a/cmd/crowdsec-cli/config_backup.go b/cmd/crowdsec-cli/config_backup.go index 73d1aca03..417b21eda 100644 --- a/cmd/crowdsec-cli/config_backup.go +++ b/cmd/crowdsec-cli/config_backup.go @@ -13,7 +13,7 @@ import ( "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require" ) -func backupHub(dirPath string) error { +func backupHub(hub *cwhub.Hub, dirPath string) error { var err error var itemDirectory string var upstreamParsers []string @@ -22,7 +22,7 @@ func backupHub(dirPath string) error { clog := log.WithFields(log.Fields{ "type": itemType, }) - itemMap := cwhub.GetItemMap(itemType) + itemMap := hub.GetItemMap(itemType) if itemMap == nil { clog.Infof("No %s to backup.", itemType) continue @@ -189,7 +189,12 @@ func backupConfigToDirectory(dirPath string) error { log.Infof("Saved profiles to %s", backupProfiles) } - if err = backupHub(dirPath); err != nil { + hub, err := cwhub.GetHub() + if err != nil { + return err + } + + if err = backupHub(hub, dirPath); err != nil { return fmt.Errorf("failed to backup hub config: %s", err) } @@ -197,7 +202,7 @@ func backupConfigToDirectory(dirPath string) error { } func runConfigBackup(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { + if _, err := require.Hub(csConfig); err != nil { return err } diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index ccd9ebc5e..ba734461e 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -23,23 +23,28 @@ type OldAPICfg struct { // it's a rip of the cli version, but in silent-mode func silentInstallItem(name string, obtype string) (string, error) { - var item = cwhub.GetItem(obtype, name) - if item == nil { - return "", fmt.Errorf("error retrieving item") - } - err := cwhub.DownloadLatest(csConfig.Hub, item, false, false) + hub, err := cwhub.GetHub() if err != nil { - return "", fmt.Errorf("error while downloading %s : %v", item.Name, err) - } - if err := cwhub.AddItem(obtype, *item); err != nil { return "", err } - err = cwhub.EnableItem(csConfig.Hub, item) + var item = hub.GetItem(obtype, name) + if item == nil { + return "", fmt.Errorf("error retrieving item") + } + err = hub.DownloadLatest(item, false, false) + if err != nil { + return "", fmt.Errorf("error while downloading %s : %v", item.Name, err) + } + if err := hub.AddItem(obtype, *item); err != nil { + return "", err + } + + err = hub.EnableItem(item) if err != nil { return "", fmt.Errorf("error while enabling %s : %v", item.Name, err) } - if err := cwhub.AddItem(obtype, *item); err != nil { + if err := hub.AddItem(obtype, *item); err != nil { return "", err } return fmt.Sprintf("Enabled %s", item.Name), nil @@ -292,7 +297,7 @@ func runConfigRestore(cmd *cobra.Command, args []string) error { return err } - if err := require.Hub(csConfig); err != nil { + if _, err := require.Hub(csConfig); err != nil { return err } diff --git a/cmd/crowdsec-cli/config_show.go b/cmd/crowdsec-cli/config_show.go index 9f5b11fc1..ca7051195 100644 --- a/cmd/crowdsec-cli/config_show.go +++ b/cmd/crowdsec-cli/config_show.go @@ -82,7 +82,6 @@ Crowdsec{{if and .Crowdsec.Enable (not (ValueBool .Crowdsec.Enable))}} (disabled cscli: - Output : {{.Cscli.Output}} - Hub Branch : {{.Cscli.HubBranch}} - - Hub Folder : {{.Cscli.HubDir}} {{- end }} {{- if .API }} diff --git a/cmd/crowdsec-cli/console.go b/cmd/crowdsec-cli/console.go index 439a8143b..06eae5e56 100644 --- a/cmd/crowdsec-cli/console.go +++ b/cmd/crowdsec-cli/console.go @@ -71,11 +71,12 @@ After running this command your will need to validate the enrollment in the weba return fmt.Errorf("could not parse CAPI URL: %s", err) } - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig) + if err != nil { return err } - scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS) if err != nil { return fmt.Errorf("failed to get installed scenarios: %s", err) } diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index 501fbc498..c4ec140e9 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -50,12 +50,13 @@ func runHubList(cmd *cobra.Command, args []string) error { return err } - if err = require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig) + if err != nil { return err } // use LocalSync to get warnings about tainted / outdated items - warn, _ := cwhub.LocalSync(csConfig.Hub) + warn, _ := hub.LocalSync() for _, v := range warn { log.Info(v) } @@ -88,19 +89,24 @@ func NewHubListCmd() *cobra.Command { } func runHubUpdate(cmd *cobra.Command, args []string) error { - if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil { + cwhub.SetHubBranch() + + // don't use require.Hub because if there is no index file, it would fail + + hub, err := cwhub.InitHubUpdate(csConfig.Hub) + if err != nil { if !errors.Is(err, cwhub.ErrIndexNotFound) { return fmt.Errorf("failed to get Hub index : %w", err) } log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch) cwhub.HubBranch = "master" - if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil { + if hub, err = cwhub.InitHubUpdate(csConfig.Hub); err != nil { return fmt.Errorf("failed to get Hub index after retry: %w", err) } } // use LocalSync to get warnings about tainted / outdated items - warn, _ := cwhub.LocalSync(csConfig.Hub) + warn, _ := hub.LocalSync() for _, v := range warn { log.Info(v) } @@ -122,8 +128,6 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde return fmt.Errorf("you must configure cli before interacting with hub") } - cwhub.SetHubBranch() - return nil }, RunE: runHubUpdate, @@ -140,27 +144,28 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error { return err } - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig) + if err != nil { return err } log.Infof("Upgrading collections") - if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.COLLECTIONS, "", force); err != nil { return err } log.Infof("Upgrading parsers") - if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.PARSERS, "", force); err != nil { return err } log.Infof("Upgrading scenarios") - if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.SCENARIOS, "", force); err != nil { return err } log.Infof("Upgrading postoverflows") - if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, "", force); err != nil { return err } diff --git a/cmd/crowdsec-cli/item_metrics.go b/cmd/crowdsec-cli/item_metrics.go index ec925262e..924139b93 100644 --- a/cmd/crowdsec-cli/item_metrics.go +++ b/cmd/crowdsec-cli/item_metrics.go @@ -18,7 +18,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) -func ShowMetrics(hubItem *cwhub.Item) { +func ShowMetrics(hub *cwhub.Hub, hubItem *cwhub.Item) { switch hubItem.Type { case cwhub.PARSERS: metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name) @@ -36,11 +36,11 @@ func ShowMetrics(hubItem *cwhub.Item) { scenarioMetricsTable(color.Output, item, metrics) } for _, item := range hubItem.Collections { - hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item) + hubItem = hub.GetItem(cwhub.COLLECTIONS, item) if hubItem == nil { log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name) } - ShowMetrics(hubItem) + ShowMetrics(hub, hubItem) } default: log.Errorf("item of type '%s' is unknown", hubItem.Type) diff --git a/cmd/crowdsec-cli/item_suggest.go b/cmd/crowdsec-cli/item_suggest.go index c52239fb4..2b48cdc12 100644 --- a/cmd/crowdsec-cli/item_suggest.go +++ b/cmd/crowdsec-cli/item_suggest.go @@ -33,7 +33,11 @@ func GetDistance(itemType string, itemName string) (*cwhub.Item, int) { allItems := make([]string, 0) nearestScore := 100 nearestItem := &cwhub.Item{} - hubItems := cwhub.GetItemMap(itemType) + + // XXX: handle error + hub, _ := cwhub.GetHub() + + hubItems := hub.GetItemMap(itemType) for _, item := range hubItems { allItems = append(allItems, item.Name) } @@ -42,19 +46,20 @@ func GetDistance(itemType string, itemName string) (*cwhub.Item, int) { d := levenshtein.Distance(itemName, s, nil) if d < nearestScore { nearestScore = d - nearestItem = cwhub.GetItem(itemType, s) + nearestItem = hub.GetItem(itemType, s) } } return nearestItem, nearestScore } func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig) + if err != nil { return nil, cobra.ShellCompDirectiveDefault } comp := make([]string, 0) - hubItems := cwhub.GetItemMap(itemType) + hubItems := hub.GetItemMap(itemType) for _, item := range hubItems { if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) { comp = append(comp, item.Name) @@ -65,11 +70,12 @@ func compAllItems(itemType string, args []string, toComplete string) ([]string, } func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig) + if err != nil { return nil, cobra.ShellCompDirectiveDefault } - items, err := cwhub.GetInstalledItemsAsString(itemType) + items, err := hub.GetInstalledItemsAsString(itemType) if err != nil { cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true) return nil, cobra.ShellCompDirectiveDefault diff --git a/cmd/crowdsec-cli/items.go b/cmd/crowdsec-cli/items.go index cebf5b723..371cdc16d 100644 --- a/cmd/crowdsec-cli/items.go +++ b/cmd/crowdsec-cli/items.go @@ -16,8 +16,8 @@ import ( ) -func selectItems(itemType string, args []string, installedOnly bool) ([]string, error) { - itemNames := cwhub.GetItemNames(itemType) +func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]string, error) { + itemNames := hub.GetItemNames(itemType) notExist := []string{} if len(args) > 0 { @@ -40,7 +40,7 @@ func selectItems(itemType string, args []string, installedOnly bool) ([]string, if installedOnly { installed := []string{} for _, item := range itemNames { - if cwhub.GetItem(itemType, item).Installed { + if hub.GetItem(itemType, item).Installed { installed = append(installed, item) } } @@ -52,9 +52,15 @@ func selectItems(itemType string, args []string, installedOnly bool) ([]string, func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) error { var err error + + hub, err := cwhub.GetHub() + if err != nil { + return err + } + items := make(map[string][]string) for _, itemType := range itemTypes { - if items[itemType], err = selectItems(itemType, args, !all); err != nil { + if items[itemType], err = selectItems(hub, itemType, args, !all); err != nil { return err } } @@ -78,7 +84,7 @@ func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, // empty slice in case there are no items of this type hubStatus[itemType] = make([]itemHubStatus, len(items[itemType])) for i, itemName := range items[itemType] { - item := cwhub.GetItem(itemType, itemName) + item := hub.GetItem(itemType, itemName) status, emo := item.Status() hubStatus[itemType][i] = itemHubStatus{ Name: item.Name, @@ -112,7 +118,7 @@ func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, } for _, itemType := range itemTypes { for _, itemName := range items[itemType] { - item := cwhub.GetItem(itemType, itemName) + item := hub.GetItem(itemType, itemName) status, _ := item.Status() if item.LocalVersion == "" { item.LocalVersion = "n/a" @@ -138,15 +144,17 @@ func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, } func InspectItem(name string, itemType string, noMetrics bool) error { - hubItem := cwhub.GetItem(itemType, name) + hub, err := cwhub.GetHub() + if err != nil { + return err + } + + hubItem := hub.GetItem(itemType, name) if hubItem == nil { return fmt.Errorf("can't find '%s' in %s", name, itemType) } - var ( - b []byte - err error - ) + var b []byte switch csConfig.Cscli.Output { case "human", "raw": @@ -168,7 +176,7 @@ func InspectItem(name string, itemType string, noMetrics bool) error { } fmt.Printf("\nCurrent metrics: \n") - ShowMetrics(hubItem) + ShowMetrics(hub, hubItem) return nil } diff --git a/cmd/crowdsec-cli/lapi.go b/cmd/crowdsec-cli/lapi.go index 37ee0088c..aa43a3d6d 100644 --- a/cmd/crowdsec-cli/lapi.go +++ b/cmd/crowdsec-cli/lapi.go @@ -38,11 +38,12 @@ func runLapiStatus(cmd *cobra.Command, args []string) error { log.Fatalf("parsing api url ('%s'): %s", apiurl, err) } - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig) + if err != nil { log.Fatal(err) } - scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS) if err != nil { log.Fatalf("failed to get scenarios : %s", err) } @@ -338,9 +339,9 @@ cscli lapi context detect crowdsecurity/sshd-logs log.Fatalf("Failed to init expr helpers : %s", err) } - // Populate cwhub package tools - if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { - log.Fatalf("Failed to load hub index : %s", err) + _, err = require.Hub(csConfig) + if err != nil { + log.Fatal(err) } csParsers := parser.NewParsers() diff --git a/cmd/crowdsec-cli/parsers.go b/cmd/crowdsec-cli/parsers.go index 49e301a6c..e0d294e01 100644 --- a/cmd/crowdsec-cli/parsers.go +++ b/cmd/crowdsec-cli/parsers.go @@ -25,7 +25,7 @@ cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs Aliases: []string{"parser"}, DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { + if _, err := require.Hub(csConfig); err != nil { return err } @@ -66,8 +66,13 @@ func runParsersInstall(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + for _, name := range args { - t := cwhub.GetItem(cwhub.PARSERS, name) + t := hub.GetItem(cwhub.PARSERS, name) if t == nil { nearestItem, score := GetDistance(cwhub.PARSERS, name) Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError) @@ -75,7 +80,7 @@ func runParsersInstall(cmd *cobra.Command, args []string) error { continue } - if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, force, downloadOnly); err != nil { + if err := hub.InstallItem(name, cwhub.PARSERS, force, downloadOnly); err != nil { if !ignoreError { return fmt.Errorf("error while installing '%s': %w", name, err) } @@ -126,8 +131,13 @@ func runParsersRemove(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + if all { - err := cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, force) + err := hub.RemoveMany(cwhub.PARSERS, "", all, purge, force) if err != nil { return err } @@ -140,7 +150,7 @@ func runParsersRemove(cmd *cobra.Command, args []string) error { } for _, name := range args { - err := cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, force) + err := hub.RemoveMany(cwhub.PARSERS, name, all, purge, force) if err != nil { return err } @@ -184,8 +194,13 @@ func runParsersUpgrade(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + if all { - if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.PARSERS, "", force); err != nil { return err } return nil @@ -196,7 +211,7 @@ func runParsersUpgrade(cmd *cobra.Command, args []string) error { } for _, name := range args { - if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, force); err != nil { + if err := hub.UpgradeConfig(cwhub.PARSERS, name, force); err != nil { return err } } diff --git a/cmd/crowdsec-cli/postoverflows.go b/cmd/crowdsec-cli/postoverflows.go index 8c5f1b175..a73bf3da4 100644 --- a/cmd/crowdsec-cli/postoverflows.go +++ b/cmd/crowdsec-cli/postoverflows.go @@ -25,7 +25,7 @@ cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns Aliases: []string{"postoverflow"}, DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { + if _, err := require.Hub(csConfig); err != nil { return err } @@ -66,8 +66,13 @@ func runPostOverflowsInstall(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + for _, name := range args { - t := cwhub.GetItem(cwhub.POSTOVERFLOWS, name) + t := hub.GetItem(cwhub.POSTOVERFLOWS, name) if t == nil { nearestItem, score := GetDistance(cwhub.POSTOVERFLOWS, name) Suggest(cwhub.POSTOVERFLOWS, name, nearestItem.Name, score, ignoreError) @@ -75,7 +80,7 @@ func runPostOverflowsInstall(cmd *cobra.Command, args []string) error { continue } - if err := cwhub.InstallItem(csConfig, name, cwhub.POSTOVERFLOWS, force, downloadOnly); err != nil { + if err := hub.InstallItem(name, cwhub.POSTOVERFLOWS, force, downloadOnly); err != nil { if !ignoreError { return fmt.Errorf("error while installing '%s': %w", name, err) } @@ -126,8 +131,13 @@ func runPostOverflowsRemove(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + if all { - err := cwhub.RemoveMany(csConfig, cwhub.POSTOVERFLOWS, "", all, purge, force) + err := hub.RemoveMany(cwhub.POSTOVERFLOWS, "", all, purge, force) if err != nil { return err } @@ -140,7 +150,7 @@ func runPostOverflowsRemove(cmd *cobra.Command, args []string) error { } for _, name := range args { - err := cwhub.RemoveMany(csConfig, cwhub.POSTOVERFLOWS, name, all, purge, force) + err := hub.RemoveMany(cwhub.POSTOVERFLOWS, name, all, purge, force) if err != nil { return err } @@ -184,8 +194,13 @@ func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + if all { - if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, "", force); err != nil { return err } return nil @@ -196,7 +211,7 @@ func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error { } for _, name := range args { - if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, name, force); err != nil { + if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, name, force); err != nil { return err } } diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index fba59cb1e..b477e9e00 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -64,16 +64,17 @@ func Notifications(c *csconfig.Config) error { return nil } -func Hub (c *csconfig.Config) error { +func Hub (c *csconfig.Config) (*cwhub.Hub, error) { if c.Hub == nil { - return fmt.Errorf("you must configure cli before interacting with hub") + return nil, fmt.Errorf("you must configure cli before interacting with hub") } cwhub.SetHubBranch() - if err := cwhub.GetHubIdx(c.Hub); err != nil { - return fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err) + hub, err := cwhub.InitHub(c.Hub) + if err != nil { + return nil, fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err) } - return nil + return hub, nil } diff --git a/cmd/crowdsec-cli/scenarios.go b/cmd/crowdsec-cli/scenarios.go index 863e50236..23df3f5ff 100644 --- a/cmd/crowdsec-cli/scenarios.go +++ b/cmd/crowdsec-cli/scenarios.go @@ -25,7 +25,7 @@ cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing Aliases: []string{"scenario"}, DisableAutoGenTag: true, PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - if err := require.Hub(csConfig); err != nil { + if _, err := require.Hub(csConfig); err != nil { return err } @@ -66,8 +66,13 @@ func runScenariosInstall(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + for _, name := range args { - t := cwhub.GetItem(cwhub.SCENARIOS, name) + t := hub.GetItem(cwhub.SCENARIOS, name) if t == nil { nearestItem, score := GetDistance(cwhub.SCENARIOS, name) Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError) @@ -75,7 +80,7 @@ func runScenariosInstall(cmd *cobra.Command, args []string) error { continue } - if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, force, downloadOnly); err != nil { + if err := hub.InstallItem(name, cwhub.SCENARIOS, force, downloadOnly); err != nil { if !ignoreError { return fmt.Errorf("error while installing '%s': %w", name, err) } @@ -126,8 +131,13 @@ func runScenariosRemove(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + if all { - err := cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, force) + err := hub.RemoveMany(cwhub.SCENARIOS, "", all, purge, force) if err != nil { return err } @@ -140,7 +150,7 @@ func runScenariosRemove(cmd *cobra.Command, args []string) error { } for _, name := range args { - err := cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, force) + err := hub.RemoveMany(cwhub.SCENARIOS, name, all, purge, force) if err != nil { return err } @@ -184,8 +194,13 @@ func runScenariosUpgrade(cmd *cobra.Command, args []string) error { return err } + hub, err := cwhub.GetHub() + if err != nil { + return err + } + if all { - if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.SCENARIOS, "", force); err != nil { return err } return nil @@ -196,7 +211,7 @@ func runScenariosUpgrade(cmd *cobra.Command, args []string) error { } for _, name := range args { - if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, force); err != nil { + if err := hub.UpgradeConfig(cwhub.SCENARIOS, name, force); err != nil { return err } } diff --git a/cmd/crowdsec-cli/simulation.go b/cmd/crowdsec-cli/simulation.go index 890785a2d..0ad4ff5ae 100644 --- a/cmd/crowdsec-cli/simulation.go +++ b/cmd/crowdsec-cli/simulation.go @@ -145,13 +145,14 @@ func NewSimulationEnableCmd() *cobra.Command { Example: `cscli simulation enable`, DisableAutoGenTag: true, Run: func(cmd *cobra.Command, args []string) { - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig) + if err != nil { log.Fatal(err) } if len(args) > 0 { for _, scenario := range args { - var item = cwhub.GetItem(cwhub.SCENARIOS, scenario) + var item = hub.GetItem(cwhub.SCENARIOS, scenario) if item == nil { log.Errorf("'%s' doesn't exist or is not a scenario", scenario) continue diff --git a/cmd/crowdsec-cli/support.go b/cmd/crowdsec-cli/support.go index 0c87ad3c9..e8462c4ae 100644 --- a/cmd/crowdsec-cli/support.go +++ b/cmd/crowdsec-cli/support.go @@ -155,7 +155,7 @@ func collectAgents(dbClient *database.Client) ([]byte, error) { return out.Bytes(), nil } -func collectAPIStatus(login string, password string, endpoint string, prefix string) []byte { +func collectAPIStatus(login string, password string, endpoint string, prefix string, hub *cwhub.Hub) []byte { if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil { return []byte("No agent credentials found, are we LAPI ?") } @@ -165,7 +165,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str if err != nil { return []byte(fmt.Sprintf("cannot parse API URL: %s", err)) } - scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) + scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS) if err != nil { return []byte(fmt.Sprintf("could not collect scenarios: %s", err)) } @@ -293,7 +293,8 @@ cscli support dump -f /tmp/crowdsec-support.zip skipAgent = true } - if err := require.Hub(csConfig); err != nil { + hub, err := require.Hub(csConfig) + if err != nil { log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected") skipHub = true infos[SUPPORT_PARSERS_PATH] = []byte(err.Error()) @@ -356,7 +357,8 @@ cscli support dump -f /tmp/crowdsec-support.zip infos[SUPPORT_CAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Server.OnlineClient.Credentials.Login, csConfig.API.Server.OnlineClient.Credentials.Password, csConfig.API.Server.OnlineClient.Credentials.URL, - CAPIURLPrefix) + CAPIURLPrefix, + hub) } if !skipLAPI { @@ -364,7 +366,8 @@ cscli support dump -f /tmp/crowdsec-support.zip infos[SUPPORT_LAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Client.Credentials.Login, csConfig.API.Client.Credentials.Password, csConfig.API.Client.Credentials.URL, - LAPIURLPrefix) + LAPIURLPrefix, + hub) infos[SUPPORT_CROWDSEC_PROFILE_PATH] = collectCrowdsecProfile() } diff --git a/cmd/crowdsec-cli/utils_table.go b/cmd/crowdsec-cli/utils_table.go index 840250f9c..1f3f62038 100644 --- a/cmd/crowdsec-cli/utils_table.go +++ b/cmd/crowdsec-cli/utils_table.go @@ -16,8 +16,10 @@ func listHubItemTable(out io.Writer, title string, itemType string, itemNames [] t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft) + hub, _ := cwhub.GetHub() + for itemName := range itemNames { - item := cwhub.GetItem(itemType, itemNames[itemName]) + item := hub.GetItem(itemType, itemNames[itemName]) status, emo := item.Status() t.AddRow(item.Name, fmt.Sprintf("%v %s", emo, status), item.LocalVersion, item.LocalPath) } diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index c573cd4d4..dd81ad345 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -23,8 +23,8 @@ import ( func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { var err error - // Populate cwhub package tools - if err = cwhub.GetHubIdx(cConfig.Hub); err != nil { + hub, err := cwhub.InitHub(cConfig.Hub) + if err != nil { return nil, fmt.Errorf("while loading hub index: %w", err) } @@ -34,7 +34,7 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) { return nil, fmt.Errorf("while loading parsers: %w", err) } - if err := LoadBuckets(cConfig); err != nil { + if err := LoadBuckets(cConfig, hub); err != nil { return nil, fmt.Errorf("while loading scenarios: %w", err) } diff --git a/cmd/crowdsec/main.go b/cmd/crowdsec/main.go index e93dbef04..d4c2ceb90 100644 --- a/cmd/crowdsec/main.go +++ b/cmd/crowdsec/main.go @@ -75,12 +75,12 @@ type Flags struct { type labelsMap map[string]string -func LoadBuckets(cConfig *csconfig.Config) error { +func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error { var ( err error files []string ) - for _, hubScenarioItem := range cwhub.GetItemMap(cwhub.SCENARIOS) { + for _, hubScenarioItem := range hub.GetItemMap(cwhub.SCENARIOS) { if hubScenarioItem.Installed { files = append(files, hubScenarioItem.LocalPath) } @@ -88,7 +88,7 @@ func LoadBuckets(cConfig *csconfig.Config) error { buckets = leakybucket.NewBuckets() log.Infof("Loading %d scenario files", len(files)) - holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, files, &bucketsTomb, buckets, flags.OrderEvent) + holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, cConfig.ConfigPaths.DataDir, files, &bucketsTomb, buckets, flags.OrderEvent) if err != nil { return fmt.Errorf("scenario loading failed: %v", err) diff --git a/cmd/crowdsec/output.go b/cmd/crowdsec/output.go index 95642bbf3..95235aa9a 100644 --- a/cmd/crowdsec/output.go +++ b/cmd/crowdsec/output.go @@ -70,7 +70,12 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky var cache []types.RuntimeAlert var cacheMutex sync.Mutex - scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS) + hub, err := cwhub.GetHub() + if err != nil { + return err + } + + scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS) if err != nil { return fmt.Errorf("loading list of installed hub scenarios: %w", err) } @@ -93,7 +98,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky URL: apiURL, PapiURL: papiURL, VersionPrefix: "v1", - UpdateScenario: func() ([]string, error) {return cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)}, + UpdateScenario: func() ([]string, error) {return hub.GetInstalledItemsAsString(cwhub.SCENARIOS)}, }) if err != nil { return fmt.Errorf("new client api: %w", err) diff --git a/pkg/csconfig/config.go b/pkg/csconfig/config.go index 4a544729a..27bb32b41 100644 --- a/pkg/csconfig/config.go +++ b/pkg/csconfig/config.go @@ -100,7 +100,7 @@ func NewDefaultConfig() *Config { commonCfg := CommonCfg{ Daemonize: false, LogMedia: "stdout", - LogLevel: &logLevel, + LogLevel: &logLevel, } prometheus := PrometheusCfg{ Enabled: true, diff --git a/pkg/csconfig/crowdsec_service.go b/pkg/csconfig/crowdsec_service.go index f9602a5be..dc226cfd6 100644 --- a/pkg/csconfig/crowdsec_service.go +++ b/pkg/csconfig/crowdsec_service.go @@ -28,10 +28,6 @@ type CrowdsecServiceCfg struct { BucketStateDumpDir string `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown BucketsGCEnabled bool `yaml:"-"` // we need to garbage collect buckets when in forensic mode - HubDir string `yaml:"-"` - DataDir string `yaml:"-"` - ConfigDir string `yaml:"-"` - HubIndexFile string `yaml:"-"` SimulationFilePath string `yaml:"-"` ContextToSend map[string][]string `yaml:"-"` } @@ -101,11 +97,6 @@ func (c *Config) LoadCrowdsec() error { return fmt.Errorf("load error (simulation): %w", err) } - c.Crowdsec.ConfigDir = c.ConfigPaths.ConfigDir - c.Crowdsec.DataDir = c.ConfigPaths.DataDir - c.Crowdsec.HubDir = c.ConfigPaths.HubDir - c.Crowdsec.HubIndexFile = c.ConfigPaths.HubIndexFile - if c.Crowdsec.ParserRoutinesCount <= 0 { c.Crowdsec.ParserRoutinesCount = 1 } @@ -149,7 +140,7 @@ func (c *Config) LoadCrowdsec() error { fallback := false if c.Crowdsec.ConsoleContextPath == "" { // fallback to default config file - c.Crowdsec.ConsoleContextPath = filepath.Join(c.Crowdsec.ConfigDir, "console", "context.yaml") + c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml") fallback = true } diff --git a/pkg/csconfig/crowdsec_service_test.go b/pkg/csconfig/crowdsec_service_test.go index 06a7e91bd..e9d7e8de3 100644 --- a/pkg/csconfig/crowdsec_service_test.go +++ b/pkg/csconfig/crowdsec_service_test.go @@ -54,11 +54,6 @@ func TestLoadCrowdsec(t *testing.T) { AcquisitionDirPath: "", ConsoleContextPath: contextFileFullPath, AcquisitionFilePath: acquisFullPath, - ConfigDir: "./testdata", - DataDir: "./data", - HubDir: "./hub", - // XXX: need to ensure a default here - HubIndexFile: "", BucketsRoutinesCount: 1, ParserRoutinesCount: 1, OutputRoutinesCount: 1, @@ -98,11 +93,6 @@ func TestLoadCrowdsec(t *testing.T) { AcquisitionDirPath: acquisDirFullPath, AcquisitionFilePath: acquisFullPath, ConsoleContextPath: contextFileFullPath, - ConfigDir: "./testdata", - // XXX: need to ensure a default here - HubIndexFile: "", - DataDir: "./data", - HubDir: "./hub", BucketsRoutinesCount: 1, ParserRoutinesCount: 1, OutputRoutinesCount: 1, @@ -139,11 +129,6 @@ func TestLoadCrowdsec(t *testing.T) { Enable: ptr.Of(true), AcquisitionDirPath: "", AcquisitionFilePath: "", - ConfigDir: "./testdata", - // XXX: need to ensure a default here - HubIndexFile: "", - DataDir: "./data", - HubDir: "./hub", ConsoleContextPath: contextFileFullPath, BucketsRoutinesCount: 1, ParserRoutinesCount: 1, diff --git a/pkg/csconfig/cscli.go b/pkg/csconfig/cscli.go index 8db0a1848..6ecce4a53 100644 --- a/pkg/csconfig/cscli.go +++ b/pkg/csconfig/cscli.go @@ -11,10 +11,7 @@ type CscliCfg struct { HubBranch string `yaml:"hub_branch"` SimulationConfig *SimulationConfig `yaml:"-"` DbConfig *DatabaseCfg `yaml:"-"` - HubDir string `yaml:"-"` - DataDir string `yaml:"-"` - ConfigDir string `yaml:"-"` - HubIndexFile string `yaml:"-"` + SimulationFilePath string `yaml:"-"` PrometheusUrl string `yaml:"prometheus_uri"` } @@ -23,10 +20,8 @@ func (c *Config) loadCSCLI() error { if c.Cscli == nil { c.Cscli = &CscliCfg{} } - c.Cscli.ConfigDir = c.ConfigPaths.ConfigDir - c.Cscli.DataDir = c.ConfigPaths.DataDir - c.Cscli.HubDir = c.ConfigPaths.HubDir - c.Cscli.HubIndexFile = c.ConfigPaths.HubIndexFile + + // XXX: HubBranch default should be set here and fed to HubCfg? if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 { c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d/metrics", c.Prometheus.ListenAddr, c.Prometheus.ListenPort) diff --git a/pkg/csconfig/cscli_test.go b/pkg/csconfig/cscli_test.go index e928eda5a..b814fda88 100644 --- a/pkg/csconfig/cscli_test.go +++ b/pkg/csconfig/cscli_test.go @@ -32,10 +32,6 @@ func TestLoadCSCLI(t *testing.T) { }, }, expected: &CscliCfg{ - ConfigDir: "./testdata", - DataDir: "./data", - HubDir: "./hub", - HubIndexFile: "./hub/.index.json", PrometheusUrl: "http://127.0.0.1:6060/metrics", }, }, diff --git a/pkg/csconfig/hub.go b/pkg/csconfig/hub.go index 2164c19b2..db49c0231 100644 --- a/pkg/csconfig/hub.go +++ b/pkg/csconfig/hub.go @@ -9,6 +9,9 @@ type HubCfg struct { } func (c *Config) loadHub() error { + + // XXX: HubBranch too -- from cscli or chooseHubBranch() ? + c.Hub = &HubCfg{ HubIndexFile: c.ConfigPaths.HubIndexFile, HubDir: c.ConfigPaths.HubDir, diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index 580bf4e81..7ae805492 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -15,7 +15,6 @@ import ( "golang.org/x/mod/semver" ) - var ( ErrMissingReference = errors.New("Reference(s) missing in collection") @@ -40,7 +39,7 @@ type Item struct { FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .index.json Author string `json:"author,omitempty"` // as seen in .index.json - References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json + References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any // remote (hub) info @@ -112,9 +111,16 @@ func (i *Item) versionStatus() int { return semver.Compare("v"+i.Version, "v"+i.LocalVersion) } +// validPath returns true if the (relative) path is allowed for the item +// dirNmae: the directory name (ie. crowdsecurity) +// fileName: the filename (ie. apache2-logs.yaml) +func (i *Item) validPath(dirName, fileName string) bool { + return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml") +} + // GetItemMap returns the map of items for a given type -func GetItemMap(itemType string) map[string]Item { - m, ok := hubIdx.Items[itemType] +func (h *Hub) GetItemMap(itemType string) map[string]Item { + m, ok := h.Items[itemType] if !ok { return nil } @@ -123,6 +129,7 @@ func GetItemMap(itemType string) map[string]Item { } // itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary +// XXX: only used by leakybucket manager func itemKey(itemPath string) (string, error) { f, err := os.Lstat(itemPath) if err != nil { @@ -150,13 +157,13 @@ func itemKey(itemPath string) (string, error) { } // GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item. -func GetItemByPath(itemType string, itemPath string) (*Item, error) { +func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) { itemKey, err := itemKey(itemPath) if err != nil { return nil, err } - m := GetItemMap(itemType) + m := h.GetItemMap(itemType) if m == nil { return nil, fmt.Errorf("item type %s doesn't exist", itemType) } @@ -170,19 +177,20 @@ func GetItemByPath(itemType string, itemPath string) (*Item, error) { } // GetItem returns the item from hub based on its type and full name (author/name) -func GetItem(itemType string, itemName string) *Item { - if m, ok := GetItemMap(itemType)[itemName]; ok { - return &m +func (h *Hub) GetItem(itemType string, itemName string) *Item { + m, ok := h.GetItemMap(itemType)[itemName] + if !ok { + return nil } - return nil + return &m } // GetItemNames returns the list of item (full) names for a given type // ie. for parsers: crowdsecurity/apache2 crowdsecurity/nginx // The names can be used to retrieve the item with GetItem() -func GetItemNames(itemType string) []string { - m := GetItemMap(itemType) +func (h *Hub) GetItemNames(itemType string) []string { + m := h.GetItemMap(itemType) if m == nil { return nil } @@ -196,10 +204,10 @@ func GetItemNames(itemType string) []string { } // AddItem adds an item to the hub index -func AddItem(itemType string, item Item) error { +func (h *Hub) AddItem(itemType string, item Item) error { for _, itype := range ItemTypes { if itype == itemType { - hubIdx.Items[itemType][item.Name] = item + h.Items[itemType][item.Name] = item return nil } } @@ -208,8 +216,8 @@ func AddItem(itemType string, item Item) error { } // GetInstalledItems returns the list of installed items -func GetInstalledItems(itemType string) ([]Item, error) { - items, ok := hubIdx.Items[itemType] +func (h *Hub) GetInstalledItems(itemType string) ([]Item, error) { + items, ok := h.Items[itemType] if !ok { return nil, fmt.Errorf("no %s in hubIdx", itemType) } @@ -226,8 +234,8 @@ func GetInstalledItems(itemType string) ([]Item, error) { } // GetInstalledItemsAsString returns the names of the installed items -func GetInstalledItemsAsString(itemType string) ([]string, error) { - items, err := GetInstalledItems(itemType) +func (h *Hub) GetInstalledItemsAsString(itemType string) ([]string, error) { + items, err := h.GetInstalledItems(itemType) if err != nil { return nil, err } diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index 7f3a50a1a..216a0df4b 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -29,23 +29,15 @@ import ( var responseByPath map[string]string func TestItemStatus(t *testing.T) { - cfg := envSetup(t) - defer envTearDown(cfg) - - // DownloadHubIdx() - err := UpdateHubIdx(cfg.Hub) - require.NoError(t, err, "failed to download index") - - err = GetHubIdx(cfg.Hub) - require.NoError(t, err, "failed to load hub index") + hub := envSetup(t) // get existing map - x := GetItemMap(COLLECTIONS) + x := hub.GetItemMap(COLLECTIONS) require.NotEmpty(t, x) // Get item : good and bad for k := range x { - item := GetItem(COLLECTIONS, k) + item := hub.GetItem(COLLECTIONS, k) require.NotNil(t, item) item.Installed = true @@ -65,76 +57,101 @@ func TestItemStatus(t *testing.T) { require.Equal(t, "disabled,local", txt) } - DisplaySummary() + err := DisplaySummary() + require.NoError(t, err) } func TestGetters(t *testing.T) { - cfg := envSetup(t) - defer envTearDown(cfg) - - // DownloadHubIdx() - err := UpdateHubIdx(cfg.Hub) - require.NoError(t, err, "failed to download index") - - err = GetHubIdx(cfg.Hub) - require.NoError(t, err, "failed to load hub index") + hub := envSetup(t) // get non existing map - empty := GetItemMap("ratata") + empty := hub.GetItemMap("ratata") require.Nil(t, empty) // get existing map - x := GetItemMap(COLLECTIONS) + x := hub.GetItemMap(COLLECTIONS) require.NotEmpty(t, x) // Get item : good and bad for k := range x { - empty := GetItem(COLLECTIONS, k+"nope") + empty := hub.GetItem(COLLECTIONS, k+"nope") require.Nil(t, empty) - item := GetItem(COLLECTIONS, k) + item := hub.GetItem(COLLECTIONS, k) require.NotNil(t, item) // Add item and get it item.Name += "nope" - err := AddItem(COLLECTIONS, *item) + err := hub.AddItem(COLLECTIONS, *item) require.NoError(t, err) - newitem := GetItem(COLLECTIONS, item.Name) + newitem := hub.GetItem(COLLECTIONS, item.Name) require.NotNil(t, newitem) - err = AddItem("ratata", *item) + err = hub.AddItem("ratata", *item) cstest.RequireErrorContains(t, err, "ItemType ratata is unknown") } } func TestIndexDownload(t *testing.T) { - cfg := envSetup(t) - defer envTearDown(cfg) + hub := envSetup(t) - // DownloadHubIdx() - err := UpdateHubIdx(cfg.Hub) + _, err := InitHubUpdate(hub.cfg) require.NoError(t, err, "failed to download index") - err = GetHubIdx(cfg.Hub) + _, err = GetHub() require.NoError(t, err, "failed to load hub index") } -func getTestCfg() *csconfig.Config { - cfg := &csconfig.Config{Hub: &csconfig.HubCfg{}} - cfg.Hub.InstallDir, _ = filepath.Abs("./install") - cfg.Hub.HubDir, _ = filepath.Abs("./hubdir") - cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json") +// testHub initializes a temporary hub with an empty json file, optionally updating it +func testHub(t *testing.T, update bool) *Hub { + tmpDir, err := os.MkdirTemp("", "testhub") + require.NoError(t, err) - return cfg + hubCfg := &csconfig.HubCfg{ + HubDir: filepath.Join(tmpDir, "crowdsec", "hub"), + HubIndexFile: filepath.Join(tmpDir, "crowdsec", "hub", ".index.json"), + InstallDir: filepath.Join(tmpDir, "crowdsec"), + InstallDataDir: filepath.Join(tmpDir, "installed-data"), + } + + err = os.MkdirAll(hubCfg.HubDir, 0700) + require.NoError(t, err) + + err = os.MkdirAll(hubCfg.InstallDir, 0700) + require.NoError(t, err) + + err = os.MkdirAll(hubCfg.InstallDataDir, 0700) + require.NoError(t, err) + + index, err := os.Create(hubCfg.HubIndexFile) + require.NoError(t, err) + + _, err = index.WriteString(`{}`) + require.NoError(t, err) + + index.Close() + + t.Cleanup(func() { + os.RemoveAll(tmpDir) + }) + + constructor := InitHub + + if update { + constructor = InitHubUpdate + } + + hub, err := constructor(hubCfg) + require.NoError(t, err) + + return hub } -func envSetup(t *testing.T) *csconfig.Config { +func envSetup(t *testing.T) *Hub { resetResponseByPath() log.SetLevel(log.DebugLevel) - cfg := getTestCfg() - defaultTransport := http.DefaultClient.Transport t.Cleanup(func() { @@ -144,14 +161,7 @@ func envSetup(t *testing.T) *csconfig.Config { // Mock the http client http.DefaultClient.Transport = newMockTransport() - err := os.MkdirAll(cfg.Hub.InstallDir, 0700) - require.NoError(t, err) - - err = os.MkdirAll(cfg.Hub.HubDir, 0700) - require.NoError(t, err) - - err = UpdateHubIdx(cfg.Hub) - require.NoError(t, err) + hub := testHub(t, true) // if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil { // log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err) @@ -159,42 +169,33 @@ func envSetup(t *testing.T) *csconfig.Config { // if err := os.MkdirAll(cfg.Hub.InstallDir, 0700); err != nil { // log.Fatalf("failed to mkdir %s : %s", cfg.Hub.InstallDir, err) // } - return cfg + return hub } -func envTearDown(cfg *csconfig.Config) { - if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil { - log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err) - } - - if err := os.RemoveAll(cfg.Hub.HubDir); err != nil { - log.Fatalf("failed to remove %s : %s", cfg.Hub.HubDir, err) - } -} - -func testInstallItem(cfg *csconfig.HubCfg, t *testing.T, item Item) { +func testInstallItem(hub *Hub, t *testing.T, item Item) { // Install the parser - err := DownloadLatest(cfg, &item, false, false) + + err := hub.DownloadLatest(&item, false, false) require.NoError(t, err, "failed to download %s", item.Name) - _, err = LocalSync(cfg) + _, err = hub.LocalSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name) - assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) - err = EnableItem(cfg, &item) + err = hub.EnableItem(&item) require.NoError(t, err, "failed to enable %s", item.Name) - _, err = LocalSync(cfg) + _, err = hub.LocalSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) } -func testTaintItem(cfg *csconfig.HubCfg, t *testing.T, item Item) { - assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) +func testTaintItem(hub *Hub, t *testing.T, item Item) { + assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name) f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600) require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name) @@ -205,54 +206,54 @@ func testTaintItem(cfg *csconfig.HubCfg, t *testing.T, item Item) { require.NoError(t, err, "failed to write to %s (%s)", item.LocalPath, item.Name) // Local sync and check status - _, err = LocalSync(cfg) + _, err = hub.LocalSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name) } -func testUpdateItem(cfg *csconfig.HubCfg, t *testing.T, item Item) { - assert.False(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) +func testUpdateItem(hub *Hub, t *testing.T, item Item) { + assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) // Update it + check status - err := DownloadLatest(cfg, &item, true, true) + err := hub.DownloadLatest(&item, true, true) require.NoError(t, err, "failed to update %s", item.Name) // Local sync and check status - _, err = LocalSync(cfg) + _, err = hub.LocalSync() require.NoError(t, err, "failed to run localSync") - assert.True(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) - assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) } -func testDisableItem(cfg *csconfig.HubCfg, t *testing.T, item Item) { - assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) +func testDisableItem(hub *Hub, t *testing.T, item Item) { + assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name) // Remove - err := DisableItem(cfg, &item, false, false) + err := hub.DisableItem(&item, false, false) require.NoError(t, err, "failed to disable %s", item.Name) // Local sync and check status - warns, err := LocalSync(cfg) + warns, err := hub.LocalSync() require.NoError(t, err, "failed to run localSync") require.Empty(t, warns, "unexpected warnings : %+v", warns) - assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) - assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) - assert.True(t, hubIdx.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) + assert.True(t, hub.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name) // Purge - err = DisableItem(cfg, &item, true, false) + err = hub.DisableItem(&item, true, false) require.NoError(t, err, "failed to purge %s", item.Name) // Local sync and check status - warns, err = LocalSync(cfg) + warns, err = hub.LocalSync() require.NoError(t, err, "failed to run localSync") require.Empty(t, warns, "unexpected warnings : %+v", warns) - assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) - assert.False(t, hubIdx.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name) + assert.False(t, hub.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name) } func TestInstallParser(t *testing.T) { @@ -265,20 +266,18 @@ func TestInstallParser(t *testing.T) { - check its status - remove it */ - cfg := envSetup(t) - defer envTearDown(cfg) + hub := envSetup(t) - getHubIdxOrFail(t) // map iteration is random by itself - for _, it := range hubIdx.Items[PARSERS] { - testInstallItem(cfg.Hub, t, it) - it = hubIdx.Items[PARSERS][it.Name] - testTaintItem(cfg.Hub, t, it) - it = hubIdx.Items[PARSERS][it.Name] - testUpdateItem(cfg.Hub, t, it) - it = hubIdx.Items[PARSERS][it.Name] - testDisableItem(cfg.Hub, t, it) - it = hubIdx.Items[PARSERS][it.Name] + for _, it := range hub.Items[PARSERS] { + testInstallItem(hub, t, it) + it = hub.Items[PARSERS][it.Name] + testTaintItem(hub, t, it) + it = hub.Items[PARSERS][it.Name] + testUpdateItem(hub, t, it) + it = hub.Items[PARSERS][it.Name] + testDisableItem(hub, t, it) + it = hub.Items[PARSERS][it.Name] break } @@ -294,19 +293,17 @@ func TestInstallCollection(t *testing.T) { - check its status - remove it */ - cfg := envSetup(t) - defer envTearDown(cfg) + hub := envSetup(t) - getHubIdxOrFail(t) // map iteration is random by itself - for _, it := range hubIdx.Items[COLLECTIONS] { - testInstallItem(cfg.Hub, t, it) - it = hubIdx.Items[COLLECTIONS][it.Name] - testTaintItem(cfg.Hub, t, it) - it = hubIdx.Items[COLLECTIONS][it.Name] - testUpdateItem(cfg.Hub, t, it) - it = hubIdx.Items[COLLECTIONS][it.Name] - testDisableItem(cfg.Hub, t, it) + for _, it := range hub.Items[COLLECTIONS] { + testInstallItem(hub, t, it) + it = hub.Items[COLLECTIONS][it.Name] + testTaintItem(hub, t, it) + it = hub.Items[COLLECTIONS][it.Name] + testUpdateItem(hub, t, it) + it = hub.Items[COLLECTIONS][it.Name] + testDisableItem(hub, t, it) break } } diff --git a/pkg/cwhub/download.go b/pkg/cwhub/download.go index 7b6771867..0f4a8fb7a 100644 --- a/pkg/cwhub/download.go +++ b/pkg/cwhub/download.go @@ -19,31 +19,39 @@ import ( var ErrIndexNotFound = fmt.Errorf("index not found") -// UpdateHubIdx downloads the latest version of the index and updates the one in memory -func UpdateHubIdx(hub *csconfig.HubCfg) error { - bidx, err := DownloadHubIdx(hub) +// InitHubUpdate is like InitHub but downloads and updates the index instead of reading from the disk +// It is used to inizialize the hub when there is no index file yet +func InitHubUpdate(cfg *csconfig.HubCfg) (*Hub, error) { + if cfg == nil { + return nil, fmt.Errorf("no configuration found for hub") + } + + bidx, err := DownloadHubIdx(cfg.HubIndexFile) if err != nil { - return fmt.Errorf("failed to download index: %w", err) + return nil, fmt.Errorf("failed to download index: %w", err) } ret, err := ParseIndex(bidx) if err != nil { if !errors.Is(err, ErrMissingReference) { - return fmt.Errorf("failed to read index: %w", err) + return nil, fmt.Errorf("failed to read index: %w", err) } } - hubIdx = HubIndex{Items: ret} - - if _, err := LocalSync(hub); err != nil { - return fmt.Errorf("failed to sync: %w", err) + theHub = &Hub{ + Items: ret, + cfg: cfg, } - return nil + if _, err := theHub.LocalSync(); err != nil { + return nil, fmt.Errorf("failed to sync: %w", err) + } + + return theHub, nil } // DownloadHubIdx downloads the latest version of the index and returns the content -func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) { +func DownloadHubIdx(indexPath string) ([]byte, error) { log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile)) req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil) @@ -70,7 +78,7 @@ func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) { return nil, fmt.Errorf("failed to read request answer for hub index: %w", err) } - oldContent, err := os.ReadFile(hub.HubIndexFile) + oldContent, err := os.ReadFile(indexPath) if err != nil { if !os.IsNotExist(err) { log.Warningf("failed to read hub index: %s", err) @@ -80,7 +88,7 @@ func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) { // write it anyway, can't hurt } - file, err := os.OpenFile(hub.HubIndexFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + file, err := os.OpenFile(indexPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) if err != nil { return nil, fmt.Errorf("while opening hub index file: %w", err) @@ -92,13 +100,13 @@ func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) { return nil, fmt.Errorf("while writing hub index file: %w", err) } - log.Infof("Wrote new %d bytes index to %s", wsize, hub.HubIndexFile) + log.Infof("Wrote new %d bytes index to %s", wsize, indexPath) return body, nil } // DownloadLatest will download the latest version of Item to the tdir directory -func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOnly bool) error { +func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) error { var err error log.Debugf("Downloading %s %s", target.Type, target.Name) @@ -109,7 +117,7 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn return nil } - return DownloadItem(hub, target, overwrite) + return h.DownloadItem(target, overwrite) } // collection @@ -117,7 +125,7 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn for idx, ptr := range tmp { ptrtype := ItemTypes[idx] for _, p := range ptr { - val, ok := hubIdx.Items[ptrtype][p] + val, ok := h.Items[ptrtype][p] if !ok { return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name) } @@ -132,7 +140,7 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn if ptrtype == COLLECTIONS { log.Tracef("collection, recurse") - err = DownloadLatest(hub, &val, overwrite, updateOnly) + err = h.DownloadLatest(&val, overwrite, updateOnly) if err != nil { return fmt.Errorf("while downloading %s: %w", val.Name, err) } @@ -140,7 +148,7 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn downloaded := val.Downloaded - err = DownloadItem(hub, &val, overwrite) + err = h.DownloadItem(&val, overwrite) if err != nil { return fmt.Errorf("while downloading %s: %w", val.Name, err) } @@ -148,16 +156,16 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn // We need to enable an item when it has been added to a collection since latest release of the collection. // We check if val.Downloaded is false because maybe the item has been disabled by the user. if !val.Installed && !downloaded { - if err = EnableItem(hub, &val); err != nil { + if err = h.EnableItem(&val); err != nil { return fmt.Errorf("enabling '%s': %w", val.Name, err) } } - hubIdx.Items[ptrtype][p] = val + h.Items[ptrtype][p] = val } } - err = DownloadItem(hub, target, overwrite) + err = h.DownloadItem(target, overwrite) if err != nil { return fmt.Errorf("failed to download item: %w", err) } @@ -165,8 +173,8 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn return nil } -func DownloadItem(hub *csconfig.HubCfg, target *Item, overwrite bool) error { - tdir := hub.HubDir +func (h *Hub) DownloadItem(target *Item, overwrite bool) error { + tdir := h.cfg.HubDir // if user didn't --force, don't overwrite local, tainted, up-to-date files if !overwrite { @@ -202,12 +210,12 @@ func DownloadItem(hub *csconfig.HubCfg, target *Item, overwrite bool) error { return fmt.Errorf("while reading %s: %w", req.URL.String(), err) } - h := sha256.New() - if _, err = h.Write(body); err != nil { + hash := sha256.New() + if _, err = hash.Write(body); err != nil { return fmt.Errorf("while hashing %s: %w", target.Name, err) } - meow := fmt.Sprintf("%x", h.Sum(nil)) + meow := fmt.Sprintf("%x", hash.Sum(nil)) if meow != target.Versions[target.Version].Digest { log.Errorf("Downloaded version doesn't match index, please 'hub update'") log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest) @@ -263,18 +271,18 @@ func DownloadItem(hub *csconfig.HubCfg, target *Item, overwrite bool) error { target.Tainted = false target.UpToDate = true - if err = downloadData(hub.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { + if err = downloadData(h.cfg.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil { return fmt.Errorf("while downloading data for %s: %w", target.FileName, err) } - hubIdx.Items[target.Type][target.Name] = *target + h.Items[target.Type][target.Name] = *target return nil } // DownloadDataIfNeeded downloads the data files for an item -func DownloadDataIfNeeded(hub *csconfig.HubCfg, target Item, force bool) error { - itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.InstallDir, target.Type, target.Stage, target.FileName) +func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error { + itemFilePath := fmt.Sprintf("%s/%s/%s/%s", h.cfg.InstallDir, target.Type, target.Stage, target.FileName) itemFile, err := os.Open(itemFilePath) if err != nil { @@ -283,7 +291,7 @@ func DownloadDataIfNeeded(hub *csconfig.HubCfg, target Item, force bool) error { defer itemFile.Close() - if err = downloadData(hub.InstallDataDir, force, itemFile); err != nil { + if err = downloadData(h.cfg.InstallDataDir, force, itemFile); err != nil { return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err) } diff --git a/pkg/cwhub/download_test.go b/pkg/cwhub/download_test.go index b1b41c579..ed0763081 100644 --- a/pkg/cwhub/download_test.go +++ b/pkg/cwhub/download_test.go @@ -2,12 +2,11 @@ package cwhub import ( "fmt" + "os" "strings" "testing" log "github.com/sirupsen/logrus" - - "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) func TestDownloadHubIdx(t *testing.T) { @@ -15,9 +14,18 @@ func TestDownloadHubIdx(t *testing.T) { // bad url template fmt.Println("Test 'bad URL'") + tmpIndex, err := os.CreateTemp("", "index.json") + if err != nil { + t.Fatalf("failed to create temp file : %s", err) + } + + t.Cleanup(func() { + os.Remove(tmpIndex.Name()) + }) + RawFileURLTemplate = "x" - ret, err := DownloadHubIdx(&csconfig.HubCfg{}) + ret, err := DownloadHubIdx(tmpIndex.Name()) if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") { log.Errorf("unexpected error %s", err) } @@ -29,7 +37,7 @@ func TestDownloadHubIdx(t *testing.T) { RawFileURLTemplate = "https://baddomain/%s/%s" - ret, err = DownloadHubIdx(&csconfig.HubCfg{}) + ret, err = DownloadHubIdx(tmpIndex.Name()) if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") { log.Errorf("unexpected error %s", err) } @@ -41,7 +49,7 @@ func TestDownloadHubIdx(t *testing.T) { RawFileURLTemplate = back - ret, err = DownloadHubIdx(&csconfig.HubCfg{HubIndexFile: "/does/not/exist/index.json"}) + ret, err = DownloadHubIdx("/does/not/exist/index.json") if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") { log.Errorf("unexpected error %s", err) } diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index 2db768636..05c34824d 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -8,7 +8,6 @@ import ( log "github.com/sirupsen/logrus" "golang.org/x/mod/semver" - "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwversion" ) @@ -62,8 +61,8 @@ func SetHubBranch() { } // InstallItem installs an item from the hub -func InstallItem(csConfig *csconfig.Config, name string, itemType string, force bool, downloadOnly bool) error { - item := GetItem(itemType, name) +func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool) error { + item := h.GetItem(itemType, name) if item == nil { return fmt.Errorf("unable to retrieve item: %s", name) } @@ -76,26 +75,26 @@ func InstallItem(csConfig *csconfig.Config, name string, itemType string, force } } - err := DownloadLatest(csConfig.Hub, item, force, true) + err := h.DownloadLatest(item, force, true) if err != nil { return fmt.Errorf("while downloading %s: %w", item.Name, err) } - if err = AddItem(itemType, *item); err != nil { + if err = h.AddItem(itemType, *item); err != nil { return fmt.Errorf("while adding %s: %w", item.Name, err) } if downloadOnly { - log.Infof("Downloaded %s to %s", item.Name, filepath.Join(csConfig.Hub.HubDir, item.RemotePath)) + log.Infof("Downloaded %s to %s", item.Name, filepath.Join(h.cfg.HubDir, item.RemotePath)) return nil } - err = EnableItem(csConfig.Hub, item) + err = h.EnableItem(item) if err != nil { return fmt.Errorf("while enabling %s: %w", item.Name, err) } - if err := AddItem(itemType, *item); err != nil { + if err := h.AddItem(itemType, *item); err != nil { return fmt.Errorf("while adding %s: %w", item.Name, err) } @@ -105,20 +104,20 @@ func InstallItem(csConfig *csconfig.Config, name string, itemType string, force } // RemoveItem removes one - or all - the items from the hub -func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) error { +func (h *Hub) RemoveMany(itemType string, name string, all bool, purge bool, forceAction bool) error { if name != "" { - item := GetItem(itemType, name) + item := h.GetItem(itemType, name) if item == nil { return fmt.Errorf("can't find '%s' in %s", name, itemType) } - err := DisableItem(csConfig.Hub, item, purge, forceAction) + err := h.DisableItem(item, purge, forceAction) if err != nil { return fmt.Errorf("unable to disable %s: %w", item.Name, err) } - if err = AddItem(itemType, *item); err != nil { + if err = h.AddItem(itemType, *item); err != nil { return fmt.Errorf("unable to add %s: %w", item.Name, err) } @@ -132,17 +131,17 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo disabled := 0 // remove all - for _, v := range GetItemMap(itemType) { + for _, v := range h.GetItemMap(itemType) { if !v.Installed { continue } - err := DisableItem(csConfig.Hub, &v, purge, forceAction) + err := h.DisableItem(&v, purge, forceAction) if err != nil { return fmt.Errorf("unable to disable %s: %w", v.Name, err) } - if err := AddItem(itemType, v); err != nil { + if err := h.AddItem(itemType, v); err != nil { return fmt.Errorf("unable to add %s: %w", v.Name, err) } disabled++ @@ -154,11 +153,11 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo } // UpgradeConfig upgrades an item from the hub -func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) error { +func (h *Hub) UpgradeConfig(itemType string, name string, force bool) error { updated := 0 found := false - for _, v := range GetItemMap(itemType) { + for _, v := range h.GetItemMap(itemType) { if name != "" && name != v.Name { continue } @@ -178,7 +177,7 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc if v.UpToDate { log.Infof("%s: up-to-date", v.Name) - if err := DownloadDataIfNeeded(csConfig.Hub, v, force); err != nil { + if err := h.DownloadDataIfNeeded(v, force); err != nil { return fmt.Errorf("%s: download failed: %w", v.Name, err) } @@ -187,7 +186,7 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc } } - if err := DownloadLatest(csConfig.Hub, &v, force, true); err != nil { + if err := h.DownloadLatest(&v, force, true); err != nil { return fmt.Errorf("%s: download failed: %w", v.Name, err) } @@ -205,7 +204,7 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc updated++ } - if err := AddItem(itemType, v); err != nil { + if err := h.AddItem(itemType, v); err != nil { return fmt.Errorf("unable to add %s: %w", v.Name, err) } } diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index ecb778fdc..72dac2342 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -4,165 +4,165 @@ import ( "testing" "github.com/stretchr/testify/require" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) // Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection // We expect the new scenario to be installed func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { - cfg := envSetup(t) - defer envTearDown(cfg) + hub := envSetup(t) // fresh install of collection - getHubIdxOrFail(t) + hub = getHubOrFail(t, hub.cfg) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false)) + require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) // This is the scenario that gets added in next version of collection - require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) - require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") // collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario" pushUpdateToCollectionInHub() - if err := UpdateHubIdx(cfg.Hub); err != nil { - t.Fatalf("failed to download index : %s", err) - } + hub, err := InitHubUpdate(hub.cfg) + require.NoError(t, err, "failed to download index: %s", err) - getHubIdxOrFail(t) + hub = getHubOrFail(t, hub.cfg) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - err := UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) + err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false) require.NoError(t, err) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") - require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) - require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) } // Install a collection, disable a scenario. // Upgrade should install should not enable/download the disabled scenario. func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { - cfg := envSetup(t) - defer envTearDown(cfg) + hub := envSetup(t) // fresh install of collection - getHubIdxOrFail(t) + hub = getHubOrFail(t, hub.cfg) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false)) + require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") - err := RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) + err := hub.RemoveMany(SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) require.NoError(t, err) - getHubIdxOrFail(t) + hub = getHubOrFail(t, hub.cfg) // scenario referenced by collection was deleted hence, collection should be tainted - require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - if err = UpdateHubIdx(cfg.Hub); err != nil { - t.Fatalf("failed to download index : %s", err) - } + hub, err = InitHubUpdate(hub.cfg) + require.NoError(t, err, "failed to download index: %s", err) - err = UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) + err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false) require.NoError(t, err) - getHubIdxOrFail(t) - require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + hub = getHubOrFail(t, hub.cfg) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) } -func getHubIdxOrFail(t *testing.T) { - if err := GetHubIdx(getTestCfg().Hub); err != nil { - t.Fatalf("failed to load hub index") - } +// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test +func getHubOrFail(t *testing.T, hubCfg *csconfig.HubCfg) *Hub { + hub, err := InitHub(hubCfg) + require.NoError(t, err, "failed to load hub index") + return hub } // Install a collection. Disable a referenced scenario. Publish new version of collection with new scenario // Upgrade should not enable/download the disabled scenario. // Upgrade should install and enable the newly added scenario. func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) { - cfg := envSetup(t) - defer envTearDown(cfg) + hub := envSetup(t) // fresh install of collection - getHubIdxOrFail(t) + hub = getHubOrFail(t, hub.cfg) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false)) + require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") - err := RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) + err := hub.RemoveMany(SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false) require.NoError(t, err) - getHubIdxOrFail(t) + hub = getHubOrFail(t, hub.cfg) // scenario referenced by collection was deleted hence, collection should be tainted - require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) + require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) // collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario" // we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario // we just removed. Nor should it install the newly added scenario pushUpdateToCollectionInHub() - if err = UpdateHubIdx(cfg.Hub); err != nil { - t.Fatalf("failed to download index : %s", err) - } + hub, err = InitHubUpdate(hub.cfg) + require.NoError(t, err, "failed to download index: %s", err) - require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - getHubIdxOrFail(t) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + hub = getHubOrFail(t, hub.cfg) - err = UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false) + err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false) require.NoError(t, err) - getHubIdxOrFail(t) - require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) + hub = getHubOrFail(t, hub.cfg) + require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) + require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed) } func assertCollectionDepsInstalled(t *testing.T, collection string) { t.Helper() - c := hubIdx.Items[COLLECTIONS][collection] - require.NoError(t, CollecDepsCheck(&c)) + hub, err := GetHub() + require.NoError(t, err) + + c := hub.Items[COLLECTIONS][collection] + require.NoError(t, hub.CollectDepsCheck(&c)) } func pushUpdateToCollectionInHub() { diff --git a/pkg/cwhub/hubindex.go b/pkg/cwhub/hub.go similarity index 76% rename from pkg/cwhub/hubindex.go rename to pkg/cwhub/hub.go index a29cd873a..6c9dce94b 100644 --- a/pkg/cwhub/hubindex.go +++ b/pkg/cwhub/hub.go @@ -6,8 +6,9 @@ import ( "strings" log "github.com/sirupsen/logrus" -) + "github.com/crowdsecurity/crowdsec/pkg/csconfig" +) const ( HubIndexFile = ".index.json" @@ -20,25 +21,34 @@ const ( ) var ( - // XXX: The order is important, as it is used to construct the - // index tree in memory --> collections must be last + // XXX: The order is important, as it is used to range over sub-items in collections ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS} - hubIdx = HubIndex{} ) - type HubItems map[string]map[string]Item -// HubIndex represents the runtime status of the hub (parsed items, etc.) -// XXX: this could be renamed "Hub" tout court once the confusion with HubCfg is cleared -type HubIndex struct { - Items HubItems +// Hub represents the runtime status of the hub (parsed items, etc.) +type Hub struct { + Items HubItems + cfg *csconfig.HubCfg skippedLocal int skippedTainted int } +var theHub *Hub + +// GetHub returns the hub singleton +// it returns an error if it's not initialized to avoid nil dereference +func GetHub() (*Hub, error) { + if theHub == nil { + return nil, fmt.Errorf("hub not initialized") + } + + return theHub, nil +} + // displaySummary prints a total count of the hub items -func (h HubIndex) displaySummary() { +func (h Hub) displaySummary() { msg := "Loaded: " for itemType := range h.Items { msg += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType) @@ -52,8 +62,14 @@ func (h HubIndex) displaySummary() { // DisplaySummary prints a total count of the hub items. // It is a wrapper around HubIndex.displaySummary() to avoid exporting the hub singleton -func DisplaySummary() { - hubIdx.displaySummary() +// XXX: to be removed later +func DisplaySummary() error { + hub, err := GetHub() + if err != nil { + return err + } + hub.displaySummary() + return nil } // ParseIndex takes the content of a .index.json file and returns the map of associated parsers/scenarios/collections diff --git a/pkg/cwhub/install.go b/pkg/cwhub/install.go index 4ac955b7b..71fb46b1b 100644 --- a/pkg/cwhub/install.go +++ b/pkg/cwhub/install.go @@ -6,12 +6,10 @@ import ( "path/filepath" log "github.com/sirupsen/logrus" - - "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) -func purgeItem(hub *csconfig.HubCfg, target Item) (Item, error) { - itempath := hub.HubDir + "/" + target.RemotePath +func (h *Hub) purgeItem(target Item) (Item, error) { + itempath := h.cfg.HubDir + "/" + target.RemotePath // disable hub file if err := os.Remove(itempath); err != nil { @@ -20,19 +18,19 @@ func purgeItem(hub *csconfig.HubCfg, target Item) (Item, error) { target.Downloaded = false log.Infof("Removed source file [%s]: %s", target.Name, itempath) - hubIdx.Items[target.Type][target.Name] = target + h.Items[target.Type][target.Name] = target return target, nil } // DisableItem to disable an item managed by the hub, removes the symlink if purge is true -func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) error { +func (h *Hub) DisableItem(target *Item, purge bool, force bool) error { var err error // already disabled, noop unless purge if !target.Installed { if purge { - *target, err = purgeItem(hub, *target) + *target, err = h.purgeItem(*target) if err != nil { return err } @@ -54,7 +52,7 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} { ptrtype := ItemTypes[idx] for _, p := range ptr { - if val, ok := hubIdx.Items[ptrtype][p]; ok { + if val, ok := h.Items[ptrtype][p]; ok { // check if the item doesn't belong to another collection before removing it toRemove := true @@ -66,7 +64,7 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err } if toRemove { - err = DisableItem(hub, &val, purge, force) + err = h.DisableItem(&val, purge, force) if err != nil { return fmt.Errorf("while disabling %s: %w", p, err) } @@ -80,7 +78,7 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err } } - syml, err := filepath.Abs(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName) + syml, err := filepath.Abs(h.cfg.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName) if err != nil { return err } @@ -103,7 +101,7 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err return fmt.Errorf("while reading symlink: %w", err) } - absPath, err := filepath.Abs(hub.HubDir + "/" + target.RemotePath) + absPath, err := filepath.Abs(h.cfg.HubDir + "/" + target.RemotePath) if err != nil { return fmt.Errorf("while abs path: %w", err) } @@ -124,23 +122,23 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err target.Installed = false if purge { - *target, err = purgeItem(hub, *target) + *target, err = h.purgeItem(*target) if err != nil { return err } } - hubIdx.Items[target.Type][target.Name] = *target + h.Items[target.Type][target.Name] = *target return nil } // creates symlink between actual config file at hub.HubDir and hub.ConfigDir // Handles collections recursively -func EnableItem(hub *csconfig.HubCfg, target *Item) error { +func (h *Hub) EnableItem(target *Item) error { var err error - parentDir := filepath.Clean(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/") + parentDir := filepath.Clean(h.cfg.InstallDir + "/" + target.Type + "/" + target.Stage + "/") // create directories if needed if target.Installed { @@ -172,12 +170,12 @@ func EnableItem(hub *csconfig.HubCfg, target *Item) error { for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} { ptrtype := ItemTypes[idx] for _, p := range ptr { - val, ok := hubIdx.Items[ptrtype][p] + val, ok := h.Items[ptrtype][p] if !ok { return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name) } - err = EnableItem(hub, &val) + err = h.EnableItem(&val) if err != nil { return fmt.Errorf("while installing %s: %w", p, err) } @@ -192,7 +190,7 @@ func EnableItem(hub *csconfig.HubCfg, target *Item) error { } // hub.ConfigDir + target.RemotePath - srcPath, err := filepath.Abs(hub.HubDir + "/" + target.RemotePath) + srcPath, err := filepath.Abs(h.cfg.HubDir + "/" + target.RemotePath) if err != nil { return fmt.Errorf("while getting source path: %w", err) } @@ -208,7 +206,7 @@ func EnableItem(hub *csconfig.HubCfg, target *Item) error { log.Infof("Enabled %s : %s", target.Type, target.Name) target.Installed = true - hubIdx.Items[target.Type][target.Name] = *target + h.Items[target.Type][target.Name] = *target return nil } diff --git a/pkg/cwhub/loader.go b/pkg/cwhub/loader.go index 4cbceb413..31e347067 100644 --- a/pkg/cwhub/loader.go +++ b/pkg/cwhub/loader.go @@ -19,10 +19,6 @@ func isYAMLFileName(path string) bool { return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") } -func validItemFileName(vname string, fauthor string, fname string) bool { - return (fauthor+"/"+fname == vname+".yaml") || (fauthor+"/"+fname == vname+".yml") -} - func handleSymlink(path string) (string, error) { hubpath, err := os.Readlink(path) if err != nil { @@ -60,19 +56,6 @@ func getSHA256(filepath string) (string, error) { return fmt.Sprintf("%x", h.Sum(nil)), nil } -type Walker struct { - // the walk/parserVisit function can't receive extra args - hubdir string - installdir string -} - -func NewWalker(hub *csconfig.HubCfg) Walker { - return Walker{ - hubdir: hub.HubDir, - installdir: hub.InstallDir, - } -} - type itemFileInfo struct { fname string stage string @@ -80,16 +63,19 @@ type itemFileInfo struct { fauthor string } -func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) { +func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) { ret := itemFileInfo{} inhub := false + hubDir := h.cfg.HubDir + installDir := h.cfg.InstallDir + subs := strings.Split(path, string(os.PathSeparator)) - log.Tracef("path:%s, hubdir:%s, installdir:%s", path, w.hubdir, w.installdir) + log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubDir, installDir) log.Tracef("subs:%v", subs) // we're in hub (~/.hub/hub/) - if strings.HasPrefix(path, w.hubdir) { + if strings.HasPrefix(path, hubDir) { log.Tracef("in hub dir") inhub = true @@ -104,7 +90,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) { ret.fauthor = subs[len(subs)-2] ret.stage = subs[len(subs)-3] ret.ftype = subs[len(subs)-4] - } else if strings.HasPrefix(path, w.installdir) { // we're in install /etc/crowdsec//... + } else if strings.HasPrefix(path, installDir) { // we're in install /etc/crowdsec//... log.Tracef("in install dir") if len(subs) < 3 { return itemFileInfo{}, false, fmt.Errorf("path is too short: %s (%d)", path, len(subs)) @@ -118,7 +104,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) { ret.ftype = subs[len(subs)-3] ret.fauthor = "" } else { - return itemFileInfo{}, false, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, w.hubdir, w.installdir) + return itemFileInfo{}, false, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubDir, installDir) } log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype) @@ -140,7 +126,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) { return ret, inhub, nil } -func (w Walker) itemVisit(path string, f os.DirEntry, err error) error { +func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { var ( local bool hubpath string @@ -166,7 +152,7 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error { return nil } - info, inhub, err := w.getItemInfo(path) + info, inhub, err := h.getItemInfo(path) if err != nil { return err } @@ -197,12 +183,12 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error { // if it's not a symlink and not in hub, it's a local file, don't bother if local && !inhub { log.Tracef("%s is a local file, skip", path) - hubIdx.skippedLocal++ + h.skippedLocal++ // log.Infof("local scenario, skip.") _, fileName := filepath.Split(path) - hubIdx.Items[info.ftype][info.fname] = Item{ + h.Items[info.ftype][info.fname] = Item{ Name: info.fname, Stage: info.stage, Installed: true, @@ -221,7 +207,7 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error { match := false - for name, item := range hubIdx.Items[info.ftype] { + for name, item := range h.Items[info.ftype] { log.Tracef("check [%s] vs [%s] : %s", info.fname, item.RemotePath, info.ftype+"/"+info.stage+"/"+info.fname+".yaml") if info.fname != item.FileName { @@ -241,12 +227,12 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error { continue } - // wrong file - if !validItemFileName(item.Name, info.fauthor, info.fname) { + // not the item we're looking for + if !item.validPath(info.fauthor, info.fname) { continue } - if path == w.hubdir+"/"+item.RemotePath { + if path == h.cfg.HubDir+"/"+item.RemotePath { log.Tracef("marking %s as downloaded", item.Name) item.Downloaded = true } @@ -303,7 +289,7 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error { if !match { log.Tracef("got tainted match for %s: %s", item.Name, path) - hubIdx.skippedTainted++ + h.skippedTainted++ // the file and the stage is right, but the hash is wrong, it has been tainted by user if !inhub { item.LocalPath = path @@ -316,7 +302,7 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error { item.LocalHash = sha } - hubIdx.Items[info.ftype][name] = item + h.Items[info.ftype][name] = item return nil } @@ -326,13 +312,13 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error { return nil } -func CollecDepsCheck(v *Item) error { - if v.versionStatus() != 0 { // not up-to-date - log.Debugf("%s dependencies not checked : not up-to-date", v.Name) +func (h *Hub) CollectDepsCheck(v *Item) error { + if v.Type != COLLECTIONS { return nil } - if v.Type != COLLECTIONS { + if v.versionStatus() != 0 { // not up-to-date + log.Debugf("%s dependencies not checked: not up-to-date", v.Name) return nil } @@ -342,7 +328,7 @@ func CollecDepsCheck(v *Item) error { for idx, itemSlice := range [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} { sliceType := ItemTypes[idx] for _, subName := range itemSlice { - subItem, ok := hubIdx.Items[sliceType][subName] + subItem, ok := h.Items[sliceType][subName] if !ok { return fmt.Errorf("referred %s %s in collection %s doesn't exist", sliceType, subName, v.Name) } @@ -356,7 +342,7 @@ func CollecDepsCheck(v *Item) error { if subItem.Type == COLLECTIONS { log.Tracef("collec, recurse.") - if err := CollecDepsCheck(&subItem); err != nil { + if err := h.CollectDepsCheck(&subItem); err != nil { if subItem.Tainted { v.Tainted = true } @@ -364,7 +350,7 @@ func CollecDepsCheck(v *Item) error { return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err) } - hubIdx.Items[sliceType][subName] = subItem + h.Items[sliceType][subName] = subItem } // propagate the state of sub-items to set @@ -395,7 +381,7 @@ func CollecDepsCheck(v *Item) error { subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name) } - hubIdx.Items[sliceType][subName] = subItem + h.Items[sliceType][subName] = subItem log.Tracef("checking for %s - tainted:%t uptodate:%t", subName, v.Tainted, v.UpToDate) } @@ -404,7 +390,7 @@ func CollecDepsCheck(v *Item) error { return nil } -func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) { +func (h *Hub) SyncDir(dir string) ([]string, error) { warnings := []string{} // For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last @@ -414,13 +400,13 @@ func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) { log.Errorf("failed %s : %s", cpath, err) } - err = filepath.WalkDir(cpath, NewWalker(hub).itemVisit) + err = filepath.WalkDir(cpath, h.itemVisit) if err != nil { return warnings, err } } - for name, item := range hubIdx.Items[COLLECTIONS] { + for name, item := range h.Items[COLLECTIONS] { if !item.Installed { continue } @@ -428,9 +414,9 @@ func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) { vs := item.versionStatus() switch vs { case 0: // latest - if err := CollecDepsCheck(&item); err != nil { + if err := h.CollectDepsCheck(&item); err != nil { warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err)) - hubIdx.Items[COLLECTIONS][name] = item + h.Items[COLLECTIONS][name] = item } case 1: // not up-to-date warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version)) @@ -445,51 +431,55 @@ func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) { } // Updates the info from HubInit() with the local state -func LocalSync(hub *csconfig.HubCfg) ([]string, error) { - hubIdx.skippedLocal = 0 - hubIdx.skippedTainted = 0 +func (h *Hub) LocalSync() ([]string, error) { + h.skippedLocal = 0 + h.skippedTainted = 0 - warnings, err := SyncDir(hub, hub.InstallDir) + warnings, err := h.SyncDir(h.cfg.InstallDir) if err != nil { - return warnings, fmt.Errorf("failed to scan %s: %w", hub.InstallDir, err) + return warnings, fmt.Errorf("failed to scan %s: %w", h.cfg.InstallDir, err) } - _, err = SyncDir(hub, hub.HubDir) + _, err = h.SyncDir(h.cfg.HubDir) if err != nil { - return warnings, fmt.Errorf("failed to scan %s: %w", hub.HubDir, err) + return warnings, fmt.Errorf("failed to scan %s: %w", h.cfg.HubDir, err) } return warnings, nil } -func GetHubIdx(hub *csconfig.HubCfg) error { - if hub == nil { - return fmt.Errorf("no configuration found for hub") +// InitHub initializes the Hub, syncs the local state and returns the singleton for immediate use +func InitHub(cfg *csconfig.HubCfg) (*Hub, error) { + if cfg == nil { + return nil, fmt.Errorf("no configuration found for hub") } - log.Debugf("loading hub idx %s", hub.HubIndexFile) + log.Debugf("loading hub idx %s", cfg.HubIndexFile) - bidx, err := os.ReadFile(hub.HubIndexFile) + bidx, err := os.ReadFile(cfg.HubIndexFile) if err != nil { - return fmt.Errorf("unable to read index file: %w", err) + return nil, fmt.Errorf("unable to read index file: %w", err) } ret, err := ParseIndex(bidx) if err != nil { if !errors.Is(err, ErrMissingReference) { - return fmt.Errorf("unable to load existing index: %w", err) + return nil, fmt.Errorf("unable to load existing index: %w", err) } // XXX: why the error check if we bail out anyway? - return err + return nil, err } - hubIdx = HubIndex{Items: ret} + theHub = &Hub{ + Items: ret, + cfg: cfg, + } - _, err = LocalSync(hub) + _, err = theHub.LocalSync() if err != nil { - return fmt.Errorf("failed to sync Hub index with local deployment : %w", err) + return nil, fmt.Errorf("failed to sync Hub index with local deployment : %w", err) } - return nil + return theHub, nil } diff --git a/pkg/hubtest/hubtest.go b/pkg/hubtest/hubtest.go index eff2aa8fb..f8d9dce5f 100644 --- a/pkg/hubtest/hubtest.go +++ b/pkg/hubtest/hubtest.go @@ -18,7 +18,7 @@ type HubTest struct { TemplateConfigPath string TemplateProfilePath string TemplateSimulationPath string - HubIndex *cwhub.HubIndex + HubIndex *cwhub.Hub Tests []*HubTestItem } @@ -80,7 +80,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest, TemplateConfigPath: templateConfigFilePath, TemplateProfilePath: templateProfilePath, TemplateSimulationPath: templateSimulationPath, - HubIndex: &cwhub.HubIndex{Items: hubIndex}, + HubIndex: &cwhub.Hub{Items: hubIndex}, }, nil } diff --git a/pkg/hubtest/hubtest_item.go b/pkg/hubtest/hubtest_item.go index 25a89d880..f2b309bb5 100644 --- a/pkg/hubtest/hubtest_item.go +++ b/pkg/hubtest/hubtest_item.go @@ -52,7 +52,7 @@ type HubTestItem struct { TemplateConfigPath string TemplateProfilePath string TemplateSimulationPath string - HubIndex *cwhub.HubIndex + HubIndex *cwhub.Hub Config *HubTestItemConfig @@ -391,16 +391,16 @@ func (t *HubTestItem) InstallHub() error { } // load installed hub - err := cwhub.GetHubIdx(t.RuntimeHubConfig) + hub, err := cwhub.InitHub(t.RuntimeHubConfig) if err != nil { - log.Fatalf("can't local sync the hub: %+v", err) + log.Fatal(err) } // install data for parsers if needed - ret := cwhub.GetItemMap(cwhub.PARSERS) + ret := hub.GetItemMap(cwhub.PARSERS) for parserName, item := range ret { if item.Installed { - if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil { + if err := hub.DownloadDataIfNeeded(item, true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err) } log.Debugf("parser '%s' installed successfully in runtime environment", parserName) @@ -408,10 +408,10 @@ func (t *HubTestItem) InstallHub() error { } // install data for scenarios if needed - ret = cwhub.GetItemMap(cwhub.SCENARIOS) + ret = hub.GetItemMap(cwhub.SCENARIOS) for scenarioName, item := range ret { if item.Installed { - if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil { + if err := hub.DownloadDataIfNeeded(item, true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err) } log.Debugf("scenario '%s' installed successfully in runtime environment", scenarioName) @@ -419,10 +419,10 @@ func (t *HubTestItem) InstallHub() error { } // install data for postoverflows if needed - ret = cwhub.GetItemMap(cwhub.POSTOVERFLOWS) + ret = hub.GetItemMap(cwhub.POSTOVERFLOWS) for postoverflowName, item := range ret { if item.Installed { - if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil { + if err := hub.DownloadDataIfNeeded(item, true); err != nil { return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err) } log.Debugf("postoverflow '%s' installed successfully in runtime environment", postoverflowName) diff --git a/pkg/leakybucket/buckets_test.go b/pkg/leakybucket/buckets_test.go index e08887be8..41d463c48 100644 --- a/pkg/leakybucket/buckets_test.go +++ b/pkg/leakybucket/buckets_test.go @@ -8,12 +8,14 @@ import ( "html/template" "io" "os" + "path/filepath" "reflect" "sync" "testing" "time" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/types" @@ -33,7 +35,20 @@ func TestBucket(t *testing.T) { envSetting = os.Getenv("TEST_ONLY") tomb = &tomb.Tomb{} ) - err := exprhelpers.Init(nil) + + testdata := "./tests" + + hubCfg := &csconfig.HubCfg{ + HubDir: filepath.Join(testdata, "hub"), + HubIndexFile: filepath.Join(testdata, "hub", "index.json"), + } + + _, err := cwhub.InitHub(hubCfg) + if err != nil { + t.Fatalf("failed to init hub : %s", err) + } + + err = exprhelpers.Init(nil) if err != nil { log.Fatalf("exprhelpers init failed: %s", err) } @@ -44,12 +59,15 @@ func TestBucket(t *testing.T) { } } else { wg := new(sync.WaitGroup) - fds, err := os.ReadDir("./tests/") + fds, err := os.ReadDir(testdata) if err != nil { t.Fatalf("Unable to read test directory : %s", err) } for _, fd := range fds { - fname := "./tests/" + fd.Name() + if fd.Name() == "hub" { + continue + } + fname := filepath.Join(testdata, fd.Name()) log.Infof("Running test on %s", fname) tomb.Go(func() error { wg.Add(1) @@ -112,10 +130,8 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error { files = append(files, x.Filename) } - cscfg := &csconfig.CrowdsecServiceCfg{ - DataDir: "tests", - } - holders, response, err := LoadBuckets(cscfg, files, tomb, buckets, false) + cscfg := &csconfig.CrowdsecServiceCfg{} + holders, response, err := LoadBuckets(cscfg, "tests", files, tomb, buckets, false) if err != nil { t.Fatalf("failed loading bucket : %s", err) } @@ -123,7 +139,7 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error { watchTomb(tomb) return nil }) - if !testFile(t, dir+"/test.json", dir+"/in-buckets_state.json", holders, response, buckets) { + if !testFile(t, filepath.Join(dir, "test.json"), filepath.Join(dir, "in-buckets_state.json"), holders, response, buckets) { return fmt.Errorf("tests from %s failed", dir) } return nil diff --git a/pkg/leakybucket/manager_load.go b/pkg/leakybucket/manager_load.go index b384e5967..484d6b9ac 100644 --- a/pkg/leakybucket/manager_load.go +++ b/pkg/leakybucket/manager_load.go @@ -179,12 +179,17 @@ func ValidateFactory(bucketFactory *BucketFactory) error { return nil } -func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) { +func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, dataDir string, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) { var ( ret = []BucketFactory{} response chan types.Event ) + hub, err := cwhub.GetHub() + if err != nil { + return nil, nil, err + } + response = make(chan types.Event, 1) for _, f := range files { log.Debugf("Loading '%s'", f) @@ -212,7 +217,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb. log.Tracef("End of yaml file") break } - bucketFactory.DataDir = cscfg.DataDir + bucketFactory.DataDir = dataDir //check empty if bucketFactory.Name == "" { log.Errorf("Won't load nameless bucket") @@ -235,7 +240,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb. bucketFactory.Filename = filepath.Clean(f) bucketFactory.BucketName = seed.Generate() bucketFactory.ret = response - hubItem, err := cwhub.GetItemByPath(cwhub.SCENARIOS, bucketFactory.Filename) + hubItem, err := hub.GetItemByPath(cwhub.SCENARIOS, bucketFactory.Filename) if err != nil { log.Errorf("scenario %s (%s) couldn't be find in hub (ignore if in unit tests)", bucketFactory.Name, bucketFactory.Filename) } else { diff --git a/pkg/leakybucket/tests/hub/index.json b/pkg/leakybucket/tests/hub/index.json new file mode 100644 index 000000000..0967ef424 --- /dev/null +++ b/pkg/leakybucket/tests/hub/index.json @@ -0,0 +1 @@ +{} diff --git a/pkg/parser/unix_parser.go b/pkg/parser/unix_parser.go index 48b09795b..4c2ae9fde 100644 --- a/pkg/parser/unix_parser.go +++ b/pkg/parser/unix_parser.go @@ -64,8 +64,12 @@ func NewParsers() *Parsers { StageFiles: make([]Stagefile, 0), PovfwStageFiles: make([]Stagefile, 0), } + + // XXX: handle error + hub, _ := cwhub.GetHub() + for _, itemType := range []string{cwhub.PARSERS, cwhub.POSTOVERFLOWS} { - for _, hubParserItem := range cwhub.GetItemMap(itemType) { + for _, hubParserItem := range hub.GetItemMap(itemType) { if hubParserItem.Installed { stagefile := Stagefile{ Filename: hubParserItem.LocalPath, @@ -97,16 +101,16 @@ func NewParsers() *Parsers { func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) { var err error - patternsDir := filepath.Join(cConfig.Crowdsec.ConfigDir, "patterns/") + patternsDir := filepath.Join(cConfig.ConfigPaths.ConfigDir, "patterns/") log.Infof("Loading grok library %s", patternsDir) /* load base regexps for two grok parsers */ parsers.Ctx, err = Init(map[string]interface{}{"patterns": patternsDir, - "data": cConfig.Crowdsec.DataDir}) + "data": cConfig.ConfigPaths.DataDir}) if err != nil { return parsers, fmt.Errorf("failed to load parser patterns : %v", err) } parsers.Povfwctx, err = Init(map[string]interface{}{"patterns": patternsDir, - "data": cConfig.Crowdsec.DataDir}) + "data": cConfig.ConfigPaths.DataDir}) if err != nil { return parsers, fmt.Errorf("failed to load postovflw parser patterns : %v", err) } @@ -116,7 +120,7 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) { */ log.Infof("Loading enrich plugins") - parsers.EnricherCtx, err = Loadplugin(cConfig.Crowdsec.DataDir) + parsers.EnricherCtx, err = Loadplugin(cConfig.ConfigPaths.DataDir) if err != nil { return parsers, fmt.Errorf("failed to load enrich plugin : %v", err) } diff --git a/pkg/setup/install.go b/pkg/setup/install.go index 4b6034009..347340253 100644 --- a/pkg/setup/install.go +++ b/pkg/setup/install.go @@ -54,7 +54,8 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error cwhub.SetHubBranch() - if err := cwhub.GetHubIdx(csConfig.Hub); err != nil { + hub, err := cwhub.InitHub(csConfig.Hub) + if err != nil { return fmt.Errorf("getting hub index: %w", err) } @@ -75,7 +76,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := cwhub.InstallItem(csConfig, collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil { + if err := hub.InstallItem(collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil { return fmt.Errorf("while installing collection %s: %w", collection, err) } } @@ -89,7 +90,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := cwhub.InstallItem(csConfig, parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil { + if err := hub.InstallItem(parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil { return fmt.Errorf("while installing parser %s: %w", parser, err) } } @@ -103,7 +104,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := cwhub.InstallItem(csConfig, scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil { + if err := hub.InstallItem(scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil { return fmt.Errorf("while installing scenario %s: %w", scenario, err) } } @@ -117,7 +118,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := cwhub.InstallItem(csConfig, postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil { + if err := hub.InstallItem(postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil { return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err) } }