diff --git a/pkg/cwhub/branch.go b/cmd/crowdsec-cli/branch.go similarity index 79% rename from pkg/cwhub/branch.go rename to cmd/crowdsec-cli/branch.go index e25a6becf..f4dae2947 100644 --- a/pkg/cwhub/branch.go +++ b/cmd/crowdsec-cli/branch.go @@ -1,4 +1,4 @@ -package cwhub +package main // Set the appropriate hub branch according to config settings and crowdsec version @@ -44,16 +44,3 @@ func chooseHubBranch() string { return csVersion } - -// SetHubBranch sets the package variable that points to the hub branch. -func SetHubBranch() { - // a branch is already set, or specified from the flags - if HubBranch != "" { - return - } - - // use the branch corresponding to the crowdsec version - HubBranch = chooseHubBranch() - - log.Debugf("Using branch '%s' for the hub", HubBranch) -} diff --git a/cmd/crowdsec-cli/config_restore.go b/cmd/crowdsec-cli/config_restore.go index 1a081d6c5..d2b31c1d1 100644 --- a/cmd/crowdsec-cli/config_restore.go +++ b/cmd/crowdsec-cli/config_restore.go @@ -22,7 +22,7 @@ type OldAPICfg struct { } // it's a rip of the cli version, but in silent-mode -func silentInstallItem(name string, obtype string) (string, error) { +func silentInstallItem(name, obtype, hubURLTemplate, branch string) (string, error) { hub, err := cwhub.GetHub() if err != nil { return "", err @@ -32,7 +32,7 @@ func silentInstallItem(name string, obtype string) (string, error) { if item == nil { return "", fmt.Errorf("error retrieving item") } - err = hub.DownloadLatest(item, false, false) + err = hub.DownloadLatest(item, false, false, hubURLTemplate, branch) if err != nil { return "", fmt.Errorf("error while downloading %s : %v", item.Name, err) } @@ -53,7 +53,7 @@ func silentInstallItem(name string, obtype string) (string, error) { func restoreHub(dirPath string) error { var err error - cwhub.SetHubBranch() + branch := chooseHubBranch() for _, itype := range cwhub.ItemTypes { itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype) @@ -73,7 +73,7 @@ func restoreHub(dirPath string) error { return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err) } for _, toinstall := range upstreamList { - label, err := silentInstallItem(toinstall, itype) + label, err := silentInstallItem(toinstall, itype, hubURLTemplate, branch) if err != nil { log.Errorf("Error while installing %s : %s", toinstall, err) } else if label != "" { diff --git a/cmd/crowdsec-cli/hub.go b/cmd/crowdsec-cli/hub.go index c0f69ea87..6023f0e86 100644 --- a/cmd/crowdsec-cli/hub.go +++ b/cmd/crowdsec-cli/hub.go @@ -12,6 +12,13 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cwhub" ) +const ( + hubURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s" + remoteIndexPath = ".index.json" +) + +var hubBranch = "" + func NewHubCmd() *cobra.Command { var cmdHub = &cobra.Command{ Use: "hub [action]", @@ -33,7 +40,6 @@ cscli hub upgrade`, return nil }, } - cmdHub.PersistentFlags().StringVarP(&cwhub.HubBranch, "branch", "b", "", "Use given branch from hub") cmdHub.AddCommand(NewHubListCmd()) cmdHub.AddCommand(NewHubUpdateCmd()) @@ -81,7 +87,7 @@ func NewHubListCmd() *cobra.Command { Short: "List all installed configurations", Args: cobra.ExactArgs(0), DisableAutoGenTag: true, - RunE: runHubList, + RunE: runHubList, } flags := cmdHubList.Flags() @@ -91,18 +97,23 @@ func NewHubListCmd() *cobra.Command { } func runHubUpdate(cmd *cobra.Command, args []string) error { - cwhub.SetHubBranch() - // don't use require.Hub because if there is no index file, it would fail - hub, err := cwhub.InitHubUpdate(csConfig.Hub) + branch := hubBranch + if branch == "" { + branch = chooseHubBranch() + } + + log.Debugf("Using branch '%s' for the hub", branch) + + hub, err := cwhub.InitHubUpdate(csConfig.Hub, hubURLTemplate, branch, remoteIndexPath) if err != nil { if !errors.Is(err, cwhub.ErrIndexNotFound) { return fmt.Errorf("failed to get Hub index : %w", err) } - log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch) - cwhub.HubBranch = "master" - if hub, err = cwhub.InitHubUpdate(csConfig.Hub); err != nil { + log.Warnf("Could not find index file for branch '%s', using 'master'", branch) + branch = "master" + if hub, err = cwhub.InitHubUpdate(csConfig.Hub, hubURLTemplate, branch, remoteIndexPath); err != nil { return fmt.Errorf("failed to get Hub index after retry: %w", err) } } @@ -146,28 +157,35 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error { return err } + branch := hubBranch + if branch == "" { + branch = chooseHubBranch() + } + + log.Debugf("Using branch '%s' for the hub", branch) + hub, err := require.Hub(csConfig) if err != nil { return err } log.Infof("Upgrading collections") - if err := hub.UpgradeConfig(cwhub.COLLECTIONS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.COLLECTIONS, "", force, hubURLTemplate, branch); err != nil { return err } log.Infof("Upgrading parsers") - if err := hub.UpgradeConfig(cwhub.PARSERS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.PARSERS, "", force, hubURLTemplate, branch); err != nil { return err } log.Infof("Upgrading scenarios") - if err := hub.UpgradeConfig(cwhub.SCENARIOS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.SCENARIOS, "", force, hubURLTemplate, branch); err != nil { return err } log.Infof("Upgrading postoverflows") - if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, "", force); err != nil { + if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, "", force, hubURLTemplate, branch); err != nil { return err } @@ -188,8 +206,6 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if return fmt.Errorf("you must configure cli before interacting with hub") } - cwhub.SetHubBranch() - return nil }, RunE: runHubUpgrade, diff --git a/cmd/crowdsec-cli/itemcommands.go b/cmd/crowdsec-cli/itemcommands.go index 0afa27787..df612ab1a 100644 --- a/cmd/crowdsec-cli/itemcommands.go +++ b/cmd/crowdsec-cli/itemcommands.go @@ -216,6 +216,8 @@ func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) return err } + branch := chooseHubBranch() + for _, name := range args { t := hub.GetItem(it.name, name) if t == nil { @@ -225,7 +227,7 @@ func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string) continue } - if err := hub.InstallItem(name, it.name, force, downloadOnly); err != nil { + if err := hub.InstallItem(name, it.name, force, downloadOnly, hubURLTemplate, branch); err != nil { if !ignoreError { return fmt.Errorf("error while installing '%s': %w", name, err) } @@ -367,8 +369,10 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) return err } + branch := chooseHubBranch() + if all { - if err := hub.UpgradeConfig(it.name, "", force); err != nil { + if err := hub.UpgradeConfig(it.name, "", force, hubURLTemplate, branch); err != nil { return err } return nil @@ -379,7 +383,7 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string) } for _, name := range args { - if err := hub.UpgradeConfig(it.name, name, force); err != nil { + if err := hub.UpgradeConfig(it.name, name, force, hubURLTemplate, branch); err != nil { return err } } diff --git a/cmd/crowdsec-cli/main.go b/cmd/crowdsec-cli/main.go index 1d3aa56a6..8f5561d3d 100644 --- a/cmd/crowdsec-cli/main.go +++ b/cmd/crowdsec-cli/main.go @@ -4,7 +4,6 @@ import ( "fmt" "os" "path/filepath" - "slices" "strings" "github.com/fatih/color" @@ -12,9 +11,9 @@ import ( log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/cobra/doc" + "slices" "github.com/crowdsecurity/crowdsec/pkg/csconfig" - "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/cwversion" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/fflag" @@ -66,9 +65,10 @@ func initConfig() { log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath) } - if cwhub.HubBranch == "" && csConfig.Cscli.HubBranch != "" { - cwhub.HubBranch = csConfig.Cscli.HubBranch + if hubBranch == "" && csConfig.Cscli.HubBranch != "" { + hubBranch = csConfig.Cscli.HubBranch } + if OutputFormat != "" { csConfig.Cscli.Output = OutputFormat if OutputFormat != "json" && OutputFormat != "raw" && OutputFormat != "human" { @@ -197,7 +197,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error") rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace") - rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "", "Override hub branch on github") + rootCmd.PersistentFlags().StringVar(&hubBranch, "branch", "", "Override hub branch on github") if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil { log.Fatalf("failed to hide flag: %s", err) } diff --git a/cmd/crowdsec-cli/require/require.go b/cmd/crowdsec-cli/require/require.go index b477e9e00..2985a06cf 100644 --- a/cmd/crowdsec-cli/require/require.go +++ b/cmd/crowdsec-cli/require/require.go @@ -64,13 +64,11 @@ func Notifications(c *csconfig.Config) error { return nil } -func Hub (c *csconfig.Config) (*cwhub.Hub, error) { +func Hub(c *csconfig.Config) (*cwhub.Hub, error) { if c.Hub == nil { return nil, fmt.Errorf("you must configure cli before interacting with hub") } - cwhub.SetHubBranch() - hub, err := cwhub.InitHub(c.Hub) if err != nil { return nil, fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err) diff --git a/cmd/crowdsec-cli/setup.go b/cmd/crowdsec-cli/setup.go index cc0a9a35d..f589951e0 100644 --- a/cmd/crowdsec-cli/setup.go +++ b/cmd/crowdsec-cli/setup.go @@ -6,10 +6,10 @@ import ( "os" "os/exec" + goccyyaml "github.com/goccy/go-yaml" log "github.com/sirupsen/logrus" "github.com/spf13/cobra" "gopkg.in/yaml.v3" - goccyyaml "github.com/goccy/go-yaml" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/setup" @@ -298,12 +298,14 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error { return err } + branch := chooseHubBranch() + input, err := os.ReadFile(fromFile) if err != nil { return fmt.Errorf("while reading file %s: %w", fromFile, err) } - if err = setup.InstallHubItems(csConfig, input, dryRun); err != nil { + if err = setup.InstallHubItems(csConfig, input, dryRun, hubURLTemplate, branch); err != nil { return err } diff --git a/docker/docker_start.sh b/docker/docker_start.sh index d328cff1a..420b7b8d9 100755 --- a/docker/docker_start.sh +++ b/docker/docker_start.sh @@ -327,22 +327,22 @@ fi ## Remove collections, parsers, scenarios & postoverflows if [ "$DISABLE_COLLECTIONS" != "" ]; then # shellcheck disable=SC2086 - cscli_if_clean collections remove "$DISABLE_COLLECTIONS" + cscli_if_clean collections remove "$DISABLE_COLLECTIONS" --force fi if [ "$DISABLE_PARSERS" != "" ]; then # shellcheck disable=SC2086 - cscli_if_clean parsers remove "$DISABLE_PARSERS" + cscli_if_clean parsers remove "$DISABLE_PARSERS" --force fi if [ "$DISABLE_SCENARIOS" != "" ]; then # shellcheck disable=SC2086 - cscli_if_clean scenarios remove "$DISABLE_SCENARIOS" + cscli_if_clean scenarios remove "$DISABLE_SCENARIOS" --force fi if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then # shellcheck disable=SC2086 - cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" + cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" --force fi ## Register bouncers via env diff --git a/pkg/csconfig/hub.go b/pkg/csconfig/hub.go index db49c0231..d3638c888 100644 --- a/pkg/csconfig/hub.go +++ b/pkg/csconfig/hub.go @@ -1,6 +1,6 @@ package csconfig -// HubConfig holds the configuration for a hub +// HubCfg holds the configuration for a hub type HubCfg struct { HubIndexFile string // Path to the local index file HubDir string // Where the hub items are downloaded @@ -9,9 +9,6 @@ type HubCfg struct { } func (c *Config) loadHub() error { - - // XXX: HubBranch too -- from cscli or chooseHubBranch() ? - c.Hub = &HubCfg{ HubIndexFile: c.ConfigPaths.HubIndexFile, HubDir: c.ConfigPaths.HubDir, diff --git a/pkg/cwhub/cwhub.go b/pkg/cwhub/cwhub.go index d1f155813..24c2d859e 100644 --- a/pkg/cwhub/cwhub.go +++ b/pkg/cwhub/cwhub.go @@ -8,9 +8,4 @@ import ( "errors" ) -var ( - ErrMissingReference = errors.New("Reference(s) missing in collection") - - RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s" - HubBranch = "master" -) +var ErrMissingReference = errors.New("Reference(s) missing in collection") diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index b51c3aa6e..45e2852bd 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -14,6 +14,8 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) +const mockURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s" + /* To test : - Download 'first' hub index @@ -58,15 +60,16 @@ func testHub(t *testing.T, update bool) *Hub { os.RemoveAll(tmpDir) }) - constructor := InitHub + var hub *Hub if update { - constructor = InitHubUpdate + hub, err = InitHubUpdate(hubCfg, mockURLTemplate, "master", ".index.json") + require.NoError(t, err) + } else { + hub, err = InitHub(hubCfg) + require.NoError(t, err) } - hub, err := constructor(hubCfg) - require.NoError(t, err) - return hub } diff --git a/pkg/cwhub/enable_test.go b/pkg/cwhub/enable_test.go index 864b3fa85..a5d731fee 100644 --- a/pkg/cwhub/enable_test.go +++ b/pkg/cwhub/enable_test.go @@ -10,7 +10,7 @@ import ( func testInstall(hub *Hub, t *testing.T, item Item) { // Install the parser - err := hub.DownloadLatest(&item, false, false) + err := hub.DownloadLatest(&item, false, false, mockURLTemplate, "master") require.NoError(t, err, "failed to download %s", item.Name) _, err = hub.LocalSync() @@ -51,7 +51,7 @@ func testUpdate(hub *Hub, t *testing.T, item Item) { assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name) // Update it + check status - err := hub.DownloadLatest(&item, true, true) + err := hub.DownloadLatest(&item, true, true, mockURLTemplate, "master") require.NoError(t, err, "failed to update %s", item.Name) // Local sync and check status diff --git a/pkg/cwhub/helpers.go b/pkg/cwhub/helpers.go index ad66336be..6f42bed65 100644 --- a/pkg/cwhub/helpers.go +++ b/pkg/cwhub/helpers.go @@ -19,7 +19,7 @@ import ( ) // InstallItem installs an item from the hub -func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool) error { +func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool, hubURLTemplate, branch string) error { item := h.GetItem(itemType, name) if item == nil { return fmt.Errorf("unable to retrieve item: %s", name) @@ -33,7 +33,7 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly } } - err := h.DownloadLatest(item, force, true) + err := h.DownloadLatest(item, force, true, hubURLTemplate, branch) if err != nil { return fmt.Errorf("while downloading %s: %w", item.Name, err) } @@ -111,7 +111,7 @@ func (h *Hub) RemoveMany(itemType string, name string, all bool, purge bool, for } // UpgradeConfig upgrades an item from the hub -func (h *Hub) UpgradeConfig(itemType string, name string, force bool) error { +func (h *Hub) UpgradeConfig(itemType string, name string, force bool, hubURLTemplate, branch string) error { updated := 0 found := false @@ -144,7 +144,7 @@ func (h *Hub) UpgradeConfig(itemType string, name string, force bool) error { } } - if err := h.DownloadLatest(&v, force, true); err != nil { + if err := h.DownloadLatest(&v, force, true, hubURLTemplate, branch); err != nil { return fmt.Errorf("%s: download failed: %w", v.Name, err) } @@ -185,7 +185,7 @@ func (h *Hub) UpgradeConfig(itemType string, name string, force bool) error { } // DownloadLatest will download the latest version of Item to the tdir directory -func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) error { +func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool, hubURLTemplate, branch string) error { var err error log.Debugf("Downloading %s %s", target.Type, target.Name) @@ -196,7 +196,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro return nil } - return h.DownloadItem(target, overwrite) + return h.DownloadItem(target, overwrite, hubURLTemplate, branch) } // collection @@ -216,7 +216,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro if sub.Type == COLLECTIONS { log.Tracef("collection, recurse") - err = h.DownloadLatest(&val, overwrite, updateOnly) + err = h.DownloadLatest(&val, overwrite, updateOnly, hubURLTemplate, branch) if err != nil { return fmt.Errorf("while downloading %s: %w", val.Name, err) } @@ -224,7 +224,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro downloaded := val.Downloaded - err = h.DownloadItem(&val, overwrite) + err = h.DownloadItem(&val, overwrite, hubURLTemplate, branch) if err != nil { return fmt.Errorf("while downloading %s: %w", val.Name, err) } @@ -240,7 +240,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro h.Items[sub.Type][sub.Name] = val } - err = h.DownloadItem(target, overwrite) + err = h.DownloadItem(target, overwrite, hubURLTemplate, branch) if err != nil { return fmt.Errorf("failed to download item: %w", err) } @@ -248,7 +248,8 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro return nil } -func (h *Hub) DownloadItem(target *Item, overwrite bool) error { +func (h *Hub) DownloadItem(target *Item, overwrite bool, hubURLTemplate, branch string) error { + url := fmt.Sprintf(hubURLTemplate, branch, target.RemotePath) tdir := h.cfg.HubDir // if user didn't --force, don't overwrite local, tainted, up-to-date files @@ -264,7 +265,7 @@ func (h *Hub) DownloadItem(target *Item, overwrite bool) error { } } - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil) + req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return fmt.Errorf("while downloading %s: %w", req.URL.String(), err) } diff --git a/pkg/cwhub/helpers_test.go b/pkg/cwhub/helpers_test.go index 1ece43269..234e54caa 100644 --- a/pkg/cwhub/helpers_test.go +++ b/pkg/cwhub/helpers_test.go @@ -17,7 +17,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) - require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) + require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false, mockURLTemplate, "master")) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) @@ -33,7 +33,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { // collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario" pushUpdateToCollectionInHub() - hub, err := InitHubUpdate(hub.cfg) + hub, err := InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json") require.NoError(t, err, "failed to download index: %s", err) hub = getHubOrFail(t, hub.cfg) @@ -43,7 +43,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) { require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted) - err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false) + err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false, mockURLTemplate, "master") require.NoError(t, err) assertCollectionDepsInstalled(t, "crowdsecurity/test_collection") @@ -61,7 +61,7 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) + require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false, mockURLTemplate, "master")) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) @@ -81,10 +81,10 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) { require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate) - hub, err = InitHubUpdate(hub.cfg) + hub, err = InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json") require.NoError(t, err, "failed to download index: %s", err) - err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false) + err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false, mockURLTemplate, "master") require.NoError(t, err) hub = getHubOrFail(t, hub.cfg) @@ -110,7 +110,7 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t * require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) - require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false)) + require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false, mockURLTemplate, "master")) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded) require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed) @@ -136,13 +136,13 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t * // we just removed. Nor should it install the newly added scenario pushUpdateToCollectionInHub() - hub, err = InitHubUpdate(hub.cfg) + hub, err = InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json") require.NoError(t, err, "failed to download index: %s", err) require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed) hub = getHubOrFail(t, hub.cfg) - err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false) + err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false, mockURLTemplate, "master") require.NoError(t, err) hub = getHubOrFail(t, hub.cfg) diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index 2a2bfbb0a..3adf1c308 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -15,7 +15,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) -const HubIndexFile = ".index.json" +// const HubIndexFile = ".index.json" // Hub represents the runtime status of the hub (parsed items, etc.) type Hub struct { @@ -78,12 +78,12 @@ func InitHub(cfg *csconfig.HubCfg) (*Hub, error) { // InitHubUpdate is like InitHub but downloads and updates the index instead of reading from the disk // It is used to inizialize the hub when there is no index file yet -func InitHubUpdate(cfg *csconfig.HubCfg) (*Hub, error) { +func InitHubUpdate(cfg *csconfig.HubCfg, urlTemplate, branch, remotePath string) (*Hub, error) { if cfg == nil { return nil, fmt.Errorf("no configuration found for hub") } - bidx, err := DownloadIndex(cfg.HubIndexFile) + bidx, err := DownloadIndex(cfg.HubIndexFile, urlTemplate, branch, remotePath) if err != nil { return nil, fmt.Errorf("failed to download index: %w", err) } @@ -108,10 +108,11 @@ func InitHubUpdate(cfg *csconfig.HubCfg) (*Hub, error) { } // DownloadIndex downloads the latest version of the index and returns the content -func DownloadIndex(indexPath string) ([]byte, error) { - log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile)) +func DownloadIndex(localPath, hubURLTemplate, branch, remotePath string) ([]byte, error) { + url := fmt.Sprintf(hubURLTemplate, branch, remotePath) + log.Debugf("fetching index from branch %s (%s)", branch, url) - req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil) + req, err := http.NewRequest(http.MethodGet, url, nil) if err != nil { return nil, fmt.Errorf("failed to build request for hub index: %w", err) } @@ -135,17 +136,17 @@ func DownloadIndex(indexPath string) ([]byte, error) { return nil, fmt.Errorf("failed to read request answer for hub index: %w", err) } - oldContent, err := os.ReadFile(indexPath) + oldContent, err := os.ReadFile(localPath) if err != nil { if !os.IsNotExist(err) { log.Warningf("failed to read hub index: %s", err) } } else if bytes.Equal(body, oldContent) { log.Info("hub index is up to date") - // write it anyway, can't hurt + return body, nil } - file, err := os.OpenFile(indexPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) + file, err := os.OpenFile(localPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644) if err != nil { return nil, fmt.Errorf("while opening hub index file: %w", err) @@ -157,7 +158,7 @@ func DownloadIndex(indexPath string) ([]byte, error) { return nil, fmt.Errorf("while writing hub index file: %w", err) } - log.Infof("Wrote new %d bytes index to %s", wsize, indexPath) + log.Infof("Wrote index to %s, %d bytes", localPath, wsize) return body, nil } @@ -209,7 +210,7 @@ func ParseIndex(buff []byte) (HubItems, error) { } // ItemStats returns total counts of the hub items -func (h Hub) ItemStats() []string { +func (h *Hub) ItemStats() []string { loaded := "" for _, itemType := range ItemTypes { // ensure the order is always the same diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index 1b3a31109..02c552c4c 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -13,7 +13,7 @@ import ( func TestInitHubUpdate(t *testing.T) { hub := envSetup(t) - _, err := InitHubUpdate(hub.cfg) + _, err := InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json") require.NoError(t, err) _, err = GetHub() @@ -21,7 +21,6 @@ func TestInitHubUpdate(t *testing.T) { } func TestDownloadIndex(t *testing.T) { - back := RawFileURLTemplate // bad url template fmt.Println("Test 'bad URL'") @@ -32,9 +31,7 @@ func TestDownloadIndex(t *testing.T) { os.Remove(tmpIndex.Name()) }) - RawFileURLTemplate = "x" - - ret, err := DownloadIndex(tmpIndex.Name()) + ret, err := DownloadIndex(tmpIndex.Name(), "x", "", "") cstest.RequireErrorContains(t, err, "failed to build request for hub index: parse ") fmt.Printf("->%+v", ret) @@ -42,9 +39,7 @@ func TestDownloadIndex(t *testing.T) { // bad domain fmt.Println("Test 'bad domain'") - RawFileURLTemplate = "https://baddomain/%s/%s" - - ret, err = DownloadIndex(tmpIndex.Name()) + ret, err = DownloadIndex(tmpIndex.Name(), "https://baddomain/%s/%s", "master", ".index.json") cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") fmt.Printf("->%+v", ret) @@ -52,12 +47,8 @@ func TestDownloadIndex(t *testing.T) { // bad target path fmt.Println("Test 'bad target path'") - RawFileURLTemplate = back - - ret, err = DownloadIndex("/does/not/exist/index.json") + ret, err = DownloadIndex("/does/not/exist/index.json", mockURLTemplate, "master", ".index.json") cstest.RequireErrorContains(t, err, "while opening hub index file: open /does/not/exist/index.json:") - RawFileURLTemplate = back - fmt.Printf("->%+v", ret) } diff --git a/pkg/cwhub/sync.go b/pkg/cwhub/sync.go index a9360a93c..24f948d3c 100644 --- a/pkg/cwhub/sync.go +++ b/pkg/cwhub/sync.go @@ -106,7 +106,7 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) { log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype) // log.Infof("%s -> name:%s stage:%s", path, fname, stage) - + if ret.stage == SCENARIOS { ret.ftype = SCENARIOS ret.stage = "" @@ -394,6 +394,12 @@ func (h *Hub) SyncDir(dir string) ([]string, error) { log.Errorf("failed %s : %s", cpath, err) } + // explicit check for non existing directory, avoid spamming log.Debug + if _, err := os.Stat(cpath); os.IsNotExist(err) { + log.Tracef("directory %s doesn't exist, skipping", cpath) + continue + } + err = filepath.WalkDir(cpath, h.itemVisit) if err != nil { return warnings, err diff --git a/pkg/setup/install.go b/pkg/setup/install.go index 347340253..0c38db361 100644 --- a/pkg/setup/install.go +++ b/pkg/setup/install.go @@ -46,14 +46,12 @@ func decodeSetup(input []byte, fancyErrors bool) (Setup, error) { } // InstallHubItems installs the objects recommended in a setup file. -func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error { +func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool, hubURLTemplate, branch string) error { setupEnvelope, err := decodeSetup(input, false) if err != nil { return err } - cwhub.SetHubBranch() - hub, err := cwhub.InitHub(csConfig.Hub) if err != nil { return fmt.Errorf("getting hub index: %w", err) @@ -76,7 +74,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := hub.InstallItem(collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil { + if err := hub.InstallItem(collection, cwhub.COLLECTIONS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil { return fmt.Errorf("while installing collection %s: %w", collection, err) } } @@ -90,7 +88,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := hub.InstallItem(parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil { + if err := hub.InstallItem(parser, cwhub.PARSERS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil { return fmt.Errorf("while installing parser %s: %w", parser, err) } } @@ -104,7 +102,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := hub.InstallItem(scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil { + if err := hub.InstallItem(scenario, cwhub.SCENARIOS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil { return fmt.Errorf("while installing scenario %s: %w", scenario, err) } } @@ -118,7 +116,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error continue } - if err := hub.InstallItem(postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil { + if err := hub.InstallItem(postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil { return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err) } }