beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Refactor hub URL/branch configuration (#2559)

Browse source 
* Refactor hub URL/branch configuration
* docker: using --force to implement $DISABLE (required for items in collections)
* use pointer receiver for consistency
This commit is contained in:
mmetc 2023-10-27 10:25:29 +02:00 committed by GitHub
parent ac98256602
commit 6b8ed0c9d0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 117 additions and 118 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
pkg/cwhub/branch.go → cmd/crowdsec-cli/branch.go
View file

@ -1,4 +1,4 @@
package cwhub
package main
// Set the appropriate hub branch according to config settings and crowdsec version
@ -44,16 +44,3 @@ func chooseHubBranch() string {
return csVersion
}
// SetHubBranch sets the package variable that points to the hub branch.
func SetHubBranch() {
// a branch is already set, or specified from the flags
if HubBranch != "" {
return
}
// use the branch corresponding to the crowdsec version
HubBranch = chooseHubBranch()
log.Debugf("Using branch '%s' for the hub", HubBranch)
}

8
cmd/crowdsec-cli/config_restore.go
View file

@ -22,7 +22,7 @@ type OldAPICfg struct {
}
// it's a rip of the cli version, but in silent-mode
func silentInstallItem(name string, obtype string) (string, error) {
func silentInstallItem(name, obtype, hubURLTemplate, branch string) (string, error) {
hub, err := cwhub.GetHub()
if err != nil {
return "", err
@ -32,7 +32,7 @@ func silentInstallItem(name string, obtype string) (string, error) {
if item == nil {
return "", fmt.Errorf("error retrieving item")
}
err = hub.DownloadLatest(item, false, false)
err = hub.DownloadLatest(item, false, false, hubURLTemplate, branch)
if err != nil {
return "", fmt.Errorf("error while downloading %s : %v", item.Name, err)
}
@ -53,7 +53,7 @@ func silentInstallItem(name string, obtype string) (string, error) {
func restoreHub(dirPath string) error {
var err error
cwhub.SetHubBranch()
branch := chooseHubBranch()
for _, itype := range cwhub.ItemTypes {
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
@ -73,7 +73,7 @@ func restoreHub(dirPath string) error {
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
}
for _, toinstall := range upstreamList {
label, err := silentInstallItem(toinstall, itype)
label, err := silentInstallItem(toinstall, itype, hubURLTemplate, branch)
if err != nil {
log.Errorf("Error while installing %s : %s", toinstall, err)
} else if label != "" {

44
cmd/crowdsec-cli/hub.go
View file

@ -12,6 +12,13 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
const (
hubURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
remoteIndexPath = ".index.json"
)
var hubBranch = ""
func NewHubCmd() *cobra.Command {
var cmdHub = &cobra.Command{
Use: "hub [action]",
@ -33,7 +40,6 @@ cscli hub upgrade`,
return nil
},
}
cmdHub.PersistentFlags().StringVarP(&cwhub.HubBranch, "branch", "b", "", "Use given branch from hub")
cmdHub.AddCommand(NewHubListCmd())
cmdHub.AddCommand(NewHubUpdateCmd())
@ -81,7 +87,7 @@ func NewHubListCmd() *cobra.Command {
Short: "List all installed configurations",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: runHubList,
RunE: runHubList,
}
flags := cmdHubList.Flags()
@ -91,18 +97,23 @@ func NewHubListCmd() *cobra.Command {
}
func runHubUpdate(cmd *cobra.Command, args []string) error {
cwhub.SetHubBranch()
// don't use require.Hub because if there is no index file, it would fail
hub, err := cwhub.InitHubUpdate(csConfig.Hub)
branch := hubBranch
if branch == "" {
branch = chooseHubBranch()
}
log.Debugf("Using branch '%s' for the hub", branch)
hub, err := cwhub.InitHubUpdate(csConfig.Hub, hubURLTemplate, branch, remoteIndexPath)
if err != nil {
if !errors.Is(err, cwhub.ErrIndexNotFound) {
return fmt.Errorf("failed to get Hub index : %w", err)
}
log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch)
cwhub.HubBranch = "master"
if hub, err = cwhub.InitHubUpdate(csConfig.Hub); err != nil {
log.Warnf("Could not find index file for branch '%s', using 'master'", branch)
branch = "master"
if hub, err = cwhub.InitHubUpdate(csConfig.Hub, hubURLTemplate, branch, remoteIndexPath); err != nil {
return fmt.Errorf("failed to get Hub index after retry: %w", err)
}
}
@ -146,28 +157,35 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error {
return err
}
branch := hubBranch
if branch == "" {
branch = chooseHubBranch()
}
log.Debugf("Using branch '%s' for the hub", branch)
hub, err := require.Hub(csConfig)
if err != nil {
return err
}
log.Infof("Upgrading collections")
if err := hub.UpgradeConfig(cwhub.COLLECTIONS, "", force); err != nil {
if err := hub.UpgradeConfig(cwhub.COLLECTIONS, "", force, hubURLTemplate, branch); err != nil {
return err
}
log.Infof("Upgrading parsers")
if err := hub.UpgradeConfig(cwhub.PARSERS, "", force); err != nil {
if err := hub.UpgradeConfig(cwhub.PARSERS, "", force, hubURLTemplate, branch); err != nil {
return err
}
log.Infof("Upgrading scenarios")
if err := hub.UpgradeConfig(cwhub.SCENARIOS, "", force); err != nil {
if err := hub.UpgradeConfig(cwhub.SCENARIOS, "", force, hubURLTemplate, branch); err != nil {
return err
}
log.Infof("Upgrading postoverflows")
if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, "", force); err != nil {
if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, "", force, hubURLTemplate, branch); err != nil {
return err
}
@ -188,8 +206,6 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
return fmt.Errorf("you must configure cli before interacting with hub")
}
cwhub.SetHubBranch()
return nil
},
RunE: runHubUpgrade,

10
cmd/crowdsec-cli/itemcommands.go
View file

@ -216,6 +216,8 @@ func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string)
return err
}
branch := chooseHubBranch()
for _, name := range args {
t := hub.GetItem(it.name, name)
if t == nil {
@ -225,7 +227,7 @@ func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string)
continue
}
if err := hub.InstallItem(name, it.name, force, downloadOnly); err != nil {
if err := hub.InstallItem(name, it.name, force, downloadOnly, hubURLTemplate, branch); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
@ -367,8 +369,10 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string)
return err
}
branch := chooseHubBranch()
if all {
if err := hub.UpgradeConfig(it.name, "", force); err != nil {
if err := hub.UpgradeConfig(it.name, "", force, hubURLTemplate, branch); err != nil {
return err
}
return nil
@ -379,7 +383,7 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string)
}
for _, name := range args {
if err := hub.UpgradeConfig(it.name, name, force); err != nil {
if err := hub.UpgradeConfig(it.name, name, force, hubURLTemplate, branch); err != nil {
return err
}
}

10
cmd/crowdsec-cli/main.go
View file

@ -4,7 +4,6 @@ import (
"fmt"
"os"
"path/filepath"
"slices"
"strings"
"github.com/fatih/color"
@ -12,9 +11,9 @@ import (
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/cobra/doc"
"slices"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/database"
"github.com/crowdsecurity/crowdsec/pkg/fflag"
@ -66,9 +65,10 @@ func initConfig() {
log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath)
}
if cwhub.HubBranch == "" && csConfig.Cscli.HubBranch != "" {
cwhub.HubBranch = csConfig.Cscli.HubBranch
if hubBranch == "" && csConfig.Cscli.HubBranch != "" {
hubBranch = csConfig.Cscli.HubBranch
}
if OutputFormat != "" {
csConfig.Cscli.Output = OutputFormat
if OutputFormat != "json" && OutputFormat != "raw" && OutputFormat != "human" {
@ -197,7 +197,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error")
rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace")
rootCmd.PersistentFlags().StringVar(&cwhub.HubBranch, "branch", "", "Override hub branch on github")
rootCmd.PersistentFlags().StringVar(&hubBranch, "branch", "", "Override hub branch on github")
if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil {
log.Fatalf("failed to hide flag: %s", err)
}

4
cmd/crowdsec-cli/require/require.go
View file

@ -64,13 +64,11 @@ func Notifications(c *csconfig.Config) error {
return nil
}
func Hub (c *csconfig.Config) (*cwhub.Hub, error) {
func Hub(c *csconfig.Config) (*cwhub.Hub, error) {
if c.Hub == nil {
return nil, fmt.Errorf("you must configure cli before interacting with hub")
}
cwhub.SetHubBranch()
hub, err := cwhub.InitHub(c.Hub)
if err != nil {
return nil, fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err)

6
cmd/crowdsec-cli/setup.go
View file

@ -6,10 +6,10 @@ import (
"os"
"os/exec"
goccyyaml "github.com/goccy/go-yaml"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
goccyyaml "github.com/goccy/go-yaml"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/setup"
@ -298,12 +298,14 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error {
return err
}
branch := chooseHubBranch()
input, err := os.ReadFile(fromFile)
if err != nil {
return fmt.Errorf("while reading file %s: %w", fromFile, err)
}
if err = setup.InstallHubItems(csConfig, input, dryRun); err != nil {
if err = setup.InstallHubItems(csConfig, input, dryRun, hubURLTemplate, branch); err != nil {
return err
}

8
docker/docker_start.sh
View file

@ -327,22 +327,22 @@ fi
## Remove collections, parsers, scenarios & postoverflows
if [ "$DISABLE_COLLECTIONS" != "" ]; then
# shellcheck disable=SC2086
cscli_if_clean collections remove "$DISABLE_COLLECTIONS"
cscli_if_clean collections remove "$DISABLE_COLLECTIONS" --force
fi
if [ "$DISABLE_PARSERS" != "" ]; then
# shellcheck disable=SC2086
cscli_if_clean parsers remove "$DISABLE_PARSERS"
cscli_if_clean parsers remove "$DISABLE_PARSERS" --force
fi
if [ "$DISABLE_SCENARIOS" != "" ]; then
# shellcheck disable=SC2086
cscli_if_clean scenarios remove "$DISABLE_SCENARIOS"
cscli_if_clean scenarios remove "$DISABLE_SCENARIOS" --force
fi
if [ "$DISABLE_POSTOVERFLOWS" != "" ]; then
# shellcheck disable=SC2086
cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS"
cscli_if_clean postoverflows remove "$DISABLE_POSTOVERFLOWS" --force
fi
## Register bouncers via env

5
pkg/csconfig/hub.go
View file

@ -1,6 +1,6 @@
package csconfig
// HubConfig holds the configuration for a hub
// HubCfg holds the configuration for a hub
type HubCfg struct {
HubIndexFile string // Path to the local index file
HubDir string // Where the hub items are downloaded
@ -9,9 +9,6 @@ type HubCfg struct {
}
func (c *Config) loadHub() error {
// XXX: HubBranch too -- from cscli or chooseHubBranch() ?
c.Hub = &HubCfg{
HubIndexFile: c.ConfigPaths.HubIndexFile,
HubDir: c.ConfigPaths.HubDir,

7
pkg/cwhub/cwhub.go
View file

@ -8,9 +8,4 @@ import (
"errors"
)
var (
ErrMissingReference = errors.New("Reference(s) missing in collection")
RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
HubBranch = "master"
)
var ErrMissingReference = errors.New("Reference(s) missing in collection")

13
pkg/cwhub/cwhub_test.go
View file

@ -14,6 +14,8 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
const mockURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
/*
To test :
- Download 'first' hub index
@ -58,15 +60,16 @@ func testHub(t *testing.T, update bool) *Hub {
os.RemoveAll(tmpDir)
})
constructor := InitHub
var hub *Hub
if update {
constructor = InitHubUpdate
hub, err = InitHubUpdate(hubCfg, mockURLTemplate, "master", ".index.json")
require.NoError(t, err)
} else {
hub, err = InitHub(hubCfg)
require.NoError(t, err)
}
hub, err := constructor(hubCfg)
require.NoError(t, err)
return hub
}

4
pkg/cwhub/enable_test.go
View file

@ -10,7 +10,7 @@ import (
func testInstall(hub *Hub, t *testing.T, item Item) {
// Install the parser
err := hub.DownloadLatest(&item, false, false)
err := hub.DownloadLatest(&item, false, false, mockURLTemplate, "master")
require.NoError(t, err, "failed to download %s", item.Name)
_, err = hub.LocalSync()
@ -51,7 +51,7 @@ func testUpdate(hub *Hub, t *testing.T, item Item) {
assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
// Update it + check status
err := hub.DownloadLatest(&item, true, true)
err := hub.DownloadLatest(&item, true, true, mockURLTemplate, "master")
require.NoError(t, err, "failed to update %s", item.Name)
// Local sync and check status

23
pkg/cwhub/helpers.go
View file

@ -19,7 +19,7 @@ import (
)
// InstallItem installs an item from the hub
func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool) error {
func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool, hubURLTemplate, branch string) error {
item := h.GetItem(itemType, name)
if item == nil {
return fmt.Errorf("unable to retrieve item: %s", name)
@ -33,7 +33,7 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly
}
}
err := h.DownloadLatest(item, force, true)
err := h.DownloadLatest(item, force, true, hubURLTemplate, branch)
if err != nil {
return fmt.Errorf("while downloading %s: %w", item.Name, err)
}
@ -111,7 +111,7 @@ func (h *Hub) RemoveMany(itemType string, name string, all bool, purge bool, for
}
// UpgradeConfig upgrades an item from the hub
func (h *Hub) UpgradeConfig(itemType string, name string, force bool) error {
func (h *Hub) UpgradeConfig(itemType string, name string, force bool, hubURLTemplate, branch string) error {
updated := 0
found := false
@ -144,7 +144,7 @@ func (h *Hub) UpgradeConfig(itemType string, name string, force bool) error {
}
}
if err := h.DownloadLatest(&v, force, true); err != nil {
if err := h.DownloadLatest(&v, force, true, hubURLTemplate, branch); err != nil {
return fmt.Errorf("%s: download failed: %w", v.Name, err)
}
@ -185,7 +185,7 @@ func (h *Hub) UpgradeConfig(itemType string, name string, force bool) error {
}
// DownloadLatest will download the latest version of Item to the tdir directory
func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) error {
func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool, hubURLTemplate, branch string) error {
var err error
log.Debugf("Downloading %s %s", target.Type, target.Name)
@ -196,7 +196,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro
return nil
}
return h.DownloadItem(target, overwrite)
return h.DownloadItem(target, overwrite, hubURLTemplate, branch)
}
// collection
@ -216,7 +216,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro
if sub.Type == COLLECTIONS {
log.Tracef("collection, recurse")
err = h.DownloadLatest(&val, overwrite, updateOnly)
err = h.DownloadLatest(&val, overwrite, updateOnly, hubURLTemplate, branch)
if err != nil {
return fmt.Errorf("while downloading %s: %w", val.Name, err)
}
@ -224,7 +224,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro
downloaded := val.Downloaded
err = h.DownloadItem(&val, overwrite)
err = h.DownloadItem(&val, overwrite, hubURLTemplate, branch)
if err != nil {
return fmt.Errorf("while downloading %s: %w", val.Name, err)
}
@ -240,7 +240,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro
h.Items[sub.Type][sub.Name] = val
}
err = h.DownloadItem(target, overwrite)
err = h.DownloadItem(target, overwrite, hubURLTemplate, branch)
if err != nil {
return fmt.Errorf("failed to download item: %w", err)
}
@ -248,7 +248,8 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) erro
return nil
}
func (h *Hub) DownloadItem(target *Item, overwrite bool) error {
func (h *Hub) DownloadItem(target *Item, overwrite bool, hubURLTemplate, branch string) error {
url := fmt.Sprintf(hubURLTemplate, branch, target.RemotePath)
tdir := h.cfg.HubDir
// if user didn't --force, don't overwrite local, tainted, up-to-date files
@ -264,7 +265,7 @@ func (h *Hub) DownloadItem(target *Item, overwrite bool) error {
}
}
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil)
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return fmt.Errorf("while downloading %s: %w", req.URL.String(), err)
}

18
pkg/cwhub/helpers_test.go
View file

@ -17,7 +17,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false))
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false, mockURLTemplate, "master"))
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
@ -33,7 +33,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
pushUpdateToCollectionInHub()
hub, err := InitHubUpdate(hub.cfg)
hub, err := InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json")
require.NoError(t, err, "failed to download index: %s", err)
hub = getHubOrFail(t, hub.cfg)
@ -43,7 +43,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false)
err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false, mockURLTemplate, "master")
require.NoError(t, err)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
@ -61,7 +61,7 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false))
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false, mockURLTemplate, "master"))
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
@ -81,10 +81,10 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
hub, err = InitHubUpdate(hub.cfg)
hub, err = InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json")
require.NoError(t, err, "failed to download index: %s", err)
err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false)
err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false, mockURLTemplate, "master")
require.NoError(t, err)
hub = getHubOrFail(t, hub.cfg)
@ -110,7 +110,7 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false))
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false, mockURLTemplate, "master"))
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
@ -136,13 +136,13 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *
// we just removed. Nor should it install the newly added scenario
pushUpdateToCollectionInHub()
hub, err = InitHubUpdate(hub.cfg)
hub, err = InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json")
require.NoError(t, err, "failed to download index: %s", err)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
hub = getHubOrFail(t, hub.cfg)
err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false)
err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false, mockURLTemplate, "master")
require.NoError(t, err)
hub = getHubOrFail(t, hub.cfg)

23
pkg/cwhub/hub.go
View file

@ -15,7 +15,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
const HubIndexFile = ".index.json"
// const HubIndexFile = ".index.json"
// Hub represents the runtime status of the hub (parsed items, etc.)
type Hub struct {
@ -78,12 +78,12 @@ func InitHub(cfg *csconfig.HubCfg) (*Hub, error) {
// InitHubUpdate is like InitHub but downloads and updates the index instead of reading from the disk
// It is used to inizialize the hub when there is no index file yet
func InitHubUpdate(cfg *csconfig.HubCfg) (*Hub, error) {
func InitHubUpdate(cfg *csconfig.HubCfg, urlTemplate, branch, remotePath string) (*Hub, error) {
if cfg == nil {
return nil, fmt.Errorf("no configuration found for hub")
}
bidx, err := DownloadIndex(cfg.HubIndexFile)
bidx, err := DownloadIndex(cfg.HubIndexFile, urlTemplate, branch, remotePath)
if err != nil {
return nil, fmt.Errorf("failed to download index: %w", err)
}
@ -108,10 +108,11 @@ func InitHubUpdate(cfg *csconfig.HubCfg) (*Hub, error) {
}
// DownloadIndex downloads the latest version of the index and returns the content
func DownloadIndex(indexPath string) ([]byte, error) {
log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
func DownloadIndex(localPath, hubURLTemplate, branch, remotePath string) ([]byte, error) {
url := fmt.Sprintf(hubURLTemplate, branch, remotePath)
log.Debugf("fetching index from branch %s (%s)", branch, url)
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
return nil, fmt.Errorf("failed to build request for hub index: %w", err)
}
@ -135,17 +136,17 @@ func DownloadIndex(indexPath string) ([]byte, error) {
return nil, fmt.Errorf("failed to read request answer for hub index: %w", err)
}
oldContent, err := os.ReadFile(indexPath)
oldContent, err := os.ReadFile(localPath)
if err != nil {
if !os.IsNotExist(err) {
log.Warningf("failed to read hub index: %s", err)
}
} else if bytes.Equal(body, oldContent) {
log.Info("hub index is up to date")
// write it anyway, can't hurt
return body, nil
}
file, err := os.OpenFile(indexPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
file, err := os.OpenFile(localPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0o644)
if err != nil {
return nil, fmt.Errorf("while opening hub index file: %w", err)
@ -157,7 +158,7 @@ func DownloadIndex(indexPath string) ([]byte, error) {
return nil, fmt.Errorf("while writing hub index file: %w", err)
}
log.Infof("Wrote new %d bytes index to %s", wsize, indexPath)
log.Infof("Wrote index to %s, %d bytes", localPath, wsize)
return body, nil
}
@ -209,7 +210,7 @@ func ParseIndex(buff []byte) (HubItems, error) {
}
// ItemStats returns total counts of the hub items
func (h Hub) ItemStats() []string {
func (h *Hub) ItemStats() []string {
loaded := ""
for _, itemType := range ItemTypes {
// ensure the order is always the same

17
pkg/cwhub/hub_test.go
View file

@ -13,7 +13,7 @@ import (
func TestInitHubUpdate(t *testing.T) {
hub := envSetup(t)
_, err := InitHubUpdate(hub.cfg)
_, err := InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json")
require.NoError(t, err)
_, err = GetHub()
@ -21,7 +21,6 @@ func TestInitHubUpdate(t *testing.T) {
}
func TestDownloadIndex(t *testing.T) {
back := RawFileURLTemplate
// bad url template
fmt.Println("Test 'bad URL'")
@ -32,9 +31,7 @@ func TestDownloadIndex(t *testing.T) {
os.Remove(tmpIndex.Name())
})
RawFileURLTemplate = "x"
ret, err := DownloadIndex(tmpIndex.Name())
ret, err := DownloadIndex(tmpIndex.Name(), "x", "", "")
cstest.RequireErrorContains(t, err, "failed to build request for hub index: parse ")
fmt.Printf("->%+v", ret)
@ -42,9 +39,7 @@ func TestDownloadIndex(t *testing.T) {
// bad domain
fmt.Println("Test 'bad domain'")
RawFileURLTemplate = "https://baddomain/%s/%s"
ret, err = DownloadIndex(tmpIndex.Name())
ret, err = DownloadIndex(tmpIndex.Name(), "https://baddomain/%s/%s", "master", ".index.json")
cstest.RequireErrorContains(t, err, "failed http request for hub index: Get")
fmt.Printf("->%+v", ret)
@ -52,12 +47,8 @@ func TestDownloadIndex(t *testing.T) {
// bad target path
fmt.Println("Test 'bad target path'")
RawFileURLTemplate = back
ret, err = DownloadIndex("/does/not/exist/index.json")
ret, err = DownloadIndex("/does/not/exist/index.json", mockURLTemplate, "master", ".index.json")
cstest.RequireErrorContains(t, err, "while opening hub index file: open /does/not/exist/index.json:")
RawFileURLTemplate = back
fmt.Printf("->%+v", ret)
}

8
pkg/cwhub/sync.go
View file

@ -106,7 +106,7 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) {
log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype)
// log.Infof("%s -> name:%s stage:%s", path, fname, stage)
if ret.stage == SCENARIOS {
ret.ftype = SCENARIOS
ret.stage = ""
@ -394,6 +394,12 @@ func (h *Hub) SyncDir(dir string) ([]string, error) {
log.Errorf("failed %s : %s", cpath, err)
}
// explicit check for non existing directory, avoid spamming log.Debug
if _, err := os.Stat(cpath); os.IsNotExist(err) {
log.Tracef("directory %s doesn't exist, skipping", cpath)
continue
}
err = filepath.WalkDir(cpath, h.itemVisit)
if err != nil {
return warnings, err

12
pkg/setup/install.go
View file

@ -46,14 +46,12 @@ func decodeSetup(input []byte, fancyErrors bool) (Setup, error) {
}
// InstallHubItems installs the objects recommended in a setup file.
func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error {
func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool, hubURLTemplate, branch string) error {
setupEnvelope, err := decodeSetup(input, false)
if err != nil {
return err
}
cwhub.SetHubBranch()
hub, err := cwhub.InitHub(csConfig.Hub)
if err != nil {
return fmt.Errorf("getting hub index: %w", err)
@ -76,7 +74,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
continue
}
if err := hub.InstallItem(collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
if err := hub.InstallItem(collection, cwhub.COLLECTIONS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil {
return fmt.Errorf("while installing collection %s: %w", collection, err)
}
}
@ -90,7 +88,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
continue
}
if err := hub.InstallItem(parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
if err := hub.InstallItem(parser, cwhub.PARSERS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil {
return fmt.Errorf("while installing parser %s: %w", parser, err)
}
}
@ -104,7 +102,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
continue
}
if err := hub.InstallItem(scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
if err := hub.InstallItem(scenario, cwhub.SCENARIOS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil {
return fmt.Errorf("while installing scenario %s: %w", scenario, err)
}
}
@ -118,7 +116,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
continue
}
if err := hub.InstallItem(postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil {
if err := hub.InstallItem(postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil {
return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err)
}
}