beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Refact pkg/csconfig, pkg/cwhub (#2555)

Browse source 
* csconfig: drop redundant hub information on *Cfg structs
* rename validItemFileName() -> item.validPath()
* Methods on hub object
* updated tests to reduce need of csconfig.Config or global state
This commit is contained in:
mmetc 2023-10-19 12:04:29 +02:00 committed by GitHub
parent 57d3ebba12
commit 88e4f7c157
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
43 changed files with 669 additions and 545 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
cmd/crowdsec-cli/capi.go
View file

@ -151,11 +151,12 @@ func NewCapiStatusCmd() *cobra.Command {
return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err)
}
if err := require.Hub(csConfig); err != nil {
hub, err := require.Hub(csConfig)
if err != nil {
return err
}
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("failed to get scenarios: %w", err)
}

31
cmd/crowdsec-cli/collections.go
View file

@ -25,7 +25,7 @@ cscli collections remove crowdsecurity/http-cve crowdsecurity/iptables
Aliases: []string{"collection"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
if _, err := require.Hub(csConfig); err != nil {
return err
}
@ -66,8 +66,13 @@ func runCollectionsInstall(cmd *cobra.Command, args []string) error {
return err
}
hub, err := cwhub.GetHub()
if err != nil {
return err
}
for _, name := range args {
t := cwhub.GetItem(cwhub.COLLECTIONS, name)
t := hub.GetItem(cwhub.COLLECTIONS, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.COLLECTIONS, name)
Suggest(cwhub.COLLECTIONS, name, nearestItem.Name, score, ignoreError)
@ -75,7 +80,7 @@ func runCollectionsInstall(cmd *cobra.Command, args []string) error {
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.COLLECTIONS, force, downloadOnly); err != nil {
if err := hub.InstallItem(name, cwhub.COLLECTIONS, force, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
@ -126,8 +131,13 @@ func runCollectionsRemove(cmd *cobra.Command, args []string) error {
return err
}
hub, err := cwhub.GetHub()
if err != nil {
return err
}
if all {
err := cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, "", all, purge, force)
err := hub.RemoveMany(cwhub.COLLECTIONS, "", all, purge, force)
if err != nil {
return err
}
@ -141,7 +151,7 @@ func runCollectionsRemove(cmd *cobra.Command, args []string) error {
for _, name := range args {
if !force {
item := cwhub.GetItem(cwhub.COLLECTIONS, name)
item := hub.GetItem(cwhub.COLLECTIONS, name)
if item == nil {
// XXX: this should be in GetItem?
return fmt.Errorf("can't find '%s' in %s", name, cwhub.COLLECTIONS)
@ -153,7 +163,7 @@ func runCollectionsRemove(cmd *cobra.Command, args []string) error {
}
}
err := cwhub.RemoveMany(csConfig, cwhub.COLLECTIONS, name, all, purge, force)
err := hub.RemoveMany(cwhub.COLLECTIONS, name, all, purge, force)
if err != nil {
return err
}
@ -197,8 +207,13 @@ func runCollectionsUpgrade(cmd *cobra.Command, args []string) error {
return err
}
hub, err := cwhub.GetHub()
if err != nil {
return err
}
if all {
if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", force); err != nil {
if err := hub.UpgradeConfig(cwhub.COLLECTIONS, "", force); err != nil {
return err
}
return nil
@ -209,7 +224,7 @@ func runCollectionsUpgrade(cmd *cobra.Command, args []string) error {
}
for _, name := range args {
if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, name, force); err != nil {
if err := hub.UpgradeConfig(cwhub.COLLECTIONS, name, force); err != nil {
return err
}
}

13
cmd/crowdsec-cli/config_backup.go
View file

@ -13,7 +13,7 @@ import (
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
func backupHub(dirPath string) error {
func backupHub(hub *cwhub.Hub, dirPath string) error {
var err error
var itemDirectory string
var upstreamParsers []string
@ -22,7 +22,7 @@ func backupHub(dirPath string) error {
clog := log.WithFields(log.Fields{
"type": itemType,
})
itemMap := cwhub.GetItemMap(itemType)
itemMap := hub.GetItemMap(itemType)
if itemMap == nil {
clog.Infof("No %s to backup.", itemType)
continue
@ -189,7 +189,12 @@ func backupConfigToDirectory(dirPath string) error {
log.Infof("Saved profiles to %s", backupProfiles)
}
if err = backupHub(dirPath); err != nil {
hub, err := cwhub.GetHub()
if err != nil {
return err
}
if err = backupHub(hub, dirPath); err != nil {
return fmt.Errorf("failed to backup hub config: %s", err)
}
@ -197,7 +202,7 @@ func backupConfigToDirectory(dirPath string) error {
}
func runConfigBackup(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
if _, err := require.Hub(csConfig); err != nil {
return err
}

27
cmd/crowdsec-cli/config_restore.go
View file

@ -23,23 +23,28 @@ type OldAPICfg struct {
// it's a rip of the cli version, but in silent-mode
func silentInstallItem(name string, obtype string) (string, error) {
var item = cwhub.GetItem(obtype, name)
if item == nil {
return "", fmt.Errorf("error retrieving item")
}
err := cwhub.DownloadLatest(csConfig.Hub, item, false, false)
hub, err := cwhub.GetHub()
if err != nil {
return "", fmt.Errorf("error while downloading %s : %v", item.Name, err)
}
if err := cwhub.AddItem(obtype, *item); err != nil {
return "", err
}
err = cwhub.EnableItem(csConfig.Hub, item)
var item = hub.GetItem(obtype, name)
if item == nil {
return "", fmt.Errorf("error retrieving item")
}
err = hub.DownloadLatest(item, false, false)
if err != nil {
return "", fmt.Errorf("error while downloading %s : %v", item.Name, err)
}
if err := hub.AddItem(obtype, *item); err != nil {
return "", err
}
err = hub.EnableItem(item)
if err != nil {
return "", fmt.Errorf("error while enabling %s : %v", item.Name, err)
}
if err := cwhub.AddItem(obtype, *item); err != nil {
if err := hub.AddItem(obtype, *item); err != nil {
return "", err
}
return fmt.Sprintf("Enabled %s", item.Name), nil
@ -292,7 +297,7 @@ func runConfigRestore(cmd *cobra.Command, args []string) error {
return err
}
if err := require.Hub(csConfig); err != nil {
if _, err := require.Hub(csConfig); err != nil {
return err
}

1
cmd/crowdsec-cli/config_show.go
View file

@ -82,7 +82,6 @@ Crowdsec{{if and .Crowdsec.Enable (not (ValueBool .Crowdsec.Enable))}} (disabled
cscli:
- Output : {{.Cscli.Output}}
- Hub Branch : {{.Cscli.HubBranch}}
- Hub Folder : {{.Cscli.HubDir}}
{{- end }}
{{- if .API }}

5
cmd/crowdsec-cli/console.go
View file

@ -71,11 +71,12 @@ After running this command your will need to validate the enrollment in the weba
return fmt.Errorf("could not parse CAPI URL: %s", err)
}
if err := require.Hub(csConfig); err != nil {
hub, err := require.Hub(csConfig)
if err != nil {
return err
}
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("failed to get installed scenarios: %s", err)
}

29
cmd/crowdsec-cli/hub.go
View file

@ -50,12 +50,13 @@ func runHubList(cmd *cobra.Command, args []string) error {
return err
}
if err = require.Hub(csConfig); err != nil {
hub, err := require.Hub(csConfig)
if err != nil {
return err
}
// use LocalSync to get warnings about tainted / outdated items
warn, _ := cwhub.LocalSync(csConfig.Hub)
warn, _ := hub.LocalSync()
for _, v := range warn {
log.Info(v)
}
@ -88,19 +89,24 @@ func NewHubListCmd() *cobra.Command {
}
func runHubUpdate(cmd *cobra.Command, args []string) error {
if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
cwhub.SetHubBranch()
// don't use require.Hub because if there is no index file, it would fail
hub, err := cwhub.InitHubUpdate(csConfig.Hub)
if err != nil {
if !errors.Is(err, cwhub.ErrIndexNotFound) {
return fmt.Errorf("failed to get Hub index : %w", err)
}
log.Warnf("Could not find index file for branch '%s', using 'master'", cwhub.HubBranch)
cwhub.HubBranch = "master"
if err := cwhub.UpdateHubIdx(csConfig.Hub); err != nil {
if hub, err = cwhub.InitHubUpdate(csConfig.Hub); err != nil {
return fmt.Errorf("failed to get Hub index after retry: %w", err)
}
}
// use LocalSync to get warnings about tainted / outdated items
warn, _ := cwhub.LocalSync(csConfig.Hub)
warn, _ := hub.LocalSync()
for _, v := range warn {
log.Info(v)
}
@ -122,8 +128,6 @@ Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.inde
return fmt.Errorf("you must configure cli before interacting with hub")
}
cwhub.SetHubBranch()
return nil
},
RunE: runHubUpdate,
@ -140,27 +144,28 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error {
return err
}
if err := require.Hub(csConfig); err != nil {
hub, err := require.Hub(csConfig)
if err != nil {
return err
}
log.Infof("Upgrading collections")
if err := cwhub.UpgradeConfig(csConfig, cwhub.COLLECTIONS, "", force); err != nil {
if err := hub.UpgradeConfig(cwhub.COLLECTIONS, "", force); err != nil {
return err
}
log.Infof("Upgrading parsers")
if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", force); err != nil {
if err := hub.UpgradeConfig(cwhub.PARSERS, "", force); err != nil {
return err
}
log.Infof("Upgrading scenarios")
if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", force); err != nil {
if err := hub.UpgradeConfig(cwhub.SCENARIOS, "", force); err != nil {
return err
}
log.Infof("Upgrading postoverflows")
if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, "", force); err != nil {
if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, "", force); err != nil {
return err
}

6
cmd/crowdsec-cli/item_metrics.go
View file

@ -18,7 +18,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func ShowMetrics(hubItem *cwhub.Item) {
func ShowMetrics(hub *cwhub.Hub, hubItem *cwhub.Item) {
switch hubItem.Type {
case cwhub.PARSERS:
metrics := GetParserMetric(csConfig.Cscli.PrometheusUrl, hubItem.Name)
@ -36,11 +36,11 @@ func ShowMetrics(hubItem *cwhub.Item) {
scenarioMetricsTable(color.Output, item, metrics)
}
for _, item := range hubItem.Collections {
hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
hubItem = hub.GetItem(cwhub.COLLECTIONS, item)
if hubItem == nil {
log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
}
ShowMetrics(hubItem)
ShowMetrics(hub, hubItem)
}
default:
log.Errorf("item of type '%s' is unknown", hubItem.Type)

18
cmd/crowdsec-cli/item_suggest.go
View file

@ -33,7 +33,11 @@ func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
allItems := make([]string, 0)
nearestScore := 100
nearestItem := &cwhub.Item{}
hubItems := cwhub.GetItemMap(itemType)
// XXX: handle error
hub, _ := cwhub.GetHub()
hubItems := hub.GetItemMap(itemType)
for _, item := range hubItems {
allItems = append(allItems, item.Name)
}
@ -42,19 +46,20 @@ func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
d := levenshtein.Distance(itemName, s, nil)
if d < nearestScore {
nearestScore = d
nearestItem = cwhub.GetItem(itemType, s)
nearestItem = hub.GetItem(itemType, s)
}
}
return nearestItem, nearestScore
}
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := require.Hub(csConfig); err != nil {
hub, err := require.Hub(csConfig)
if err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
comp := make([]string, 0)
hubItems := cwhub.GetItemMap(itemType)
hubItems := hub.GetItemMap(itemType)
for _, item := range hubItems {
if !slices.Contains(args, item.Name) && strings.Contains(item.Name, toComplete) {
comp = append(comp, item.Name)
@ -65,11 +70,12 @@ func compAllItems(itemType string, args []string, toComplete string) ([]string,
}
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
if err := require.Hub(csConfig); err != nil {
hub, err := require.Hub(csConfig)
if err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
items, err := cwhub.GetInstalledItemsAsString(itemType)
items, err := hub.GetInstalledItemsAsString(itemType)
if err != nil {
cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
return nil, cobra.ShellCompDirectiveDefault

32
cmd/crowdsec-cli/items.go
View file

@ -16,8 +16,8 @@ import (
)
func selectItems(itemType string, args []string, installedOnly bool) ([]string, error) {
itemNames := cwhub.GetItemNames(itemType)
func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]string, error) {
itemNames := hub.GetItemNames(itemType)
notExist := []string{}
if len(args) > 0 {
@ -40,7 +40,7 @@ func selectItems(itemType string, args []string, installedOnly bool) ([]string,
if installedOnly {
installed := []string{}
for _, item := range itemNames {
if cwhub.GetItem(itemType, item).Installed {
if hub.GetItem(itemType, item).Installed {
installed = append(installed, item)
}
}
@ -52,9 +52,15 @@ func selectItems(itemType string, args []string, installedOnly bool) ([]string,
func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) error {
var err error
hub, err := cwhub.GetHub()
if err != nil {
return err
}
items := make(map[string][]string)
for _, itemType := range itemTypes {
if items[itemType], err = selectItems(itemType, args, !all); err != nil {
if items[itemType], err = selectItems(hub, itemType, args, !all); err != nil {
return err
}
}
@ -78,7 +84,7 @@ func ListItems(out io.Writer, itemTypes []string, args []string, showType bool,
// empty slice in case there are no items of this type
hubStatus[itemType] = make([]itemHubStatus, len(items[itemType]))
for i, itemName := range items[itemType] {
item := cwhub.GetItem(itemType, itemName)
item := hub.GetItem(itemType, itemName)
status, emo := item.Status()
hubStatus[itemType][i] = itemHubStatus{
Name: item.Name,
@ -112,7 +118,7 @@ func ListItems(out io.Writer, itemTypes []string, args []string, showType bool,
}
for _, itemType := range itemTypes {
for _, itemName := range items[itemType] {
item := cwhub.GetItem(itemType, itemName)
item := hub.GetItem(itemType, itemName)
status, _ := item.Status()
if item.LocalVersion == "" {
item.LocalVersion = "n/a"
@ -138,15 +144,17 @@ func ListItems(out io.Writer, itemTypes []string, args []string, showType bool,
}
func InspectItem(name string, itemType string, noMetrics bool) error {
hubItem := cwhub.GetItem(itemType, name)
hub, err := cwhub.GetHub()
if err != nil {
return err
}
hubItem := hub.GetItem(itemType, name)
if hubItem == nil {
return fmt.Errorf("can't find '%s' in %s", name, itemType)
}
var (
b []byte
err error
)
var b []byte
switch csConfig.Cscli.Output {
case "human", "raw":
@ -168,7 +176,7 @@ func InspectItem(name string, itemType string, noMetrics bool) error {
}
fmt.Printf("\nCurrent metrics: \n")
ShowMetrics(hubItem)
ShowMetrics(hub, hubItem)
return nil
}

11
cmd/crowdsec-cli/lapi.go
View file

@ -38,11 +38,12 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
}
if err := require.Hub(csConfig); err != nil {
hub, err := require.Hub(csConfig)
if err != nil {
log.Fatal(err)
}
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
if err != nil {
log.Fatalf("failed to get scenarios : %s", err)
}
@ -338,9 +339,9 @@ cscli lapi context detect crowdsecurity/sshd-logs
log.Fatalf("Failed to init expr helpers : %s", err)
}
// Populate cwhub package tools
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
log.Fatalf("Failed to load hub index : %s", err)
_, err = require.Hub(csConfig)
if err != nil {
log.Fatal(err)
}
csParsers := parser.NewParsers()

29
cmd/crowdsec-cli/parsers.go
View file

@ -25,7 +25,7 @@ cscli parsers remove crowdsecurity/caddy-logs crowdsecurity/sshd-logs
Aliases: []string{"parser"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
if _, err := require.Hub(csConfig); err != nil {
return err
}
@ -66,8 +66,13 @@ func runParsersInstall(cmd *cobra.Command, args []string) error {
return err
}
hub, err := cwhub.GetHub()
if err != nil {
return err
}
for _, name := range args {
t := cwhub.GetItem(cwhub.PARSERS, name)
t := hub.GetItem(cwhub.PARSERS, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.PARSERS, name)
Suggest(cwhub.PARSERS, name, nearestItem.Name, score, ignoreError)
@ -75,7 +80,7 @@ func runParsersInstall(cmd *cobra.Command, args []string) error {
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.PARSERS, force, downloadOnly); err != nil {
if err := hub.InstallItem(name, cwhub.PARSERS, force, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
@ -126,8 +131,13 @@ func runParsersRemove(cmd *cobra.Command, args []string) error {
return err
}
hub, err := cwhub.GetHub()
if err != nil {
return err
}
if all {
err := cwhub.RemoveMany(csConfig, cwhub.PARSERS, "", all, purge, force)
err := hub.RemoveMany(cwhub.PARSERS, "", all, purge, force)
if err != nil {
return err
}
@ -140,7 +150,7 @@ func runParsersRemove(cmd *cobra.Command, args []string) error {
}
for _, name := range args {
err := cwhub.RemoveMany(csConfig, cwhub.PARSERS, name, all, purge, force)
err := hub.RemoveMany(cwhub.PARSERS, name, all, purge, force)
if err != nil {
return err
}
@ -184,8 +194,13 @@ func runParsersUpgrade(cmd *cobra.Command, args []string) error {
return err
}
hub, err := cwhub.GetHub()
if err != nil {
return err
}
if all {
if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, "", force); err != nil {
if err := hub.UpgradeConfig(cwhub.PARSERS, "", force); err != nil {
return err
}
return nil
@ -196,7 +211,7 @@ func runParsersUpgrade(cmd *cobra.Command, args []string) error {
}
for _, name := range args {
if err := cwhub.UpgradeConfig(csConfig, cwhub.PARSERS, name, force); err != nil {
if err := hub.UpgradeConfig(cwhub.PARSERS, name, force); err != nil {
return err
}
}

29
cmd/crowdsec-cli/postoverflows.go
View file

@ -25,7 +25,7 @@ cscli postoverflows remove crowdsecurity/cdn-whitelist crowdsecurity/rdns
Aliases: []string{"postoverflow"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
if _, err := require.Hub(csConfig); err != nil {
return err
}
@ -66,8 +66,13 @@ func runPostOverflowsInstall(cmd *cobra.Command, args []string) error {
return err
}
hub, err := cwhub.GetHub()
if err != nil {
return err
}
for _, name := range args {
t := cwhub.GetItem(cwhub.POSTOVERFLOWS, name)
t := hub.GetItem(cwhub.POSTOVERFLOWS, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.POSTOVERFLOWS, name)
Suggest(cwhub.POSTOVERFLOWS, name, nearestItem.Name, score, ignoreError)
@ -75,7 +80,7 @@ func runPostOverflowsInstall(cmd *cobra.Command, args []string) error {
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.POSTOVERFLOWS, force, downloadOnly); err != nil {
if err := hub.InstallItem(name, cwhub.POSTOVERFLOWS, force, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
@ -126,8 +131,13 @@ func runPostOverflowsRemove(cmd *cobra.Command, args []string) error {
return err
}
hub, err := cwhub.GetHub()
if err != nil {
return err
}
if all {
err := cwhub.RemoveMany(csConfig, cwhub.POSTOVERFLOWS, "", all, purge, force)
err := hub.RemoveMany(cwhub.POSTOVERFLOWS, "", all, purge, force)
if err != nil {
return err
}
@ -140,7 +150,7 @@ func runPostOverflowsRemove(cmd *cobra.Command, args []string) error {
}
for _, name := range args {
err := cwhub.RemoveMany(csConfig, cwhub.POSTOVERFLOWS, name, all, purge, force)
err := hub.RemoveMany(cwhub.POSTOVERFLOWS, name, all, purge, force)
if err != nil {
return err
}
@ -184,8 +194,13 @@ func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error {
return err
}
hub, err := cwhub.GetHub()
if err != nil {
return err
}
if all {
if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, "", force); err != nil {
if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, "", force); err != nil {
return err
}
return nil
@ -196,7 +211,7 @@ func runPostOverflowUpgrade(cmd *cobra.Command, args []string) error {
}
for _, name := range args {
if err := cwhub.UpgradeConfig(csConfig, cwhub.POSTOVERFLOWS, name, force); err != nil {
if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, name, force); err != nil {
return err
}
}

11
cmd/crowdsec-cli/require/require.go
View file

@ -64,16 +64,17 @@ func Notifications(c *csconfig.Config) error {
return nil
}
func Hub (c *csconfig.Config) error {
func Hub (c *csconfig.Config) (*cwhub.Hub, error) {
if c.Hub == nil {
return fmt.Errorf("you must configure cli before interacting with hub")
return nil, fmt.Errorf("you must configure cli before interacting with hub")
}
cwhub.SetHubBranch()
if err := cwhub.GetHubIdx(c.Hub); err != nil {
return fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err)
hub, err := cwhub.InitHub(c.Hub)
if err != nil {
return nil, fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err)
}
return nil
return hub, nil
}

29
cmd/crowdsec-cli/scenarios.go
View file

@ -25,7 +25,7 @@ cscli scenarios remove crowdsecurity/ssh-bf crowdsecurity/http-probing
Aliases: []string{"scenario"},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if err := require.Hub(csConfig); err != nil {
if _, err := require.Hub(csConfig); err != nil {
return err
}
@ -66,8 +66,13 @@ func runScenariosInstall(cmd *cobra.Command, args []string) error {
return err
}
hub, err := cwhub.GetHub()
if err != nil {
return err
}
for _, name := range args {
t := cwhub.GetItem(cwhub.SCENARIOS, name)
t := hub.GetItem(cwhub.SCENARIOS, name)
if t == nil {
nearestItem, score := GetDistance(cwhub.SCENARIOS, name)
Suggest(cwhub.SCENARIOS, name, nearestItem.Name, score, ignoreError)
@ -75,7 +80,7 @@ func runScenariosInstall(cmd *cobra.Command, args []string) error {
continue
}
if err := cwhub.InstallItem(csConfig, name, cwhub.SCENARIOS, force, downloadOnly); err != nil {
if err := hub.InstallItem(name, cwhub.SCENARIOS, force, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
@ -126,8 +131,13 @@ func runScenariosRemove(cmd *cobra.Command, args []string) error {
return err
}
hub, err := cwhub.GetHub()
if err != nil {
return err
}
if all {
err := cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, "", all, purge, force)
err := hub.RemoveMany(cwhub.SCENARIOS, "", all, purge, force)
if err != nil {
return err
}
@ -140,7 +150,7 @@ func runScenariosRemove(cmd *cobra.Command, args []string) error {
}
for _, name := range args {
err := cwhub.RemoveMany(csConfig, cwhub.SCENARIOS, name, all, purge, force)
err := hub.RemoveMany(cwhub.SCENARIOS, name, all, purge, force)
if err != nil {
return err
}
@ -184,8 +194,13 @@ func runScenariosUpgrade(cmd *cobra.Command, args []string) error {
return err
}
hub, err := cwhub.GetHub()
if err != nil {
return err
}
if all {
if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, "", force); err != nil {
if err := hub.UpgradeConfig(cwhub.SCENARIOS, "", force); err != nil {
return err
}
return nil
@ -196,7 +211,7 @@ func runScenariosUpgrade(cmd *cobra.Command, args []string) error {
}
for _, name := range args {
if err := cwhub.UpgradeConfig(csConfig, cwhub.SCENARIOS, name, force); err != nil {
if err := hub.UpgradeConfig(cwhub.SCENARIOS, name, force); err != nil {
return err
}
}

5
cmd/crowdsec-cli/simulation.go
View file

@ -145,13 +145,14 @@ func NewSimulationEnableCmd() *cobra.Command {
Example: `cscli simulation enable`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
if err := require.Hub(csConfig); err != nil {
hub, err := require.Hub(csConfig)
if err != nil {
log.Fatal(err)
}
if len(args) > 0 {
for _, scenario := range args {
var item = cwhub.GetItem(cwhub.SCENARIOS, scenario)
var item = hub.GetItem(cwhub.SCENARIOS, scenario)
if item == nil {
log.Errorf("'%s' doesn't exist or is not a scenario", scenario)
continue

13
cmd/crowdsec-cli/support.go
View file

@ -155,7 +155,7 @@ func collectAgents(dbClient *database.Client) ([]byte, error) {
return out.Bytes(), nil
}
func collectAPIStatus(login string, password string, endpoint string, prefix string) []byte {
func collectAPIStatus(login string, password string, endpoint string, prefix string, hub *cwhub.Hub) []byte {
if csConfig.API.Client == nil || csConfig.API.Client.Credentials == nil {
return []byte("No agent credentials found, are we LAPI ?")
}
@ -165,7 +165,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
if err != nil {
return []byte(fmt.Sprintf("cannot parse API URL: %s", err))
}
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
if err != nil {
return []byte(fmt.Sprintf("could not collect scenarios: %s", err))
}
@ -293,7 +293,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
skipAgent = true
}
if err := require.Hub(csConfig); err != nil {
hub, err := require.Hub(csConfig)
if err != nil {
log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
skipHub = true
infos[SUPPORT_PARSERS_PATH] = []byte(err.Error())
@ -356,7 +357,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_CAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Server.OnlineClient.Credentials.Login,
csConfig.API.Server.OnlineClient.Credentials.Password,
csConfig.API.Server.OnlineClient.Credentials.URL,
CAPIURLPrefix)
CAPIURLPrefix,
hub)
}
if !skipLAPI {
@ -364,7 +366,8 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_LAPI_STATUS_PATH] = collectAPIStatus(csConfig.API.Client.Credentials.Login,
csConfig.API.Client.Credentials.Password,
csConfig.API.Client.Credentials.URL,
LAPIURLPrefix)
LAPIURLPrefix,
hub)
infos[SUPPORT_CROWDSEC_PROFILE_PATH] = collectCrowdsecProfile()
}

4
cmd/crowdsec-cli/utils_table.go
View file

@ -16,8 +16,10 @@ func listHubItemTable(out io.Writer, title string, itemType string, itemNames []
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
hub, _ := cwhub.GetHub()
for itemName := range itemNames {
item := cwhub.GetItem(itemType, itemNames[itemName])
item := hub.GetItem(itemType, itemNames[itemName])
status, emo := item.Status()
t.AddRow(item.Name, fmt.Sprintf("%v %s", emo, status), item.LocalVersion, item.LocalPath)
}

6
cmd/crowdsec/crowdsec.go
View file

@ -23,8 +23,8 @@ import (
func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
var err error
// Populate cwhub package tools
if err = cwhub.GetHubIdx(cConfig.Hub); err != nil {
hub, err := cwhub.InitHub(cConfig.Hub)
if err != nil {
return nil, fmt.Errorf("while loading hub index: %w", err)
}
@ -34,7 +34,7 @@ func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
return nil, fmt.Errorf("while loading parsers: %w", err)
}
if err := LoadBuckets(cConfig); err != nil {
if err := LoadBuckets(cConfig, hub); err != nil {
return nil, fmt.Errorf("while loading scenarios: %w", err)
}

6
cmd/crowdsec/main.go
View file

@ -75,12 +75,12 @@ type Flags struct {
type labelsMap map[string]string
func LoadBuckets(cConfig *csconfig.Config) error {
func LoadBuckets(cConfig *csconfig.Config, hub *cwhub.Hub) error {
var (
err error
files []string
)
for _, hubScenarioItem := range cwhub.GetItemMap(cwhub.SCENARIOS) {
for _, hubScenarioItem := range hub.GetItemMap(cwhub.SCENARIOS) {
if hubScenarioItem.Installed {
files = append(files, hubScenarioItem.LocalPath)
}
@ -88,7 +88,7 @@ func LoadBuckets(cConfig *csconfig.Config) error {
buckets = leakybucket.NewBuckets()
log.Infof("Loading %d scenario files", len(files))
holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, files, &bucketsTomb, buckets, flags.OrderEvent)
holders, outputEventChan, err = leakybucket.LoadBuckets(cConfig.Crowdsec, cConfig.ConfigPaths.DataDir, files, &bucketsTomb, buckets, flags.OrderEvent)
if err != nil {
return fmt.Errorf("scenario loading failed: %v", err)

9
cmd/crowdsec/output.go
View file

@ -70,7 +70,12 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
var cache []types.RuntimeAlert
var cacheMutex sync.Mutex
scenarios, err := cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)
hub, err := cwhub.GetHub()
if err != nil {
return err
}
scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("loading list of installed hub scenarios: %w", err)
}
@ -93,7 +98,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
URL: apiURL,
PapiURL: papiURL,
VersionPrefix: "v1",
UpdateScenario: func() ([]string, error) {return cwhub.GetInstalledItemsAsString(cwhub.SCENARIOS)},
UpdateScenario: func() ([]string, error) {return hub.GetInstalledItemsAsString(cwhub.SCENARIOS)},
})
if err != nil {
return fmt.Errorf("new client api: %w", err)

2
pkg/csconfig/config.go
View file

@ -100,7 +100,7 @@ func NewDefaultConfig() *Config {
commonCfg := CommonCfg{
Daemonize: false,
LogMedia: "stdout",
LogLevel: &logLevel,
LogLevel: &logLevel,
}
prometheus := PrometheusCfg{
Enabled: true,

11
pkg/csconfig/crowdsec_service.go
View file

@ -28,10 +28,6 @@ type CrowdsecServiceCfg struct {
BucketStateDumpDir string `yaml:"state_output_dir,omitempty"` // if we need to unserialize buckets on shutdown
BucketsGCEnabled bool `yaml:"-"` // we need to garbage collect buckets when in forensic mode
HubDir string `yaml:"-"`
DataDir string `yaml:"-"`
ConfigDir string `yaml:"-"`
HubIndexFile string `yaml:"-"`
SimulationFilePath string `yaml:"-"`
ContextToSend map[string][]string `yaml:"-"`
}
@ -101,11 +97,6 @@ func (c *Config) LoadCrowdsec() error {
return fmt.Errorf("load error (simulation): %w", err)
}
c.Crowdsec.ConfigDir = c.ConfigPaths.ConfigDir
c.Crowdsec.DataDir = c.ConfigPaths.DataDir
c.Crowdsec.HubDir = c.ConfigPaths.HubDir
c.Crowdsec.HubIndexFile = c.ConfigPaths.HubIndexFile
if c.Crowdsec.ParserRoutinesCount <= 0 {
c.Crowdsec.ParserRoutinesCount = 1
}
@ -149,7 +140,7 @@ func (c *Config) LoadCrowdsec() error {
fallback := false
if c.Crowdsec.ConsoleContextPath == "" {
// fallback to default config file
c.Crowdsec.ConsoleContextPath = filepath.Join(c.Crowdsec.ConfigDir, "console", "context.yaml")
c.Crowdsec.ConsoleContextPath = filepath.Join(c.ConfigPaths.ConfigDir, "console", "context.yaml")
fallback = true
}

15
pkg/csconfig/crowdsec_service_test.go
View file

@ -54,11 +54,6 @@ func TestLoadCrowdsec(t *testing.T) {
AcquisitionDirPath: "",
ConsoleContextPath: contextFileFullPath,
AcquisitionFilePath: acquisFullPath,
ConfigDir: "./testdata",
DataDir: "./data",
HubDir: "./hub",
// XXX: need to ensure a default here
HubIndexFile: "",
BucketsRoutinesCount: 1,
ParserRoutinesCount: 1,
OutputRoutinesCount: 1,
@ -98,11 +93,6 @@ func TestLoadCrowdsec(t *testing.T) {
AcquisitionDirPath: acquisDirFullPath,
AcquisitionFilePath: acquisFullPath,
ConsoleContextPath: contextFileFullPath,
ConfigDir: "./testdata",
// XXX: need to ensure a default here
HubIndexFile: "",
DataDir: "./data",
HubDir: "./hub",
BucketsRoutinesCount: 1,
ParserRoutinesCount: 1,
OutputRoutinesCount: 1,
@ -139,11 +129,6 @@ func TestLoadCrowdsec(t *testing.T) {
Enable: ptr.Of(true),
AcquisitionDirPath: "",
AcquisitionFilePath: "",
ConfigDir: "./testdata",
// XXX: need to ensure a default here
HubIndexFile: "",
DataDir: "./data",
HubDir: "./hub",
ConsoleContextPath: contextFileFullPath,
BucketsRoutinesCount: 1,
ParserRoutinesCount: 1,

11
pkg/csconfig/cscli.go
View file

@ -11,10 +11,7 @@ type CscliCfg struct {
HubBranch string `yaml:"hub_branch"`
SimulationConfig *SimulationConfig `yaml:"-"`
DbConfig *DatabaseCfg `yaml:"-"`
HubDir string `yaml:"-"`
DataDir string `yaml:"-"`
ConfigDir string `yaml:"-"`
HubIndexFile string `yaml:"-"`
SimulationFilePath string `yaml:"-"`
PrometheusUrl string `yaml:"prometheus_uri"`
}
@ -23,10 +20,8 @@ func (c *Config) loadCSCLI() error {
if c.Cscli == nil {
c.Cscli = &CscliCfg{}
}
c.Cscli.ConfigDir = c.ConfigPaths.ConfigDir
c.Cscli.DataDir = c.ConfigPaths.DataDir
c.Cscli.HubDir = c.ConfigPaths.HubDir
c.Cscli.HubIndexFile = c.ConfigPaths.HubIndexFile
// XXX: HubBranch default should be set here and fed to HubCfg?
if c.Prometheus.ListenAddr != "" && c.Prometheus.ListenPort != 0 {
c.Cscli.PrometheusUrl = fmt.Sprintf("http://%s:%d/metrics", c.Prometheus.ListenAddr, c.Prometheus.ListenPort)

4
pkg/csconfig/cscli_test.go
View file

@ -32,10 +32,6 @@ func TestLoadCSCLI(t *testing.T) {
},
},
expected: &CscliCfg{
ConfigDir: "./testdata",
DataDir: "./data",
HubDir: "./hub",
HubIndexFile: "./hub/.index.json",
PrometheusUrl: "http://127.0.0.1:6060/metrics",
},
},

3
pkg/csconfig/hub.go
View file

@ -9,6 +9,9 @@ type HubCfg struct {
}
func (c *Config) loadHub() error {
// XXX: HubBranch too -- from cscli or chooseHubBranch() ?
c.Hub = &HubCfg{
HubIndexFile: c.ConfigPaths.HubIndexFile,
HubDir: c.ConfigPaths.HubDir,

44
pkg/cwhub/cwhub.go
View file

@ -15,7 +15,6 @@ import (
"golang.org/x/mod/semver"
)
var (
ErrMissingReference = errors.New("Reference(s) missing in collection")
@ -40,7 +39,7 @@ type Item struct {
FileName string `json:"file_name,omitempty"` // the filename, ie. apache2-logs.yaml
Description string `json:"description,omitempty" yaml:"description,omitempty"` // as seen in .index.json
Author string `json:"author,omitempty"` // as seen in .index.json
References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json
References []string `json:"references,omitempty" yaml:"references,omitempty"` // as seen in .index.json
BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"` // parent collection if any
// remote (hub) info
@ -112,9 +111,16 @@ func (i *Item) versionStatus() int {
return semver.Compare("v"+i.Version, "v"+i.LocalVersion)
}
// validPath returns true if the (relative) path is allowed for the item
// dirNmae: the directory name (ie. crowdsecurity)
// fileName: the filename (ie. apache2-logs.yaml)
func (i *Item) validPath(dirName, fileName string) bool {
return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml")
}
// GetItemMap returns the map of items for a given type
func GetItemMap(itemType string) map[string]Item {
m, ok := hubIdx.Items[itemType]
func (h *Hub) GetItemMap(itemType string) map[string]Item {
m, ok := h.Items[itemType]
if !ok {
return nil
}
@ -123,6 +129,7 @@ func GetItemMap(itemType string) map[string]Item {
}
// itemKey extracts the map key of an item (i.e. author/name) from its pathname. Follows a symlink if necessary
// XXX: only used by leakybucket manager
func itemKey(itemPath string) (string, error) {
f, err := os.Lstat(itemPath)
if err != nil {
@ -150,13 +157,13 @@ func itemKey(itemPath string) (string, error) {
}
// GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item.
func GetItemByPath(itemType string, itemPath string) (*Item, error) {
func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) {
itemKey, err := itemKey(itemPath)
if err != nil {
return nil, err
}
m := GetItemMap(itemType)
m := h.GetItemMap(itemType)
if m == nil {
return nil, fmt.Errorf("item type %s doesn't exist", itemType)
}
@ -170,19 +177,20 @@ func GetItemByPath(itemType string, itemPath string) (*Item, error) {
}
// GetItem returns the item from hub based on its type and full name (author/name)
func GetItem(itemType string, itemName string) *Item {
if m, ok := GetItemMap(itemType)[itemName]; ok {
return &m
func (h *Hub) GetItem(itemType string, itemName string) *Item {
m, ok := h.GetItemMap(itemType)[itemName]
if !ok {
return nil
}
return nil
return &m
}
// GetItemNames returns the list of item (full) names for a given type
// ie. for parsers: crowdsecurity/apache2 crowdsecurity/nginx
// The names can be used to retrieve the item with GetItem()
func GetItemNames(itemType string) []string {
m := GetItemMap(itemType)
func (h *Hub) GetItemNames(itemType string) []string {
m := h.GetItemMap(itemType)
if m == nil {
return nil
}
@ -196,10 +204,10 @@ func GetItemNames(itemType string) []string {
}
// AddItem adds an item to the hub index
func AddItem(itemType string, item Item) error {
func (h *Hub) AddItem(itemType string, item Item) error {
for _, itype := range ItemTypes {
if itype == itemType {
hubIdx.Items[itemType][item.Name] = item
h.Items[itemType][item.Name] = item
return nil
}
}
@ -208,8 +216,8 @@ func AddItem(itemType string, item Item) error {
}
// GetInstalledItems returns the list of installed items
func GetInstalledItems(itemType string) ([]Item, error) {
items, ok := hubIdx.Items[itemType]
func (h *Hub) GetInstalledItems(itemType string) ([]Item, error) {
items, ok := h.Items[itemType]
if !ok {
return nil, fmt.Errorf("no %s in hubIdx", itemType)
}
@ -226,8 +234,8 @@ func GetInstalledItems(itemType string) ([]Item, error) {
}
// GetInstalledItemsAsString returns the names of the installed items
func GetInstalledItemsAsString(itemType string) ([]string, error) {
items, err := GetInstalledItems(itemType)
func (h *Hub) GetInstalledItemsAsString(itemType string) ([]string, error) {
items, err := h.GetInstalledItems(itemType)
if err != nil {
return nil, err
}

225
pkg/cwhub/cwhub_test.go
View file

@ -29,23 +29,15 @@ import (
var responseByPath map[string]string
func TestItemStatus(t *testing.T) {
cfg := envSetup(t)
defer envTearDown(cfg)
// DownloadHubIdx()
err := UpdateHubIdx(cfg.Hub)
require.NoError(t, err, "failed to download index")
err = GetHubIdx(cfg.Hub)
require.NoError(t, err, "failed to load hub index")
hub := envSetup(t)
// get existing map
x := GetItemMap(COLLECTIONS)
x := hub.GetItemMap(COLLECTIONS)
require.NotEmpty(t, x)
// Get item : good and bad
for k := range x {
item := GetItem(COLLECTIONS, k)
item := hub.GetItem(COLLECTIONS, k)
require.NotNil(t, item)
item.Installed = true
@ -65,76 +57,101 @@ func TestItemStatus(t *testing.T) {
require.Equal(t, "disabled,local", txt)
}
DisplaySummary()
err := DisplaySummary()
require.NoError(t, err)
}
func TestGetters(t *testing.T) {
cfg := envSetup(t)
defer envTearDown(cfg)
// DownloadHubIdx()
err := UpdateHubIdx(cfg.Hub)
require.NoError(t, err, "failed to download index")
err = GetHubIdx(cfg.Hub)
require.NoError(t, err, "failed to load hub index")
hub := envSetup(t)
// get non existing map
empty := GetItemMap("ratata")
empty := hub.GetItemMap("ratata")
require.Nil(t, empty)
// get existing map
x := GetItemMap(COLLECTIONS)
x := hub.GetItemMap(COLLECTIONS)
require.NotEmpty(t, x)
// Get item : good and bad
for k := range x {
empty := GetItem(COLLECTIONS, k+"nope")
empty := hub.GetItem(COLLECTIONS, k+"nope")
require.Nil(t, empty)
item := GetItem(COLLECTIONS, k)
item := hub.GetItem(COLLECTIONS, k)
require.NotNil(t, item)
// Add item and get it
item.Name += "nope"
err := AddItem(COLLECTIONS, *item)
err := hub.AddItem(COLLECTIONS, *item)
require.NoError(t, err)
newitem := GetItem(COLLECTIONS, item.Name)
newitem := hub.GetItem(COLLECTIONS, item.Name)
require.NotNil(t, newitem)
err = AddItem("ratata", *item)
err = hub.AddItem("ratata", *item)
cstest.RequireErrorContains(t, err, "ItemType ratata is unknown")
}
}
func TestIndexDownload(t *testing.T) {
cfg := envSetup(t)
defer envTearDown(cfg)
hub := envSetup(t)
// DownloadHubIdx()
err := UpdateHubIdx(cfg.Hub)
_, err := InitHubUpdate(hub.cfg)
require.NoError(t, err, "failed to download index")
err = GetHubIdx(cfg.Hub)
_, err = GetHub()
require.NoError(t, err, "failed to load hub index")
}
func getTestCfg() *csconfig.Config {
cfg := &csconfig.Config{Hub: &csconfig.HubCfg{}}
cfg.Hub.InstallDir, _ = filepath.Abs("./install")
cfg.Hub.HubDir, _ = filepath.Abs("./hubdir")
cfg.Hub.HubIndexFile = filepath.Clean("./hubdir/.index.json")
// testHub initializes a temporary hub with an empty json file, optionally updating it
func testHub(t *testing.T, update bool) *Hub {
tmpDir, err := os.MkdirTemp("", "testhub")
require.NoError(t, err)
return cfg
hubCfg := &csconfig.HubCfg{
HubDir: filepath.Join(tmpDir, "crowdsec", "hub"),
HubIndexFile: filepath.Join(tmpDir, "crowdsec", "hub", ".index.json"),
InstallDir: filepath.Join(tmpDir, "crowdsec"),
InstallDataDir: filepath.Join(tmpDir, "installed-data"),
}
err = os.MkdirAll(hubCfg.HubDir, 0700)
require.NoError(t, err)
err = os.MkdirAll(hubCfg.InstallDir, 0700)
require.NoError(t, err)
err = os.MkdirAll(hubCfg.InstallDataDir, 0700)
require.NoError(t, err)
index, err := os.Create(hubCfg.HubIndexFile)
require.NoError(t, err)
_, err = index.WriteString(`{}`)
require.NoError(t, err)
index.Close()
t.Cleanup(func() {
os.RemoveAll(tmpDir)
})
constructor := InitHub
if update {
constructor = InitHubUpdate
}
hub, err := constructor(hubCfg)
require.NoError(t, err)
return hub
}
func envSetup(t *testing.T) *csconfig.Config {
func envSetup(t *testing.T) *Hub {
resetResponseByPath()
log.SetLevel(log.DebugLevel)
cfg := getTestCfg()
defaultTransport := http.DefaultClient.Transport
t.Cleanup(func() {
@ -144,14 +161,7 @@ func envSetup(t *testing.T) *csconfig.Config {
// Mock the http client
http.DefaultClient.Transport = newMockTransport()
err := os.MkdirAll(cfg.Hub.InstallDir, 0700)
require.NoError(t, err)
err = os.MkdirAll(cfg.Hub.HubDir, 0700)
require.NoError(t, err)
err = UpdateHubIdx(cfg.Hub)
require.NoError(t, err)
hub := testHub(t, true)
// if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil {
// log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err)
@ -159,42 +169,33 @@ func envSetup(t *testing.T) *csconfig.Config {
// if err := os.MkdirAll(cfg.Hub.InstallDir, 0700); err != nil {
// log.Fatalf("failed to mkdir %s : %s", cfg.Hub.InstallDir, err)
// }
return cfg
return hub
}
func envTearDown(cfg *csconfig.Config) {
if err := os.RemoveAll(cfg.Hub.InstallDir); err != nil {
log.Fatalf("failed to remove %s : %s", cfg.Hub.InstallDir, err)
}
if err := os.RemoveAll(cfg.Hub.HubDir); err != nil {
log.Fatalf("failed to remove %s : %s", cfg.Hub.HubDir, err)
}
}
func testInstallItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
func testInstallItem(hub *Hub, t *testing.T, item Item) {
// Install the parser
err := DownloadLatest(cfg, &item, false, false)
err := hub.DownloadLatest(&item, false, false)
require.NoError(t, err, "failed to download %s", item.Name)
_, err = LocalSync(cfg)
_, err = hub.LocalSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name)
assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
err = EnableItem(cfg, &item)
err = hub.EnableItem(&item)
require.NoError(t, err, "failed to enable %s", item.Name)
_, err = LocalSync(cfg)
_, err = hub.LocalSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
}
func testTaintItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
func testTaintItem(hub *Hub, t *testing.T, item Item) {
assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted", item.Name)
f, err := os.OpenFile(item.LocalPath, os.O_APPEND|os.O_WRONLY, 0600)
require.NoError(t, err, "failed to open %s (%s)", item.LocalPath, item.Name)
@ -205,54 +206,54 @@ func testTaintItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
require.NoError(t, err, "failed to write to %s (%s)", item.LocalPath, item.Name)
// Local sync and check status
_, err = LocalSync(cfg)
_, err = hub.LocalSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name)
assert.True(t, hub.Items[item.Type][item.Name].Tainted, "%s should be tainted", item.Name)
}
func testUpdateItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
assert.False(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
func testUpdateItem(hub *Hub, t *testing.T, item Item) {
assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
// Update it + check status
err := DownloadLatest(cfg, &item, true, true)
err := hub.DownloadLatest(&item, true, true)
require.NoError(t, err, "failed to update %s", item.Name)
// Local sync and check status
_, err = LocalSync(cfg)
_, err = hub.LocalSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hubIdx.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
assert.True(t, hub.Items[item.Type][item.Name].UpToDate, "%s should be up-to-date", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
}
func testDisableItem(cfg *csconfig.HubCfg, t *testing.T, item Item) {
assert.True(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
func testDisableItem(hub *Hub, t *testing.T, item Item) {
assert.True(t, hub.Items[item.Type][item.Name].Installed, "%s should be installed", item.Name)
// Remove
err := DisableItem(cfg, &item, false, false)
err := hub.DisableItem(&item, false, false)
require.NoError(t, err, "failed to disable %s", item.Name)
// Local sync and check status
warns, err := LocalSync(cfg)
warns, err := hub.LocalSync()
require.NoError(t, err, "failed to run localSync")
require.Empty(t, warns, "unexpected warnings : %+v", warns)
assert.False(t, hubIdx.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
assert.True(t, hubIdx.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].Tainted, "%s should not be tainted anymore", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
assert.True(t, hub.Items[item.Type][item.Name].Downloaded, "%s should still be downloaded", item.Name)
// Purge
err = DisableItem(cfg, &item, true, false)
err = hub.DisableItem(&item, true, false)
require.NoError(t, err, "failed to purge %s", item.Name)
// Local sync and check status
warns, err = LocalSync(cfg)
warns, err = hub.LocalSync()
require.NoError(t, err, "failed to run localSync")
require.Empty(t, warns, "unexpected warnings : %+v", warns)
assert.False(t, hubIdx.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
assert.False(t, hubIdx.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].Installed, "%s should not be installed anymore", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].Downloaded, "%s should not be downloaded", item.Name)
}
func TestInstallParser(t *testing.T) {
@ -265,20 +266,18 @@ func TestInstallParser(t *testing.T) {
- check its status
- remove it
*/
cfg := envSetup(t)
defer envTearDown(cfg)
hub := envSetup(t)
getHubIdxOrFail(t)
// map iteration is random by itself
for _, it := range hubIdx.Items[PARSERS] {
testInstallItem(cfg.Hub, t, it)
it = hubIdx.Items[PARSERS][it.Name]
testTaintItem(cfg.Hub, t, it)
it = hubIdx.Items[PARSERS][it.Name]
testUpdateItem(cfg.Hub, t, it)
it = hubIdx.Items[PARSERS][it.Name]
testDisableItem(cfg.Hub, t, it)
it = hubIdx.Items[PARSERS][it.Name]
for _, it := range hub.Items[PARSERS] {
testInstallItem(hub, t, it)
it = hub.Items[PARSERS][it.Name]
testTaintItem(hub, t, it)
it = hub.Items[PARSERS][it.Name]
testUpdateItem(hub, t, it)
it = hub.Items[PARSERS][it.Name]
testDisableItem(hub, t, it)
it = hub.Items[PARSERS][it.Name]
break
}
@ -294,19 +293,17 @@ func TestInstallCollection(t *testing.T) {
- check its status
- remove it
*/
cfg := envSetup(t)
defer envTearDown(cfg)
hub := envSetup(t)
getHubIdxOrFail(t)
// map iteration is random by itself
for _, it := range hubIdx.Items[COLLECTIONS] {
testInstallItem(cfg.Hub, t, it)
it = hubIdx.Items[COLLECTIONS][it.Name]
testTaintItem(cfg.Hub, t, it)
it = hubIdx.Items[COLLECTIONS][it.Name]
testUpdateItem(cfg.Hub, t, it)
it = hubIdx.Items[COLLECTIONS][it.Name]
testDisableItem(cfg.Hub, t, it)
for _, it := range hub.Items[COLLECTIONS] {
testInstallItem(hub, t, it)
it = hub.Items[COLLECTIONS][it.Name]
testTaintItem(hub, t, it)
it = hub.Items[COLLECTIONS][it.Name]
testUpdateItem(hub, t, it)
it = hub.Items[COLLECTIONS][it.Name]
testDisableItem(hub, t, it)
break
}
}

72
pkg/cwhub/download.go
View file

@ -19,31 +19,39 @@ import (
var ErrIndexNotFound = fmt.Errorf("index not found")
// UpdateHubIdx downloads the latest version of the index and updates the one in memory
func UpdateHubIdx(hub *csconfig.HubCfg) error {
bidx, err := DownloadHubIdx(hub)
// InitHubUpdate is like InitHub but downloads and updates the index instead of reading from the disk
// It is used to inizialize the hub when there is no index file yet
func InitHubUpdate(cfg *csconfig.HubCfg) (*Hub, error) {
if cfg == nil {
return nil, fmt.Errorf("no configuration found for hub")
}
bidx, err := DownloadHubIdx(cfg.HubIndexFile)
if err != nil {
return fmt.Errorf("failed to download index: %w", err)
return nil, fmt.Errorf("failed to download index: %w", err)
}
ret, err := ParseIndex(bidx)
if err != nil {
if !errors.Is(err, ErrMissingReference) {
return fmt.Errorf("failed to read index: %w", err)
return nil, fmt.Errorf("failed to read index: %w", err)
}
}
hubIdx = HubIndex{Items: ret}
if _, err := LocalSync(hub); err != nil {
return fmt.Errorf("failed to sync: %w", err)
theHub = &Hub{
Items: ret,
cfg: cfg,
}
return nil
if _, err := theHub.LocalSync(); err != nil {
return nil, fmt.Errorf("failed to sync: %w", err)
}
return theHub, nil
}
// DownloadHubIdx downloads the latest version of the index and returns the content
func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) {
func DownloadHubIdx(indexPath string) ([]byte, error) {
log.Debugf("fetching index from branch %s (%s)", HubBranch, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile))
req, err := http.NewRequest(http.MethodGet, fmt.Sprintf(RawFileURLTemplate, HubBranch, HubIndexFile), nil)
@ -70,7 +78,7 @@ func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) {
return nil, fmt.Errorf("failed to read request answer for hub index: %w", err)
}
oldContent, err := os.ReadFile(hub.HubIndexFile)
oldContent, err := os.ReadFile(indexPath)
if err != nil {
if !os.IsNotExist(err) {
log.Warningf("failed to read hub index: %s", err)
@ -80,7 +88,7 @@ func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) {
// write it anyway, can't hurt
}
file, err := os.OpenFile(hub.HubIndexFile, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
file, err := os.OpenFile(indexPath, os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return nil, fmt.Errorf("while opening hub index file: %w", err)
@ -92,13 +100,13 @@ func DownloadHubIdx(hub *csconfig.HubCfg) ([]byte, error) {
return nil, fmt.Errorf("while writing hub index file: %w", err)
}
log.Infof("Wrote new %d bytes index to %s", wsize, hub.HubIndexFile)
log.Infof("Wrote new %d bytes index to %s", wsize, indexPath)
return body, nil
}
// DownloadLatest will download the latest version of Item to the tdir directory
func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOnly bool) error {
func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) error {
var err error
log.Debugf("Downloading %s %s", target.Type, target.Name)
@ -109,7 +117,7 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn
return nil
}
return DownloadItem(hub, target, overwrite)
return h.DownloadItem(target, overwrite)
}
// collection
@ -117,7 +125,7 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn
for idx, ptr := range tmp {
ptrtype := ItemTypes[idx]
for _, p := range ptr {
val, ok := hubIdx.Items[ptrtype][p]
val, ok := h.Items[ptrtype][p]
if !ok {
return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
}
@ -132,7 +140,7 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn
if ptrtype == COLLECTIONS {
log.Tracef("collection, recurse")
err = DownloadLatest(hub, &val, overwrite, updateOnly)
err = h.DownloadLatest(&val, overwrite, updateOnly)
if err != nil {
return fmt.Errorf("while downloading %s: %w", val.Name, err)
}
@ -140,7 +148,7 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn
downloaded := val.Downloaded
err = DownloadItem(hub, &val, overwrite)
err = h.DownloadItem(&val, overwrite)
if err != nil {
return fmt.Errorf("while downloading %s: %w", val.Name, err)
}
@ -148,16 +156,16 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn
// We need to enable an item when it has been added to a collection since latest release of the collection.
// We check if val.Downloaded is false because maybe the item has been disabled by the user.
if !val.Installed && !downloaded {
if err = EnableItem(hub, &val); err != nil {
if err = h.EnableItem(&val); err != nil {
return fmt.Errorf("enabling '%s': %w", val.Name, err)
}
}
hubIdx.Items[ptrtype][p] = val
h.Items[ptrtype][p] = val
}
}
err = DownloadItem(hub, target, overwrite)
err = h.DownloadItem(target, overwrite)
if err != nil {
return fmt.Errorf("failed to download item: %w", err)
}
@ -165,8 +173,8 @@ func DownloadLatest(hub *csconfig.HubCfg, target *Item, overwrite bool, updateOn
return nil
}
func DownloadItem(hub *csconfig.HubCfg, target *Item, overwrite bool) error {
tdir := hub.HubDir
func (h *Hub) DownloadItem(target *Item, overwrite bool) error {
tdir := h.cfg.HubDir
// if user didn't --force, don't overwrite local, tainted, up-to-date files
if !overwrite {
@ -202,12 +210,12 @@ func DownloadItem(hub *csconfig.HubCfg, target *Item, overwrite bool) error {
return fmt.Errorf("while reading %s: %w", req.URL.String(), err)
}
h := sha256.New()
if _, err = h.Write(body); err != nil {
hash := sha256.New()
if _, err = hash.Write(body); err != nil {
return fmt.Errorf("while hashing %s: %w", target.Name, err)
}
meow := fmt.Sprintf("%x", h.Sum(nil))
meow := fmt.Sprintf("%x", hash.Sum(nil))
if meow != target.Versions[target.Version].Digest {
log.Errorf("Downloaded version doesn't match index, please 'hub update'")
log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest)
@ -263,18 +271,18 @@ func DownloadItem(hub *csconfig.HubCfg, target *Item, overwrite bool) error {
target.Tainted = false
target.UpToDate = true
if err = downloadData(hub.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
if err = downloadData(h.cfg.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
return fmt.Errorf("while downloading data for %s: %w", target.FileName, err)
}
hubIdx.Items[target.Type][target.Name] = *target
h.Items[target.Type][target.Name] = *target
return nil
}
// DownloadDataIfNeeded downloads the data files for an item
func DownloadDataIfNeeded(hub *csconfig.HubCfg, target Item, force bool) error {
itemFilePath := fmt.Sprintf("%s/%s/%s/%s", hub.InstallDir, target.Type, target.Stage, target.FileName)
func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error {
itemFilePath := fmt.Sprintf("%s/%s/%s/%s", h.cfg.InstallDir, target.Type, target.Stage, target.FileName)
itemFile, err := os.Open(itemFilePath)
if err != nil {
@ -283,7 +291,7 @@ func DownloadDataIfNeeded(hub *csconfig.HubCfg, target Item, force bool) error {
defer itemFile.Close()
if err = downloadData(hub.InstallDataDir, force, itemFile); err != nil {
if err = downloadData(h.cfg.InstallDataDir, force, itemFile); err != nil {
return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
}

18
pkg/cwhub/download_test.go
View file

@ -2,12 +2,11 @@ package cwhub
import (
"fmt"
"os"
"strings"
"testing"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
func TestDownloadHubIdx(t *testing.T) {
@ -15,9 +14,18 @@ func TestDownloadHubIdx(t *testing.T) {
// bad url template
fmt.Println("Test 'bad URL'")
tmpIndex, err := os.CreateTemp("", "index.json")
if err != nil {
t.Fatalf("failed to create temp file : %s", err)
}
t.Cleanup(func() {
os.Remove(tmpIndex.Name())
})
RawFileURLTemplate = "x"
ret, err := DownloadHubIdx(&csconfig.HubCfg{})
ret, err := DownloadHubIdx(tmpIndex.Name())
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed to build request for hub index: parse ") {
log.Errorf("unexpected error %s", err)
}
@ -29,7 +37,7 @@ func TestDownloadHubIdx(t *testing.T) {
RawFileURLTemplate = "https://baddomain/%s/%s"
ret, err = DownloadHubIdx(&csconfig.HubCfg{})
ret, err = DownloadHubIdx(tmpIndex.Name())
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "failed http request for hub index: Get") {
log.Errorf("unexpected error %s", err)
}
@ -41,7 +49,7 @@ func TestDownloadHubIdx(t *testing.T) {
RawFileURLTemplate = back
ret, err = DownloadHubIdx(&csconfig.HubCfg{HubIndexFile: "/does/not/exist/index.json"})
ret, err = DownloadHubIdx("/does/not/exist/index.json")
if err == nil || !strings.HasPrefix(fmt.Sprintf("%s", err), "while opening hub index file: open /does/not/exist/index.json:") {
log.Errorf("unexpected error %s", err)
}

39
pkg/cwhub/helpers.go
View file

@ -8,7 +8,6 @@ import (
log "github.com/sirupsen/logrus"
"golang.org/x/mod/semver"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
)
@ -62,8 +61,8 @@ func SetHubBranch() {
}
// InstallItem installs an item from the hub
func InstallItem(csConfig *csconfig.Config, name string, itemType string, force bool, downloadOnly bool) error {
item := GetItem(itemType, name)
func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool) error {
item := h.GetItem(itemType, name)
if item == nil {
return fmt.Errorf("unable to retrieve item: %s", name)
}
@ -76,26 +75,26 @@ func InstallItem(csConfig *csconfig.Config, name string, itemType string, force
}
}
err := DownloadLatest(csConfig.Hub, item, force, true)
err := h.DownloadLatest(item, force, true)
if err != nil {
return fmt.Errorf("while downloading %s: %w", item.Name, err)
}
if err = AddItem(itemType, *item); err != nil {
if err = h.AddItem(itemType, *item); err != nil {
return fmt.Errorf("while adding %s: %w", item.Name, err)
}
if downloadOnly {
log.Infof("Downloaded %s to %s", item.Name, filepath.Join(csConfig.Hub.HubDir, item.RemotePath))
log.Infof("Downloaded %s to %s", item.Name, filepath.Join(h.cfg.HubDir, item.RemotePath))
return nil
}
err = EnableItem(csConfig.Hub, item)
err = h.EnableItem(item)
if err != nil {
return fmt.Errorf("while enabling %s: %w", item.Name, err)
}
if err := AddItem(itemType, *item); err != nil {
if err := h.AddItem(itemType, *item); err != nil {
return fmt.Errorf("while adding %s: %w", item.Name, err)
}
@ -105,20 +104,20 @@ func InstallItem(csConfig *csconfig.Config, name string, itemType string, force
}
// RemoveItem removes one - or all - the items from the hub
func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all bool, purge bool, forceAction bool) error {
func (h *Hub) RemoveMany(itemType string, name string, all bool, purge bool, forceAction bool) error {
if name != "" {
item := GetItem(itemType, name)
item := h.GetItem(itemType, name)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", name, itemType)
}
err := DisableItem(csConfig.Hub, item, purge, forceAction)
err := h.DisableItem(item, purge, forceAction)
if err != nil {
return fmt.Errorf("unable to disable %s: %w", item.Name, err)
}
if err = AddItem(itemType, *item); err != nil {
if err = h.AddItem(itemType, *item); err != nil {
return fmt.Errorf("unable to add %s: %w", item.Name, err)
}
@ -132,17 +131,17 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo
disabled := 0
// remove all
for _, v := range GetItemMap(itemType) {
for _, v := range h.GetItemMap(itemType) {
if !v.Installed {
continue
}
err := DisableItem(csConfig.Hub, &v, purge, forceAction)
err := h.DisableItem(&v, purge, forceAction)
if err != nil {
return fmt.Errorf("unable to disable %s: %w", v.Name, err)
}
if err := AddItem(itemType, v); err != nil {
if err := h.AddItem(itemType, v); err != nil {
return fmt.Errorf("unable to add %s: %w", v.Name, err)
}
disabled++
@ -154,11 +153,11 @@ func RemoveMany(csConfig *csconfig.Config, itemType string, name string, all boo
}
// UpgradeConfig upgrades an item from the hub
func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, force bool) error {
func (h *Hub) UpgradeConfig(itemType string, name string, force bool) error {
updated := 0
found := false
for _, v := range GetItemMap(itemType) {
for _, v := range h.GetItemMap(itemType) {
if name != "" && name != v.Name {
continue
}
@ -178,7 +177,7 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc
if v.UpToDate {
log.Infof("%s: up-to-date", v.Name)
if err := DownloadDataIfNeeded(csConfig.Hub, v, force); err != nil {
if err := h.DownloadDataIfNeeded(v, force); err != nil {
return fmt.Errorf("%s: download failed: %w", v.Name, err)
}
@ -187,7 +186,7 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc
}
}
if err := DownloadLatest(csConfig.Hub, &v, force, true); err != nil {
if err := h.DownloadLatest(&v, force, true); err != nil {
return fmt.Errorf("%s: download failed: %w", v.Name, err)
}
@ -205,7 +204,7 @@ func UpgradeConfig(csConfig *csconfig.Config, itemType string, name string, forc
updated++
}
if err := AddItem(itemType, v); err != nil {
if err := h.AddItem(itemType, v); err != nil {
return fmt.Errorf("unable to add %s: %w", v.Name, err)
}
}

166
pkg/cwhub/helpers_test.go
View file

@ -4,165 +4,165 @@ import (
"testing"
"github.com/stretchr/testify/require"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection
// We expect the new scenario to be installed
func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
cfg := envSetup(t)
defer envTearDown(cfg)
hub := envSetup(t)
// fresh install of collection
getHubIdxOrFail(t)
hub = getHubOrFail(t, hub.cfg)
require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false))
require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
// This is the scenario that gets added in next version of collection
require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
pushUpdateToCollectionInHub()
if err := UpdateHubIdx(cfg.Hub); err != nil {
t.Fatalf("failed to download index : %s", err)
}
hub, err := InitHubUpdate(hub.cfg)
require.NoError(t, err, "failed to download index: %s", err)
getHubIdxOrFail(t)
hub = getHubOrFail(t, hub.cfg)
require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
err := UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false)
require.NoError(t, err)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
}
// Install a collection, disable a scenario.
// Upgrade should install should not enable/download the disabled scenario.
func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
cfg := envSetup(t)
defer envTearDown(cfg)
hub := envSetup(t)
// fresh install of collection
getHubIdxOrFail(t)
hub = getHubOrFail(t, hub.cfg)
require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false))
require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
err := RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
err := hub.RemoveMany(SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
require.NoError(t, err)
getHubIdxOrFail(t)
hub = getHubOrFail(t, hub.cfg)
// scenario referenced by collection was deleted hence, collection should be tainted
require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
if err = UpdateHubIdx(cfg.Hub); err != nil {
t.Fatalf("failed to download index : %s", err)
}
hub, err = InitHubUpdate(hub.cfg)
require.NoError(t, err, "failed to download index: %s", err)
err = UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false)
require.NoError(t, err)
getHubIdxOrFail(t)
require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
hub = getHubOrFail(t, hub.cfg)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
}
func getHubIdxOrFail(t *testing.T) {
if err := GetHubIdx(getTestCfg().Hub); err != nil {
t.Fatalf("failed to load hub index")
}
// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test
func getHubOrFail(t *testing.T, hubCfg *csconfig.HubCfg) *Hub {
hub, err := InitHub(hubCfg)
require.NoError(t, err, "failed to load hub index")
return hub
}
// Install a collection. Disable a referenced scenario. Publish new version of collection with new scenario
// Upgrade should not enable/download the disabled scenario.
// Upgrade should install and enable the newly added scenario.
func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) {
cfg := envSetup(t)
defer envTearDown(cfg)
hub := envSetup(t)
// fresh install of collection
getHubIdxOrFail(t)
hub = getHubOrFail(t, hub.cfg)
require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.NoError(t, InstallItem(cfg, "crowdsecurity/test_collection", COLLECTIONS, false, false))
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false))
require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
err := RemoveMany(cfg, SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
err := hub.RemoveMany(SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
require.NoError(t, err)
getHubIdxOrFail(t)
hub = getHubOrFail(t, hub.cfg)
// scenario referenced by collection was deleted hence, collection should be tainted
require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails
require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hubIdx.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
// we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario
// we just removed. Nor should it install the newly added scenario
pushUpdateToCollectionInHub()
if err = UpdateHubIdx(cfg.Hub); err != nil {
t.Fatalf("failed to download index : %s", err)
}
hub, err = InitHubUpdate(hub.cfg)
require.NoError(t, err, "failed to download index: %s", err)
require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
getHubIdxOrFail(t)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
hub = getHubOrFail(t, hub.cfg)
err = UpgradeConfig(cfg, COLLECTIONS, "crowdsecurity/test_collection", false)
err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false)
require.NoError(t, err)
getHubIdxOrFail(t)
require.False(t, hubIdx.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.True(t, hubIdx.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
hub = getHubOrFail(t, hub.cfg)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
}
func assertCollectionDepsInstalled(t *testing.T, collection string) {
t.Helper()
c := hubIdx.Items[COLLECTIONS][collection]
require.NoError(t, CollecDepsCheck(&c))
hub, err := GetHub()
require.NoError(t, err)
c := hub.Items[COLLECTIONS][collection]
require.NoError(t, hub.CollectDepsCheck(&c))
}
func pushUpdateToCollectionInHub() {

40
pkg/cwhub/hubindex.go → pkg/cwhub/hub.go
View file

@ -6,8 +6,9 @@ import (
"strings"
log "github.com/sirupsen/logrus"
)
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
const (
HubIndexFile = ".index.json"
@ -20,25 +21,34 @@ const (
)
var (
// XXX: The order is important, as it is used to construct the
// index tree in memory --> collections must be last
// XXX: The order is important, as it is used to range over sub-items in collections
ItemTypes = []string{PARSERS, POSTOVERFLOWS, SCENARIOS, COLLECTIONS}
hubIdx = HubIndex{}
)
type HubItems map[string]map[string]Item
// HubIndex represents the runtime status of the hub (parsed items, etc.)
// XXX: this could be renamed "Hub" tout court once the confusion with HubCfg is cleared
type HubIndex struct {
Items HubItems
// Hub represents the runtime status of the hub (parsed items, etc.)
type Hub struct {
Items HubItems
cfg *csconfig.HubCfg
skippedLocal int
skippedTainted int
}
var theHub *Hub
// GetHub returns the hub singleton
// it returns an error if it's not initialized to avoid nil dereference
func GetHub() (*Hub, error) {
if theHub == nil {
return nil, fmt.Errorf("hub not initialized")
}
return theHub, nil
}
// displaySummary prints a total count of the hub items
func (h HubIndex) displaySummary() {
func (h Hub) displaySummary() {
msg := "Loaded: "
for itemType := range h.Items {
msg += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType)
@ -52,8 +62,14 @@ func (h HubIndex) displaySummary() {
// DisplaySummary prints a total count of the hub items.
// It is a wrapper around HubIndex.displaySummary() to avoid exporting the hub singleton
func DisplaySummary() {
hubIdx.displaySummary()
// XXX: to be removed later
func DisplaySummary() error {
hub, err := GetHub()
if err != nil {
return err
}
hub.displaySummary()
return nil
}
// ParseIndex takes the content of a .index.json file and returns the map of associated parsers/scenarios/collections

36
pkg/cwhub/install.go
View file

@ -6,12 +6,10 @@ import (
"path/filepath"
log "github.com/sirupsen/logrus"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
func purgeItem(hub *csconfig.HubCfg, target Item) (Item, error) {
itempath := hub.HubDir + "/" + target.RemotePath
func (h *Hub) purgeItem(target Item) (Item, error) {
itempath := h.cfg.HubDir + "/" + target.RemotePath
// disable hub file
if err := os.Remove(itempath); err != nil {
@ -20,19 +18,19 @@ func purgeItem(hub *csconfig.HubCfg, target Item) (Item, error) {
target.Downloaded = false
log.Infof("Removed source file [%s]: %s", target.Name, itempath)
hubIdx.Items[target.Type][target.Name] = target
h.Items[target.Type][target.Name] = target
return target, nil
}
// DisableItem to disable an item managed by the hub, removes the symlink if purge is true
func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) error {
func (h *Hub) DisableItem(target *Item, purge bool, force bool) error {
var err error
// already disabled, noop unless purge
if !target.Installed {
if purge {
*target, err = purgeItem(hub, *target)
*target, err = h.purgeItem(*target)
if err != nil {
return err
}
@ -54,7 +52,7 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err
for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} {
ptrtype := ItemTypes[idx]
for _, p := range ptr {
if val, ok := hubIdx.Items[ptrtype][p]; ok {
if val, ok := h.Items[ptrtype][p]; ok {
// check if the item doesn't belong to another collection before removing it
toRemove := true
@ -66,7 +64,7 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err
}
if toRemove {
err = DisableItem(hub, &val, purge, force)
err = h.DisableItem(&val, purge, force)
if err != nil {
return fmt.Errorf("while disabling %s: %w", p, err)
}
@ -80,7 +78,7 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err
}
}
syml, err := filepath.Abs(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName)
syml, err := filepath.Abs(h.cfg.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName)
if err != nil {
return err
}
@ -103,7 +101,7 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err
return fmt.Errorf("while reading symlink: %w", err)
}
absPath, err := filepath.Abs(hub.HubDir + "/" + target.RemotePath)
absPath, err := filepath.Abs(h.cfg.HubDir + "/" + target.RemotePath)
if err != nil {
return fmt.Errorf("while abs path: %w", err)
}
@ -124,23 +122,23 @@ func DisableItem(hub *csconfig.HubCfg, target *Item, purge bool, force bool) err
target.Installed = false
if purge {
*target, err = purgeItem(hub, *target)
*target, err = h.purgeItem(*target)
if err != nil {
return err
}
}
hubIdx.Items[target.Type][target.Name] = *target
h.Items[target.Type][target.Name] = *target
return nil
}
// creates symlink between actual config file at hub.HubDir and hub.ConfigDir
// Handles collections recursively
func EnableItem(hub *csconfig.HubCfg, target *Item) error {
func (h *Hub) EnableItem(target *Item) error {
var err error
parentDir := filepath.Clean(hub.InstallDir + "/" + target.Type + "/" + target.Stage + "/")
parentDir := filepath.Clean(h.cfg.InstallDir + "/" + target.Type + "/" + target.Stage + "/")
// create directories if needed
if target.Installed {
@ -172,12 +170,12 @@ func EnableItem(hub *csconfig.HubCfg, target *Item) error {
for idx, ptr := range [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections} {
ptrtype := ItemTypes[idx]
for _, p := range ptr {
val, ok := hubIdx.Items[ptrtype][p]
val, ok := h.Items[ptrtype][p]
if !ok {
return fmt.Errorf("required %s %s of %s doesn't exist, abort", ptrtype, p, target.Name)
}
err = EnableItem(hub, &val)
err = h.EnableItem(&val)
if err != nil {
return fmt.Errorf("while installing %s: %w", p, err)
}
@ -192,7 +190,7 @@ func EnableItem(hub *csconfig.HubCfg, target *Item) error {
}
// hub.ConfigDir + target.RemotePath
srcPath, err := filepath.Abs(hub.HubDir + "/" + target.RemotePath)
srcPath, err := filepath.Abs(h.cfg.HubDir + "/" + target.RemotePath)
if err != nil {
return fmt.Errorf("while getting source path: %w", err)
}
@ -208,7 +206,7 @@ func EnableItem(hub *csconfig.HubCfg, target *Item) error {
log.Infof("Enabled %s : %s", target.Type, target.Name)
target.Installed = true
hubIdx.Items[target.Type][target.Name] = *target
h.Items[target.Type][target.Name] = *target
return nil
}

118
pkg/cwhub/loader.go
View file

@ -19,10 +19,6 @@ func isYAMLFileName(path string) bool {
return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
}
func validItemFileName(vname string, fauthor string, fname string) bool {
return (fauthor+"/"+fname == vname+".yaml") || (fauthor+"/"+fname == vname+".yml")
}
func handleSymlink(path string) (string, error) {
hubpath, err := os.Readlink(path)
if err != nil {
@ -60,19 +56,6 @@ func getSHA256(filepath string) (string, error) {
return fmt.Sprintf("%x", h.Sum(nil)), nil
}
type Walker struct {
// the walk/parserVisit function can't receive extra args
hubdir string
installdir string
}
func NewWalker(hub *csconfig.HubCfg) Walker {
return Walker{
hubdir: hub.HubDir,
installdir: hub.InstallDir,
}
}
type itemFileInfo struct {
fname string
stage string
@ -80,16 +63,19 @@ type itemFileInfo struct {
fauthor string
}
func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) {
func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) {
ret := itemFileInfo{}
inhub := false
hubDir := h.cfg.HubDir
installDir := h.cfg.InstallDir
subs := strings.Split(path, string(os.PathSeparator))
log.Tracef("path:%s, hubdir:%s, installdir:%s", path, w.hubdir, w.installdir)
log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubDir, installDir)
log.Tracef("subs:%v", subs)
// we're in hub (~/.hub/hub/)
if strings.HasPrefix(path, w.hubdir) {
if strings.HasPrefix(path, hubDir) {
log.Tracef("in hub dir")
inhub = true
@ -104,7 +90,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) {
ret.fauthor = subs[len(subs)-2]
ret.stage = subs[len(subs)-3]
ret.ftype = subs[len(subs)-4]
} else if strings.HasPrefix(path, w.installdir) { // we're in install /etc/crowdsec/<type>/...
} else if strings.HasPrefix(path, installDir) { // we're in install /etc/crowdsec/<type>/...
log.Tracef("in install dir")
if len(subs) < 3 {
return itemFileInfo{}, false, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
@ -118,7 +104,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) {
ret.ftype = subs[len(subs)-3]
ret.fauthor = ""
} else {
return itemFileInfo{}, false, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, w.hubdir, w.installdir)
return itemFileInfo{}, false, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubDir, installDir)
}
log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype)
@ -140,7 +126,7 @@ func (w Walker) getItemInfo(path string) (itemFileInfo, bool, error) {
return ret, inhub, nil
}
func (w Walker) itemVisit(path string, f os.DirEntry, err error) error {
func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
var (
local bool
hubpath string
@ -166,7 +152,7 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error {
return nil
}
info, inhub, err := w.getItemInfo(path)
info, inhub, err := h.getItemInfo(path)
if err != nil {
return err
}
@ -197,12 +183,12 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error {
// if it's not a symlink and not in hub, it's a local file, don't bother
if local && !inhub {
log.Tracef("%s is a local file, skip", path)
hubIdx.skippedLocal++
h.skippedLocal++
// log.Infof("local scenario, skip.")
_, fileName := filepath.Split(path)
hubIdx.Items[info.ftype][info.fname] = Item{
h.Items[info.ftype][info.fname] = Item{
Name: info.fname,
Stage: info.stage,
Installed: true,
@ -221,7 +207,7 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error {
match := false
for name, item := range hubIdx.Items[info.ftype] {
for name, item := range h.Items[info.ftype] {
log.Tracef("check [%s] vs [%s] : %s", info.fname, item.RemotePath, info.ftype+"/"+info.stage+"/"+info.fname+".yaml")
if info.fname != item.FileName {
@ -241,12 +227,12 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error {
continue
}
// wrong file
if !validItemFileName(item.Name, info.fauthor, info.fname) {
// not the item we're looking for
if !item.validPath(info.fauthor, info.fname) {
continue
}
if path == w.hubdir+"/"+item.RemotePath {
if path == h.cfg.HubDir+"/"+item.RemotePath {
log.Tracef("marking %s as downloaded", item.Name)
item.Downloaded = true
}
@ -303,7 +289,7 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error {
if !match {
log.Tracef("got tainted match for %s: %s", item.Name, path)
hubIdx.skippedTainted++
h.skippedTainted++
// the file and the stage is right, but the hash is wrong, it has been tainted by user
if !inhub {
item.LocalPath = path
@ -316,7 +302,7 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error {
item.LocalHash = sha
}
hubIdx.Items[info.ftype][name] = item
h.Items[info.ftype][name] = item
return nil
}
@ -326,13 +312,13 @@ func (w Walker) itemVisit(path string, f os.DirEntry, err error) error {
return nil
}
func CollecDepsCheck(v *Item) error {
if v.versionStatus() != 0 { // not up-to-date
log.Debugf("%s dependencies not checked : not up-to-date", v.Name)
func (h *Hub) CollectDepsCheck(v *Item) error {
if v.Type != COLLECTIONS {
return nil
}
if v.Type != COLLECTIONS {
if v.versionStatus() != 0 { // not up-to-date
log.Debugf("%s dependencies not checked: not up-to-date", v.Name)
return nil
}
@ -342,7 +328,7 @@ func CollecDepsCheck(v *Item) error {
for idx, itemSlice := range [][]string{v.Parsers, v.PostOverflows, v.Scenarios, v.Collections} {
sliceType := ItemTypes[idx]
for _, subName := range itemSlice {
subItem, ok := hubIdx.Items[sliceType][subName]
subItem, ok := h.Items[sliceType][subName]
if !ok {
return fmt.Errorf("referred %s %s in collection %s doesn't exist", sliceType, subName, v.Name)
}
@ -356,7 +342,7 @@ func CollecDepsCheck(v *Item) error {
if subItem.Type == COLLECTIONS {
log.Tracef("collec, recurse.")
if err := CollecDepsCheck(&subItem); err != nil {
if err := h.CollectDepsCheck(&subItem); err != nil {
if subItem.Tainted {
v.Tainted = true
}
@ -364,7 +350,7 @@ func CollecDepsCheck(v *Item) error {
return fmt.Errorf("sub collection %s is broken: %w", subItem.Name, err)
}
hubIdx.Items[sliceType][subName] = subItem
h.Items[sliceType][subName] = subItem
}
// propagate the state of sub-items to set
@ -395,7 +381,7 @@ func CollecDepsCheck(v *Item) error {
subItem.BelongsToCollections = append(subItem.BelongsToCollections, v.Name)
}
hubIdx.Items[sliceType][subName] = subItem
h.Items[sliceType][subName] = subItem
log.Tracef("checking for %s - tainted:%t uptodate:%t", subName, v.Tainted, v.UpToDate)
}
@ -404,7 +390,7 @@ func CollecDepsCheck(v *Item) error {
return nil
}
func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) {
func (h *Hub) SyncDir(dir string) ([]string, error) {
warnings := []string{}
// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
@ -414,13 +400,13 @@ func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) {
log.Errorf("failed %s : %s", cpath, err)
}
err = filepath.WalkDir(cpath, NewWalker(hub).itemVisit)
err = filepath.WalkDir(cpath, h.itemVisit)
if err != nil {
return warnings, err
}
}
for name, item := range hubIdx.Items[COLLECTIONS] {
for name, item := range h.Items[COLLECTIONS] {
if !item.Installed {
continue
}
@ -428,9 +414,9 @@ func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) {
vs := item.versionStatus()
switch vs {
case 0: // latest
if err := CollecDepsCheck(&item); err != nil {
if err := h.CollectDepsCheck(&item); err != nil {
warnings = append(warnings, fmt.Sprintf("dependency of %s: %s", item.Name, err))
hubIdx.Items[COLLECTIONS][name] = item
h.Items[COLLECTIONS][name] = item
}
case 1: // not up-to-date
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.LocalVersion, item.Version))
@ -445,51 +431,55 @@ func SyncDir(hub *csconfig.HubCfg, dir string) ([]string, error) {
}
// Updates the info from HubInit() with the local state
func LocalSync(hub *csconfig.HubCfg) ([]string, error) {
hubIdx.skippedLocal = 0
hubIdx.skippedTainted = 0
func (h *Hub) LocalSync() ([]string, error) {
h.skippedLocal = 0
h.skippedTainted = 0
warnings, err := SyncDir(hub, hub.InstallDir)
warnings, err := h.SyncDir(h.cfg.InstallDir)
if err != nil {
return warnings, fmt.Errorf("failed to scan %s: %w", hub.InstallDir, err)
return warnings, fmt.Errorf("failed to scan %s: %w", h.cfg.InstallDir, err)
}
_, err = SyncDir(hub, hub.HubDir)
_, err = h.SyncDir(h.cfg.HubDir)
if err != nil {
return warnings, fmt.Errorf("failed to scan %s: %w", hub.HubDir, err)
return warnings, fmt.Errorf("failed to scan %s: %w", h.cfg.HubDir, err)
}
return warnings, nil
}
func GetHubIdx(hub *csconfig.HubCfg) error {
if hub == nil {
return fmt.Errorf("no configuration found for hub")
// InitHub initializes the Hub, syncs the local state and returns the singleton for immediate use
func InitHub(cfg *csconfig.HubCfg) (*Hub, error) {
if cfg == nil {
return nil, fmt.Errorf("no configuration found for hub")
}
log.Debugf("loading hub idx %s", hub.HubIndexFile)
log.Debugf("loading hub idx %s", cfg.HubIndexFile)
bidx, err := os.ReadFile(hub.HubIndexFile)
bidx, err := os.ReadFile(cfg.HubIndexFile)
if err != nil {
return fmt.Errorf("unable to read index file: %w", err)
return nil, fmt.Errorf("unable to read index file: %w", err)
}
ret, err := ParseIndex(bidx)
if err != nil {
if !errors.Is(err, ErrMissingReference) {
return fmt.Errorf("unable to load existing index: %w", err)
return nil, fmt.Errorf("unable to load existing index: %w", err)
}
// XXX: why the error check if we bail out anyway?
return err
return nil, err
}
hubIdx = HubIndex{Items: ret}
theHub = &Hub{
Items: ret,
cfg: cfg,
}
_, err = LocalSync(hub)
_, err = theHub.LocalSync()
if err != nil {
return fmt.Errorf("failed to sync Hub index with local deployment : %w", err)
return nil, fmt.Errorf("failed to sync Hub index with local deployment : %w", err)
}
return nil
return theHub, nil
}

4
pkg/hubtest/hubtest.go
View file

@ -18,7 +18,7 @@ type HubTest struct {
TemplateConfigPath string
TemplateProfilePath string
TemplateSimulationPath string
HubIndex *cwhub.HubIndex
HubIndex *cwhub.Hub
Tests []*HubTestItem
}
@ -80,7 +80,7 @@ func NewHubTest(hubPath string, crowdsecPath string, cscliPath string) (HubTest,
TemplateConfigPath: templateConfigFilePath,
TemplateProfilePath: templateProfilePath,
TemplateSimulationPath: templateSimulationPath,
HubIndex: &cwhub.HubIndex{Items: hubIndex},
HubIndex: &cwhub.Hub{Items: hubIndex},
}, nil
}

18
pkg/hubtest/hubtest_item.go
View file

@ -52,7 +52,7 @@ type HubTestItem struct {
TemplateConfigPath string
TemplateProfilePath string
TemplateSimulationPath string
HubIndex *cwhub.HubIndex
HubIndex *cwhub.Hub
Config *HubTestItemConfig
@ -391,16 +391,16 @@ func (t *HubTestItem) InstallHub() error {
}
// load installed hub
err := cwhub.GetHubIdx(t.RuntimeHubConfig)
hub, err := cwhub.InitHub(t.RuntimeHubConfig)
if err != nil {
log.Fatalf("can't local sync the hub: %+v", err)
log.Fatal(err)
}
// install data for parsers if needed
ret := cwhub.GetItemMap(cwhub.PARSERS)
ret := hub.GetItemMap(cwhub.PARSERS)
for parserName, item := range ret {
if item.Installed {
if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {
if err := hub.DownloadDataIfNeeded(item, true); err != nil {
return fmt.Errorf("unable to download data for parser '%s': %+v", parserName, err)
}
log.Debugf("parser '%s' installed successfully in runtime environment", parserName)
@ -408,10 +408,10 @@ func (t *HubTestItem) InstallHub() error {
}
// install data for scenarios if needed
ret = cwhub.GetItemMap(cwhub.SCENARIOS)
ret = hub.GetItemMap(cwhub.SCENARIOS)
for scenarioName, item := range ret {
if item.Installed {
if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {
if err := hub.DownloadDataIfNeeded(item, true); err != nil {
return fmt.Errorf("unable to download data for parser '%s': %+v", scenarioName, err)
}
log.Debugf("scenario '%s' installed successfully in runtime environment", scenarioName)
@ -419,10 +419,10 @@ func (t *HubTestItem) InstallHub() error {
}
// install data for postoverflows if needed
ret = cwhub.GetItemMap(cwhub.POSTOVERFLOWS)
ret = hub.GetItemMap(cwhub.POSTOVERFLOWS)
for postoverflowName, item := range ret {
if item.Installed {
if err := cwhub.DownloadDataIfNeeded(t.RuntimeHubConfig, item, true); err != nil {
if err := hub.DownloadDataIfNeeded(item, true); err != nil {
return fmt.Errorf("unable to download data for parser '%s': %+v", postoverflowName, err)
}
log.Debugf("postoverflow '%s' installed successfully in runtime environment", postoverflowName)

32
pkg/leakybucket/buckets_test.go
View file

@ -8,12 +8,14 @@ import (
"html/template"
"io"
"os"
"path/filepath"
"reflect"
"sync"
"testing"
"time"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
"github.com/crowdsecurity/crowdsec/pkg/exprhelpers"
"github.com/crowdsecurity/crowdsec/pkg/parser"
"github.com/crowdsecurity/crowdsec/pkg/types"
@ -33,7 +35,20 @@ func TestBucket(t *testing.T) {
envSetting = os.Getenv("TEST_ONLY")
tomb = &tomb.Tomb{}
)
err := exprhelpers.Init(nil)
testdata := "./tests"
hubCfg := &csconfig.HubCfg{
HubDir: filepath.Join(testdata, "hub"),
HubIndexFile: filepath.Join(testdata, "hub", "index.json"),
}
_, err := cwhub.InitHub(hubCfg)
if err != nil {
t.Fatalf("failed to init hub : %s", err)
}
err = exprhelpers.Init(nil)
if err != nil {
log.Fatalf("exprhelpers init failed: %s", err)
}
@ -44,12 +59,15 @@ func TestBucket(t *testing.T) {
}
} else {
wg := new(sync.WaitGroup)
fds, err := os.ReadDir("./tests/")
fds, err := os.ReadDir(testdata)
if err != nil {
t.Fatalf("Unable to read test directory : %s", err)
}
for _, fd := range fds {
fname := "./tests/" + fd.Name()
if fd.Name() == "hub" {
continue
}
fname := filepath.Join(testdata, fd.Name())
log.Infof("Running test on %s", fname)
tomb.Go(func() error {
wg.Add(1)
@ -112,10 +130,8 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error {
files = append(files, x.Filename)
}
cscfg := &csconfig.CrowdsecServiceCfg{
DataDir: "tests",
}
holders, response, err := LoadBuckets(cscfg, files, tomb, buckets, false)
cscfg := &csconfig.CrowdsecServiceCfg{}
holders, response, err := LoadBuckets(cscfg, "tests", files, tomb, buckets, false)
if err != nil {
t.Fatalf("failed loading bucket : %s", err)
}
@ -123,7 +139,7 @@ func testOneBucket(t *testing.T, dir string, tomb *tomb.Tomb) error {
watchTomb(tomb)
return nil
})
if !testFile(t, dir+"/test.json", dir+"/in-buckets_state.json", holders, response, buckets) {
if !testFile(t, filepath.Join(dir, "test.json"), filepath.Join(dir, "in-buckets_state.json"), holders, response, buckets) {
return fmt.Errorf("tests from %s failed", dir)
}
return nil

11
pkg/leakybucket/manager_load.go
View file

@ -179,12 +179,17 @@ func ValidateFactory(bucketFactory *BucketFactory) error {
return nil
}
func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) {
func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, dataDir string, files []string, tomb *tomb.Tomb, buckets *Buckets, orderEvent bool) ([]BucketFactory, chan types.Event, error) {
var (
ret = []BucketFactory{}
response chan types.Event
)
hub, err := cwhub.GetHub()
if err != nil {
return nil, nil, err
}
response = make(chan types.Event, 1)
for _, f := range files {
log.Debugf("Loading '%s'", f)
@ -212,7 +217,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
log.Tracef("End of yaml file")
break
}
bucketFactory.DataDir = cscfg.DataDir
bucketFactory.DataDir = dataDir
//check empty
if bucketFactory.Name == "" {
log.Errorf("Won't load nameless bucket")
@ -235,7 +240,7 @@ func LoadBuckets(cscfg *csconfig.CrowdsecServiceCfg, files []string, tomb *tomb.
bucketFactory.Filename = filepath.Clean(f)
bucketFactory.BucketName = seed.Generate()
bucketFactory.ret = response
hubItem, err := cwhub.GetItemByPath(cwhub.SCENARIOS, bucketFactory.Filename)
hubItem, err := hub.GetItemByPath(cwhub.SCENARIOS, bucketFactory.Filename)
if err != nil {
log.Errorf("scenario %s (%s) couldn't be find in hub (ignore if in unit tests)", bucketFactory.Name, bucketFactory.Filename)
} else {

1
pkg/leakybucket/tests/hub/index.json Normal file
View file

@ -0,0 +1 @@
{}

14
pkg/parser/unix_parser.go
View file

@ -64,8 +64,12 @@ func NewParsers() *Parsers {
StageFiles: make([]Stagefile, 0),
PovfwStageFiles: make([]Stagefile, 0),
}
// XXX: handle error
hub, _ := cwhub.GetHub()
for _, itemType := range []string{cwhub.PARSERS, cwhub.POSTOVERFLOWS} {
for _, hubParserItem := range cwhub.GetItemMap(itemType) {
for _, hubParserItem := range hub.GetItemMap(itemType) {
if hubParserItem.Installed {
stagefile := Stagefile{
Filename: hubParserItem.LocalPath,
@ -97,16 +101,16 @@ func NewParsers() *Parsers {
func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
var err error
patternsDir := filepath.Join(cConfig.Crowdsec.ConfigDir, "patterns/")
patternsDir := filepath.Join(cConfig.ConfigPaths.ConfigDir, "patterns/")
log.Infof("Loading grok library %s", patternsDir)
/* load base regexps for two grok parsers */
parsers.Ctx, err = Init(map[string]interface{}{"patterns": patternsDir,
"data": cConfig.Crowdsec.DataDir})
"data": cConfig.ConfigPaths.DataDir})
if err != nil {
return parsers, fmt.Errorf("failed to load parser patterns : %v", err)
}
parsers.Povfwctx, err = Init(map[string]interface{}{"patterns": patternsDir,
"data": cConfig.Crowdsec.DataDir})
"data": cConfig.ConfigPaths.DataDir})
if err != nil {
return parsers, fmt.Errorf("failed to load postovflw parser patterns : %v", err)
}
@ -116,7 +120,7 @@ func LoadParsers(cConfig *csconfig.Config, parsers *Parsers) (*Parsers, error) {
*/
log.Infof("Loading enrich plugins")
parsers.EnricherCtx, err = Loadplugin(cConfig.Crowdsec.DataDir)
parsers.EnricherCtx, err = Loadplugin(cConfig.ConfigPaths.DataDir)
if err != nil {
return parsers, fmt.Errorf("failed to load enrich plugin : %v", err)
}

11
pkg/setup/install.go
View file

@ -54,7 +54,8 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
cwhub.SetHubBranch()
if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
hub, err := cwhub.InitHub(csConfig.Hub)
if err != nil {
return fmt.Errorf("getting hub index: %w", err)
}
@ -75,7 +76,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
continue
}
if err := cwhub.InstallItem(csConfig, collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
if err := hub.InstallItem(collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing collection %s: %w", collection, err)
}
}
@ -89,7 +90,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
continue
}
if err := cwhub.InstallItem(csConfig, parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
if err := hub.InstallItem(parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing parser %s: %w", parser, err)
}
}
@ -103,7 +104,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
continue
}
if err := cwhub.InstallItem(csConfig, scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
if err := hub.InstallItem(scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing scenario %s: %w", scenario, err)
}
}
@ -117,7 +118,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool) error
continue
}
if err := cwhub.InstallItem(csConfig, postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil {
if err := hub.InstallItem(postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err)
}
}