beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Merge branch 'master' into coraza_poc_acquis

Browse source
This commit is contained in:
blotus 2023-12-04 10:00:10 +01:00 committed by GitHub
commit 872e218b31
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 221 additions and 203 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

40
cmd/crowdsec-cli/hub.go
View file

@ -1,11 +1,13 @@
package main
import (
"encoding/json"
"fmt"
"github.com/fatih/color"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
@ -29,6 +31,7 @@ cscli hub upgrade`,
cmdHub.AddCommand(NewHubListCmd())
cmdHub.AddCommand(NewHubUpdateCmd())
cmdHub.AddCommand(NewHubUpgradeCmd())
cmdHub.AddCommand(NewHubTypesCmd())
return cmdHub
}
@ -172,3 +175,40 @@ Upgrade all configs installed from Crowdsec Hub. Run 'sudo cscli hub update' if
return cmdHubUpgrade
}
func runHubTypes(cmd *cobra.Command, args []string) error {
switch csConfig.Cscli.Output {
case "human":
s, err := yaml.Marshal(cwhub.ItemTypes)
if err != nil {
return err
}
fmt.Print(string(s))
case "json":
jsonStr, err := json.Marshal(cwhub.ItemTypes)
if err != nil {
return err
}
fmt.Println(string(jsonStr))
case "raw":
for _, itemType := range cwhub.ItemTypes {
fmt.Println(itemType)
}
}
return nil
}
func NewHubTypesCmd() *cobra.Command {
cmdHubTypes := &cobra.Command{
Use: "types",
Short: "List supported item types",
Long: `
List the types of supported hub items.
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
RunE: runHubTypes,
}
return cmdHubTypes
}

8
cmd/crowdsec-cli/itemcommands.go
View file

@ -36,7 +36,7 @@ type hubItemType struct {
var hubItemTypes = map[string]hubItemType{
"parsers": {
name: "parsers",
name: cwhub.PARSERS,
singular: "parser",
oneOrMore: "parser(s)",
help: cmdHelp{
@ -68,7 +68,7 @@ List only enabled parsers unless "-a" or names are specified.`,
},
},
"postoverflows": {
name: "postoverflows",
name: cwhub.POSTOVERFLOWS,
singular: "postoverflow",
oneOrMore: "postoverflow(s)",
help: cmdHelp{
@ -100,7 +100,7 @@ List only enabled postoverflows unless "-a" or names are specified.`,
},
},
"scenarios": {
name: "scenarios",
name: cwhub.SCENARIOS,
singular: "scenario",
oneOrMore: "scenario(s)",
help: cmdHelp{
@ -192,7 +192,7 @@ cscli waap-configs list crowdsecurity/vpatch`,
},
},
"collections": {
name: "collections",
name: cwhub.COLLECTIONS,
singular: "collection",
oneOrMore: "collection(s)",
help: cmdHelp{

39
pkg/cwhub/hub.go
View file

@ -9,13 +9,14 @@ import (
"strings"
log "github.com/sirupsen/logrus"
"slices"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
// Hub is the main structure for the package.
type Hub struct {
Items HubItems // Items read from HubDir and InstallDir
items HubItems // Items read from HubDir and InstallDir
local *csconfig.LocalHubCfg
remote *RemoteHubCfg
Warnings []string // Warnings encountered during sync
@ -65,7 +66,7 @@ func (h *Hub) parseIndex() error {
return fmt.Errorf("unable to read index file: %w", err)
}
if err := json.Unmarshal(bidx, &h.Items); err != nil {
if err := json.Unmarshal(bidx, &h.items); err != nil {
return fmt.Errorf("failed to unmarshal index: %w", err)
}
@ -73,9 +74,9 @@ func (h *Hub) parseIndex() error {
// Iterate over the different types to complete the struct
for _, itemType := range ItemTypes {
log.Tracef("%s: %d items", itemType, len(h.Items[itemType]))
log.Tracef("%s: %d items", itemType, len(h.GetItemMap(itemType)))
for name, item := range h.Items[itemType] {
for name, item := range h.GetItemMap(itemType) {
item.hub = h
item.Name = name
@ -101,13 +102,13 @@ func (h *Hub) ItemStats() []string {
tainted := 0
for _, itemType := range ItemTypes {
if len(h.Items[itemType]) == 0 {
if len(h.GetItemMap(itemType)) == 0 {
continue
}
loaded += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType)
loaded += fmt.Sprintf("%d %s, ", len(h.GetItemMap(itemType)), itemType)
for _, item := range h.Items[itemType] {
for _, item := range h.GetItemMap(itemType) {
if item.IsLocal() {
local++
}
@ -160,9 +161,17 @@ func (h *Hub) updateIndex() error {
return nil
}
func (h *Hub) addItem(item *Item) {
if h.items[item.Type] == nil {
h.items[item.Type] = make(map[string]*Item)
}
h.items[item.Type][item.Name] = item
}
// GetItemMap returns the map of items for a given type.
func (h *Hub) GetItemMap(itemType string) map[string]*Item {
return h.Items[itemType]
return h.items[itemType]
}
// GetItem returns an item from hub based on its type and full name (author/name).
@ -188,11 +197,12 @@ func (h *Hub) GetItemNames(itemType string) []string {
// GetAllItems returns a slice of all the items of a given type, installed or not.
func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
items, ok := h.Items[itemType]
if !ok {
return nil, fmt.Errorf("no %s in the hub index", itemType)
if !slices.Contains(ItemTypes, itemType) {
return nil, fmt.Errorf("invalid item type %s", itemType)
}
items := h.items[itemType]
ret := make([]*Item, len(items))
idx := 0
@ -207,11 +217,12 @@ func (h *Hub) GetAllItems(itemType string) ([]*Item, error) {
// GetInstalledItems returns a slice of the installed items of a given type.
func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
items, ok := h.Items[itemType]
if !ok {
return nil, fmt.Errorf("no %s in the hub index", itemType)
if !slices.Contains(ItemTypes, itemType) {
return nil, fmt.Errorf("invalid item type %s", itemType)
}
items := h.items[itemType]
retItems := make([]*Item, 0)
for _, item := range items {

2
pkg/cwhub/item_test.go
View file

@ -63,7 +63,7 @@ func TestGetters(t *testing.T) {
// Add item and get it
item.Name += "nope"
hub.Items[item.Type][item.Name] = item
hub.addItem(item)
newitem := hub.GetItem(COLLECTIONS, item.Name)
require.NotNil(t, newitem)

42
pkg/cwhub/iteminstall_test.go
View file

@ -16,9 +16,9 @@ func testInstall(hub *Hub, t *testing.T, item *Item) {
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name)
assert.True(t, item.State.UpToDate, "%s should be up-to-date", item.Name)
assert.False(t, item.State.Installed, "%s should not be installed", item.Name)
assert.False(t, item.State.Tainted, "%s should not be tainted", item.Name)
err = item.enable()
require.NoError(t, err, "failed to enable %s", item.Name)
@ -26,11 +26,11 @@ func testInstall(hub *Hub, t *testing.T, item *Item) {
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name)
assert.True(t, item.State.Installed, "%s should be installed", item.Name)
}
func testTaint(hub *Hub, t *testing.T, item *Item) {
assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted", item.Name)
assert.False(t, item.State.Tainted, "%s should not be tainted", item.Name)
// truncate the file
f, err := os.Create(item.State.LocalPath)
@ -41,11 +41,11 @@ func testTaint(hub *Hub, t *testing.T, item *Item) {
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should be tainted", item.Name)
assert.True(t, item.State.Tainted, "%s should be tainted", item.Name)
}
func testUpdate(hub *Hub, t *testing.T, item *Item) {
assert.False(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should not be up-to-date", item.Name)
assert.False(t, item.State.UpToDate, "%s should not be up-to-date", item.Name)
// Update it + check status
_, err := item.downloadLatest(true, true)
@ -55,12 +55,12 @@ func testUpdate(hub *Hub, t *testing.T, item *Item) {
err = hub.localSync()
require.NoError(t, err, "failed to run localSync")
assert.True(t, hub.Items[item.Type][item.Name].State.UpToDate, "%s should be up-to-date", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name)
assert.True(t, item.State.UpToDate, "%s should be up-to-date", item.Name)
assert.False(t, item.State.Tainted, "%s should not be tainted anymore", item.Name)
}
func testDisable(hub *Hub, t *testing.T, item *Item) {
assert.True(t, hub.Items[item.Type][item.Name].State.Installed, "%s should be installed", item.Name)
assert.True(t, item.State.Installed, "%s should be installed", item.Name)
// Remove
_, err := item.disable(false, false)
@ -71,9 +71,9 @@ func testDisable(hub *Hub, t *testing.T, item *Item) {
require.NoError(t, err, "failed to run localSync")
require.Empty(t, hub.Warnings)
assert.False(t, hub.Items[item.Type][item.Name].State.Tainted, "%s should not be tainted anymore", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name)
assert.True(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should still be downloaded", item.Name)
assert.False(t, item.State.Tainted, "%s should not be tainted anymore", item.Name)
assert.False(t, item.State.Installed, "%s should not be installed anymore", item.Name)
assert.True(t, item.State.Downloaded, "%s should still be downloaded", item.Name)
// Purge
_, err = item.disable(true, false)
@ -84,8 +84,8 @@ func testDisable(hub *Hub, t *testing.T, item *Item) {
require.NoError(t, err, "failed to run localSync")
require.Empty(t, hub.Warnings)
assert.False(t, hub.Items[item.Type][item.Name].State.Installed, "%s should not be installed anymore", item.Name)
assert.False(t, hub.Items[item.Type][item.Name].State.Downloaded, "%s should not be downloaded", item.Name)
assert.False(t, item.State.Installed, "%s should not be installed anymore", item.Name)
assert.False(t, item.State.Downloaded, "%s should not be downloaded", item.Name)
}
func TestInstallParser(t *testing.T) {
@ -101,15 +101,11 @@ func TestInstallParser(t *testing.T) {
hub := envSetup(t)
// map iteration is random by itself
for _, it := range hub.Items[PARSERS] {
for _, it := range hub.GetItemMap(PARSERS) {
testInstall(hub, t, it)
it = hub.Items[PARSERS][it.Name]
testTaint(hub, t, it)
it = hub.Items[PARSERS][it.Name]
testUpdate(hub, t, it)
it = hub.Items[PARSERS][it.Name]
testDisable(hub, t, it)
break
}
}
@ -127,15 +123,11 @@ func TestInstallCollection(t *testing.T) {
hub := envSetup(t)
// map iteration is random by itself
for _, it := range hub.Items[COLLECTIONS] {
for _, it := range hub.GetItemMap(COLLECTIONS) {
testInstall(hub, t, it)
it = hub.Items[COLLECTIONS][it.Name]
testTaint(hub, t, it)
it = hub.Items[COLLECTIONS][it.Name]
testUpdate(hub, t, it)
it = hub.Items[COLLECTIONS][it.Name]
testDisable(hub, t, it)
break
}
}

109
pkg/cwhub/itemupgrade_test.go
View file

@ -13,20 +13,21 @@ import (
func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
hub := envSetup(t)
// fresh install of collection
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
// fresh install of collection
require.False(t, item.State.Downloaded)
require.False(t, item.State.Installed)
require.NoError(t, item.Install(false, false))
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
require.True(t, item.State.Downloaded)
require.True(t, item.State.Installed)
require.True(t, item.State.UpToDate)
require.False(t, item.State.Tainted)
// This is the scenario that gets added in next version of collection
require.Nil(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"])
require.Nil(t, hub.GetItem(SCENARIOS, "crowdsecurity/barfoo_scenario"))
assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
@ -44,19 +45,20 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
hub = getHubOrFail(t, hub.local, remote)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
require.True(t, item.State.Downloaded)
require.True(t, item.State.Installed)
require.False(t, item.State.UpToDate)
require.False(t, item.State.Tainted)
didUpdate, err := item.Upgrade(false)
require.NoError(t, err)
require.True(t, didUpdate)
assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Downloaded)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed)
require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/barfoo_scenario").State.Downloaded)
require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/barfoo_scenario").State.Installed)
}
// Install a collection, disable a scenario.
@ -64,19 +66,20 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
hub := envSetup(t)
// fresh install of collection
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
// fresh install of collection
require.False(t, item.State.Downloaded)
require.False(t, item.State.Installed)
require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
require.NoError(t, item.Install(false, false))
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
require.True(t, item.State.Downloaded)
require.True(t, item.State.Installed)
require.True(t, item.State.UpToDate)
require.False(t, item.State.Tainted)
require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario")
@ -92,11 +95,12 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
hub = getHubOrFail(t, hub.local, remote)
// scenario referenced by collection was deleted hence, collection should be tainted
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Tainted)
require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Downloaded)
require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Installed)
require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.UpToDate)
hub, err = NewHub(hub.local, remote, true)
require.NoError(t, err, "failed to download index: %s", err)
@ -107,7 +111,7 @@ func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
require.False(t, didUpdate)
hub = getHubOrFail(t, hub.local, remote)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
}
// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test.
@ -124,19 +128,20 @@ func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCf
func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) {
hub := envSetup(t)
// fresh install of collection
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
// fresh install of collection
require.False(t, item.State.Downloaded)
require.False(t, item.State.Installed)
require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
require.NoError(t, item.Install(false, false))
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
require.True(t, item.State.Downloaded)
require.True(t, item.State.Installed)
require.True(t, item.State.UpToDate)
require.False(t, item.State.Tainted)
require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
assertCollectionDepsInstalled(t, hub, "crowdsecurity/test_collection")
item = hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario")
@ -152,12 +157,12 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te
hub = getHubOrFail(t, hub.local, remote)
// scenario referenced by collection was deleted hence, collection should be tainted
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Downloaded) // this fails
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Tainted)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].State.UpToDate)
require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Downloaded) // this fails
require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Tainted)
require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Downloaded)
require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.Installed)
require.True(t, hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection").State.UpToDate)
// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
// we now attempt to upgrade the collection, however it shouldn't install the foobar_scenario
@ -167,7 +172,7 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te
hub, err = NewHub(hub.local, remote, true)
require.NoError(t, err, "failed to download index: %s", err)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
hub = getHubOrFail(t, hub.local, remote)
item = hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
@ -176,14 +181,14 @@ func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *te
require.True(t, didUpdate)
hub = getHubOrFail(t, hub.local, remote)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].State.Installed)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].State.Installed)
require.False(t, hub.GetItem(SCENARIOS, "crowdsecurity/foobar_scenario").State.Installed)
require.True(t, hub.GetItem(SCENARIOS, "crowdsecurity/barfoo_scenario").State.Installed)
}
func assertCollectionDepsInstalled(t *testing.T, hub *Hub, collection string) {
t.Helper()
c := hub.Items[COLLECTIONS][collection]
c := hub.GetItem(COLLECTIONS, collection)
require.NoError(t, c.checkSubItemVersions())
}

8
pkg/cwhub/sync.go
View file

@ -232,7 +232,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
if err != nil {
return err
}
h.Items[info.ftype][item.Name] = item
h.addItem(item)
return nil
}
@ -252,7 +252,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
// try to find which configuration item it is
log.Tracef("check [%s] of %s", info.fname, info.ftype)
for name, item := range h.Items[info.ftype] {
for _, item := range h.GetItemMap(info.ftype) {
if info.fname != item.FileName {
continue
}
@ -293,8 +293,6 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
return err
}
h.Items[info.ftype][name] = item
return nil
}
@ -401,7 +399,7 @@ func (h *Hub) localSync() error {
warnings := make([]string, 0)
for _, item := range h.Items[COLLECTIONS] {
for _, item := range h.GetItemMap(COLLECTIONS) {
// check for cyclic dependencies
subs, err := item.descendants()
if err != nil {

8
pkg/hubtest/coverage.go
View file

@ -79,12 +79,12 @@ func (h *HubTest) GetWaapCoverage() ([]Coverage, error) {
}
func (h *HubTest) GetParsersCoverage() ([]Coverage, error) {
if _, ok := h.HubIndex.Items[cwhub.PARSERS]; !ok {
if len(h.HubIndex.GetItemMap(cwhub.PARSERS)) == 0 {
return nil, fmt.Errorf("no parsers in hub index")
}
// populate from hub, iterate in alphabetical order
pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.PARSERS])
pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.PARSERS))
coverage := make([]Coverage, len(pkeys))
for i, name := range pkeys {
@ -165,12 +165,12 @@ func (h *HubTest) GetParsersCoverage() ([]Coverage, error) {
}
func (h *HubTest) GetScenariosCoverage() ([]Coverage, error) {
if _, ok := h.HubIndex.Items[cwhub.SCENARIOS]; !ok {
if len(h.HubIndex.GetItemMap(cwhub.SCENARIOS)) == 0 {
return nil, fmt.Errorf("no scenarios in hub index")
}
// populate from hub, iterate in alphabetical order
pkeys := sortedMapKeys(h.HubIndex.Items[cwhub.SCENARIOS])
pkeys := sortedMapKeys(h.HubIndex.GetItemMap(cwhub.SCENARIOS))
coverage := make([]Coverage, len(pkeys))
for i, name := range pkeys {

6
pkg/hubtest/hubtest_item.go
View file

@ -162,7 +162,7 @@ func (t *HubTestItem) InstallHub() error {
continue
}
if hubParser, ok := t.HubIndex.Items[cwhub.PARSERS][parser]; ok {
if hubParser := t.HubIndex.GetItem(cwhub.PARSERS, parser); hubParser != nil {
parserSource, err := filepath.Abs(filepath.Join(t.HubPath, hubParser.RemotePath))
if err != nil {
return fmt.Errorf("can't get absolute path of '%s': %s", parserSource, err)
@ -248,7 +248,7 @@ func (t *HubTestItem) InstallHub() error {
continue
}
if hubScenario, ok := t.HubIndex.Items[cwhub.SCENARIOS][scenario]; ok {
if hubScenario := t.HubIndex.GetItem(cwhub.SCENARIOS, scenario); hubScenario != nil {
scenarioSource, err := filepath.Abs(filepath.Join(t.HubPath, hubScenario.RemotePath))
if err != nil {
return fmt.Errorf("can't get absolute path to: %s", scenarioSource)
@ -392,7 +392,7 @@ func (t *HubTestItem) InstallHub() error {
continue
}
if hubPostOverflow, ok := t.HubIndex.Items[cwhub.POSTOVERFLOWS][postoverflow]; ok {
if hubPostOverflow := t.HubIndex.GetItem(cwhub.POSTOVERFLOWS, postoverflow); hubPostOverflow != nil {
postoverflowSource, err := filepath.Abs(filepath.Join(t.HubPath, hubPostOverflow.RemotePath))
if err != nil {
return fmt.Errorf("can't get absolute path of '%s': %s", postoverflowSource, err)

13
test/bats/20_hub.bats
View file

@ -126,3 +126,16 @@ teardown() {
rune -0 cscli hub upgrade
assert_stderr --partial "not upgrading foo.yaml: local item"
}
@test "cscli hub types" {
rune -0 cscli hub types -o raw
assert_line "parsers"
assert_line "postoverflows"
assert_line "scenarios"
assert_line "collections"
rune -0 cscli hub types -o human
rune -0 yq -o json <(output)
assert_json '["parsers","postoverflows","scenarios","collections"]'
rune -0 cscli hub types -o json
assert_json '["parsers","postoverflows","scenarios","collections"]'
}

42
test/bin/preload-hub-items Executable file
View file

@ -0,0 +1,42 @@
#!/usr/bin/env bash
set -eu
# shellcheck disable=SC1007
THIS_DIR=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd)
# shellcheck disable=SC1091
. "${THIS_DIR}/../.environment.sh"
# pre-download everything but don't install anything
echo -n "Purging existing hub..."
types=$("$CSCLI" hub types -o raw)
for itemtype in $types; do
"$CSCLI" "${itemtype}" delete --all --error --purge --force
done
echo " done."
echo -n "Pre-downloading Hub content..."
for itemtype in $types; do
ALL_ITEMS=$("$CSCLI" "$itemtype" list -a -o json | jq --arg itemtype "$itemtype" -r '.[$itemtype][].name')
if [[ -n "${ALL_ITEMS}" ]]; then
#shellcheck disable=SC2086
"$CSCLI" "$itemtype" install \
$ALL_ITEMS \
--download-only \
--error
fi
done
# XXX: download-only works only for collections, not for parsers, scenarios, postoverflows.
# so we have to delete the links manually, and leave the downloaded files in place
for itemtype in $types; do
"$CSCLI" "$itemtype" delete --all --error
done
echo " done."

47
test/lib/config/config-global
View file

@ -54,50 +54,6 @@ remove_init_data() {
# we need a separate function for initializing config when testing package
# because we want to test the configuration as well
preload_hub_items() {
# pre-download everything but don't install anything
# each test can install what it needs
echo "Purging existing hub..."
"$CSCLI" parsers delete --all --error --purge --force
"$CSCLI" scenarios delete --all --error --purge --force
"$CSCLI" postoverflows delete --all --error --purge --force
"$CSCLI" collections delete --all --error --purge --force
echo "Pre-downloading hub content..."
#shellcheck disable=SC2046
"$CSCLI" collections install \
$("$CSCLI" collections list -a -o json | jq -r '.collections[].name') \
--download-only \
--error
#shellcheck disable=SC2046
"$CSCLI" parsers install \
$("$CSCLI" parsers list -a -o json | jq -r '.parsers[].name') \
--download-only \
--error
#shellcheck disable=SC2046
"$CSCLI" scenarios install \
$("$CSCLI" scenarios list -a -o json | jq -r '.scenarios[].name') \
--download-only \
--error
#shellcheck disable=SC2046
"$CSCLI" postoverflows install \
$("$CSCLI" postoverflows list -a -o json | jq -r '.postoverflows[].name') \
--download-only \
--error
# XXX: download-only works only for collections, not for parsers, scenarios, postoverflows.
# so we have to delete the links manually, and leave the downloaded files in place
"$CSCLI" parsers delete --all --error
"$CSCLI" scenarios delete --all --error
"$CSCLI" postoverflows delete --all --error
}
make_init_data() {
./bin/assert-crowdsec-not-running || die "Cannot create fixture data."
@ -108,7 +64,7 @@ make_init_data() {
# when installed packages are always using sqlite, so no need to regenerate
# local credz for sqlite
preload_hub_items
./bin/preload-hub-items
[[ "${DB_BACKEND}" == "sqlite" ]] || ${CSCLI} machines add --auto
@ -145,7 +101,6 @@ load_init_data() {
./instance-db restore "${LOCAL_INIT_DIR}/database"
}
# ---------------------------
[[ $# -lt 1 ]] && about

48
test/lib/config/config-local
View file

@ -101,51 +101,6 @@ config_generate() {
' ../config/config.yaml >"${CONFIG_DIR}/config.yaml"
}
preload_hub_items() {
# pre-download everything but don't install anything
# each test can install what it needs
echo "Purging existing hub..."
"$CSCLI" parsers delete --all --error --purge --force
"$CSCLI" scenarios delete --all --error --purge --force
"$CSCLI" postoverflows delete --all --error --purge --force
"$CSCLI" collections delete --all --error --purge --force
echo "Pre-downloading hub content..."
#shellcheck disable=SC2046
"$CSCLI" collections install \
$("$CSCLI" collections list -a -o json | jq -r '.collections[].name') \
--download-only \
--error
#shellcheck disable=SC2046
"$CSCLI" parsers install \
$("$CSCLI" parsers list -a -o json | jq -r '.parsers[].name') \
--download-only \
--error
#shellcheck disable=SC2046
"$CSCLI" scenarios install \
$("$CSCLI" scenarios list -a -o json | jq -r '.scenarios[].name') \
--download-only \
--error
#shellcheck disable=SC2046
"$CSCLI" postoverflows install \
$("$CSCLI" postoverflows list -a -o json | jq -r '.postoverflows[].name') \
--download-only \
--error
# XXX: download-only works only for collections, not for parsers, scenarios, postoverflows.
# so we have to delete the links manually, and leave the downloaded files in place
"$CSCLI" parsers delete --all --error
"$CSCLI" scenarios delete --all --error
"$CSCLI" postoverflows delete --all --error
}
make_init_data() {
./bin/assert-crowdsec-not-running || die "Cannot create fixture data."
@ -163,7 +118,7 @@ make_init_data() {
"$CSCLI" --warning machines add githubciXXXXXXXXXXXXXXXXXXXXXXXX --auto
"$CSCLI" --warning hub update
preload_hub_items
./bin/preload-hub-items
mkdir -p "$LOCAL_INIT_DIR"
@ -197,7 +152,6 @@ load_init_data() {
./instance-db restore "${LOCAL_INIT_DIR}/database"
}
# ---------------------------
[[ $# -lt 1 ]] && about

12
test/lib/setup_file.sh
View file

@ -243,8 +243,16 @@ export -f assert_stderr_line
hub_purge_all() {
local CONFIG_DIR
CONFIG_DIR=$(dirname "$CONFIG_YAML")
rm -rf "$CONFIG_DIR"/collections/* "$CONFIG_DIR"/parsers/*/* "$CONFIG_DIR"/scenarios/* "$CONFIG_DIR"/postoverflows/*
rm -rf "$CONFIG_DIR"/hub/collections/* "$CONFIG_DIR"/hub/parsers/*/* "$CONFIG_DIR"/hub/scenarios/* "$CONFIG_DIR"/hub/postoverflows/*
rm -rf \
"$CONFIG_DIR"/collections/* \
"$CONFIG_DIR"/parsers/*/* \
"$CONFIG_DIR"/scenarios/* \
"$CONFIG_DIR"/postoverflows/*
rm -rf \
"$CONFIG_DIR"/hub/collections/* \
"$CONFIG_DIR"/hub/parsers/*/* \
"$CONFIG_DIR"/hub/scenarios/* \
"$CONFIG_DIR"/hub/postoverflows/*
local DATA_DIR
DATA_DIR=$(config_get .config_paths.data_dir)
# should remove everything except the db (find $DATA_DIR -not -name "crowdsec.db*" -delete),