0ct0pu5/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Refact pkg/cwhub, cscli: hub upgrades (#2568)

Browse source 
* fix bats test for "upgrade all items"
* refact UpgradeConfig() -> UpgradeItem(): one item only
* refact RemoveMany() -> RemoveItem()
* Computed value: Item.Local -> Item.IsLocal()
* refact url/branch configuration with LocalHubCfg/RemoteHubCfg
This commit is contained in:
mmetc 2023-10-30 17:23:50 +01:00 • committed by GitHub
parent 6b8ed0c9d0
commit 17662e59a9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
47 changed files with 856 additions and 687 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

46
cmd/crowdsec-cli/branch.go
View file

@ -1,46 +0,0 @@
package main
// Set the appropriate hub branch according to config settings and crowdsec version
import (
log "github.com/sirupsen/logrus"
"golang.org/x/mod/semver"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
)
// chooseHubBranch returns the branch name to use for the hub
// It can be "master" or the branch corresponding to the current crowdsec version
func chooseHubBranch() string {
latest, err := cwversion.Latest()
if err != nil {
log.Warningf("Unable to retrieve latest crowdsec version: %s, defaulting to master", err)
return "master"
}
csVersion := cwversion.VersionStrip()
if csVersion == latest {
log.Debugf("current version is equal to latest (%s)", csVersion)
return "master"
}
// if current version is greater than the latest we are in pre-release
if semver.Compare(csVersion, latest) == 1 {
log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion)
return "master"
}
if csVersion == "" {
log.Warning("Crowdsec version is not set, using master branch for the hub")
return "master"
}
log.Warnf("Crowdsec is not the latest version. "+
"Current version is '%s' and the latest stable version is '%s'. Please update it!",
csVersion, latest)
log.Warnf("As a result, you will not be able to use parsers/scenarios/collections "+
"added to Crowdsec Hub after CrowdSec %s", latest)
return csVersion
}

2
cmd/crowdsec-cli/capi.go
View file

@ -151,7 +151,7 @@ func NewCapiStatusCmd() *cobra.Command {
return fmt.Errorf("parsing api url ('%s'): %w", csConfig.API.Server.OnlineClient.Credentials.URL, err)
}
hub, err := require.Hub(csConfig)
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}

29
cmd/crowdsec-cli/config_backup.go
View file

@ -13,11 +13,15 @@ import (
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
func backupHub(hub *cwhub.Hub, dirPath string) error {
var err error
func backupHub(dirPath string) error {
var itemDirectory string
var upstreamParsers []string
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
for _, itemType := range cwhub.ItemTypes {
clog := log.WithFields(log.Fields{
"type": itemType,
@ -28,7 +32,7 @@ func backupHub(hub *cwhub.Hub, dirPath string) error {
continue
}
itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
if err = os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
}
upstreamParsers = []string{}
@ -42,15 +46,15 @@ func backupHub(hub *cwhub.Hub, dirPath string) error {
}
//for the local/tainted ones, we backup the full file
if v.Tainted || v.Local || !v.UpToDate {
if v.Tainted || v.IsLocal() || !v.UpToDate {
//we need to backup stages for parsers
if itemType == cwhub.PARSERS || itemType == cwhub.POSTOVERFLOWS {
fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
if err = os.MkdirAll(fstagedir, os.ModePerm); err != nil {
return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
}
}
clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
clog.Debugf("[%s]: backing up file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.IsLocal(), v.UpToDate)
tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
if err = CopyFile(v.LocalPath, tfile); err != nil {
return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
@ -100,7 +104,7 @@ func backupConfigToDirectory(dirPath string) error {
/*if parent directory doesn't exist, bail out. create final dir with Mkdir*/
parentDir := filepath.Dir(dirPath)
if _, err := os.Stat(parentDir); err != nil {
if _, err = os.Stat(parentDir); err != nil {
return fmt.Errorf("while checking parent directory %s existence: %w", parentDir, err)
}
@ -189,12 +193,7 @@ func backupConfigToDirectory(dirPath string) error {
log.Infof("Saved profiles to %s", backupProfiles)
}
hub, err := cwhub.GetHub()
if err != nil {
return err
}
if err = backupHub(hub, dirPath); err != nil {
if err = backupHub(dirPath); err != nil {
return fmt.Errorf("failed to backup hub config: %s", err)
}
@ -202,10 +201,6 @@ func backupConfigToDirectory(dirPath string) error {
}
func runConfigBackup(cmd *cobra.Command, args []string) error {
if _, err := require.Hub(csConfig); err != nil {
return err
}
if err := backupConfigToDirectory(args[0]); err != nil {
return fmt.Errorf("failed to backup config: %w", err)
}

24
cmd/crowdsec-cli/config_restore.go
View file

@ -22,21 +22,16 @@ type OldAPICfg struct {
}
// it's a rip of the cli version, but in silent-mode
func silentInstallItem(name, obtype, hubURLTemplate, branch string) (string, error) {
hub, err := cwhub.GetHub()
if err != nil {
return "", err
}
func silentInstallItem(hub *cwhub.Hub, name, obtype string) (string, error) {
var item = hub.GetItem(obtype, name)
if item == nil {
return "", fmt.Errorf("error retrieving item")
}
err = hub.DownloadLatest(item, false, false, hubURLTemplate, branch)
err := hub.DownloadLatest(item, false, false)
if err != nil {
return "", fmt.Errorf("error while downloading %s : %v", item.Name, err)
}
if err := hub.AddItem(*item); err != nil {
if err = hub.AddItem(*item); err != nil {
return "", err
}
@ -51,9 +46,10 @@ func silentInstallItem(name, obtype, hubURLTemplate, branch string) (string, err
}
func restoreHub(dirPath string) error {
var err error
branch := chooseHubBranch()
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
for _, itype := range cwhub.ItemTypes {
itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
@ -73,7 +69,7 @@ func restoreHub(dirPath string) error {
return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
}
for _, toinstall := range upstreamList {
label, err := silentInstallItem(toinstall, itype, hubURLTemplate, branch)
label, err := silentInstallItem(hub, toinstall, itype)
if err != nil {
log.Errorf("Error while installing %s : %s", toinstall, err)
} else if label != "" {
@ -297,10 +293,6 @@ func runConfigRestore(cmd *cobra.Command, args []string) error {
return err
}
if _, err := require.Hub(csConfig); err != nil {
return err
}
if err := restoreConfigFromDirectory(args[0], oldBackup); err != nil {
return fmt.Errorf("failed to restore config from %s: %w", args[0], err)
}

2
cmd/crowdsec-cli/console.go
View file

@ -71,7 +71,7 @@ After running this command your will need to validate the enrollment in the weba
return fmt.Errorf("could not parse CAPI URL: %s", err)
}
hub, err := require.Hub(csConfig)
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}

77
cmd/crowdsec-cli/hub.go
View file

@ -12,13 +12,6 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
const (
hubURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
remoteIndexPath = ".index.json"
)
var hubBranch = ""
func NewHubCmd() *cobra.Command {
var cmdHub = &cobra.Command{
Use: "hub [action]",
@ -56,7 +49,7 @@ func runHubList(cmd *cobra.Command, args []string) error {
return err
}
hub, err := require.Hub(csConfig)
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
@ -71,7 +64,7 @@ func runHubList(cmd *cobra.Command, args []string) error {
log.Info(line)
}
err = ListItems(color.Output, []string{
err = ListItems(hub, color.Output, []string{
cwhub.COLLECTIONS, cwhub.PARSERS, cwhub.SCENARIOS, cwhub.POSTOVERFLOWS,
}, nil, true, false, all)
if err != nil {
@ -97,23 +90,20 @@ func NewHubListCmd() *cobra.Command {
}
func runHubUpdate(cmd *cobra.Command, args []string) error {
local := csConfig.Hub
remote := require.RemoteHub(csConfig)
// don't use require.Hub because if there is no index file, it would fail
branch := hubBranch
if branch == "" {
branch = chooseHubBranch()
}
log.Debugf("Using branch '%s' for the hub", branch)
hub, err := cwhub.InitHubUpdate(csConfig.Hub, hubURLTemplate, branch, remoteIndexPath)
hub, err := cwhub.InitHubUpdate(local, remote)
if err != nil {
// XXX: this should be done when downloading items, too
// but what is the fallback to master actually solving?
if !errors.Is(err, cwhub.ErrIndexNotFound) {
return fmt.Errorf("failed to get Hub index : %w", err)
}
log.Warnf("Could not find index file for branch '%s', using 'master'", branch)
branch = "master"
if hub, err = cwhub.InitHubUpdate(csConfig.Hub, hubURLTemplate, branch, remoteIndexPath); err != nil {
log.Warnf("Could not find index file for branch '%s', using 'master'", remote.Branch)
remote.Branch = "master"
if hub, err = cwhub.InitHubUpdate(local, remote); err != nil {
return fmt.Errorf("failed to get Hub index after retry: %w", err)
}
}
@ -132,7 +122,7 @@ func NewHubUpdateCmd() *cobra.Command {
Use: "update",
Short: "Download the latest index (catalog of available configurations)",
Long: `
Fetches the [.index.json](https://github.com/crowdsecurity/hub/blob/master/.index.json) file from hub, containing the list of available configs.
Fetches the .index.json file from the hub, containing the list of available configs.
`,
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
@ -157,36 +147,29 @@ func runHubUpgrade(cmd *cobra.Command, args []string) error {
return err
}
branch := hubBranch
if branch == "" {
branch = chooseHubBranch()
}
log.Debugf("Using branch '%s' for the hub", branch)
hub, err := require.Hub(csConfig)
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
log.Infof("Upgrading collections")
if err := hub.UpgradeConfig(cwhub.COLLECTIONS, "", force, hubURLTemplate, branch); err != nil {
return err
}
for _, itemType := range cwhub.ItemTypes {
items, err := hub.GetInstalledItems(itemType)
if err != nil {
return err
}
log.Infof("Upgrading parsers")
if err := hub.UpgradeConfig(cwhub.PARSERS, "", force, hubURLTemplate, branch); err != nil {
return err
}
log.Infof("Upgrading scenarios")
if err := hub.UpgradeConfig(cwhub.SCENARIOS, "", force, hubURLTemplate, branch); err != nil {
return err
}
log.Infof("Upgrading postoverflows")
if err := hub.UpgradeConfig(cwhub.POSTOVERFLOWS, "", force, hubURLTemplate, branch); err != nil {
return err
updated := 0
log.Infof("Upgrading %s", itemType)
for _, item := range items {
didUpdate, err := hub.UpgradeItem(itemType, item.Name, force)
if err != nil {
return err
}
if didUpdate {
updated++
}
}
log.Infof("Upgraded %d %s", updated, itemType)
}
return nil

1
cmd/crowdsec-cli/item_metrics.go
View file

@ -18,6 +18,7 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
// XXX: this should not need hub?
func ShowMetrics(hub *cwhub.Hub, hubItem *cwhub.Item) {
switch hubItem.Type {
case cwhub.PARSERS:

9
cmd/crowdsec-cli/item_suggest.go
View file

@ -29,14 +29,11 @@ func Suggest(itemType string, baseItem string, suggestItem string, score int, ig
}
}
func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
func GetDistance(hub *cwhub.Hub, itemType string, itemName string) (*cwhub.Item, int) {
allItems := make([]string, 0)
nearestScore := 100
nearestItem := &cwhub.Item{}
// XXX: handle error
hub, _ := cwhub.GetHub()
hubItems := hub.GetItemMap(itemType)
for _, item := range hubItems {
allItems = append(allItems, item.Name)
@ -53,7 +50,7 @@ func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
}
func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
hub, err := require.Hub(csConfig)
hub, err := require.Hub(csConfig, nil)
if err != nil {
return nil, cobra.ShellCompDirectiveDefault
}
@ -70,7 +67,7 @@ func compAllItems(itemType string, args []string, toComplete string) ([]string,
}
func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
hub, err := require.Hub(csConfig)
hub, err := require.Hub(csConfig, nil)
if err != nil {
return nil, cobra.ShellCompDirectiveDefault
}

106
cmd/crowdsec-cli/itemcommands.go
View file

@ -10,7 +10,6 @@ import (
"github.com/crowdsecurity/go-cs-lib/coalesce"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
type cmdHelp struct {
@ -168,19 +167,6 @@ func NewItemsCmd(typeName string) *cobra.Command {
Args: cobra.MinimumNArgs(1),
Aliases: []string{it.singular},
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
if _, err := require.Hub(csConfig); err != nil {
return err
}
return nil
},
PersistentPostRun: func(cmd *cobra.Command, args []string) {
if cmd.Name() == "inspect" || cmd.Name() == "list" {
return
}
log.Infof(ReloadMessage())
},
}
cmd.AddCommand(NewItemsInstallCmd(typeName))
@ -211,29 +197,29 @@ func itemsInstallRunner(it hubItemType) func(cmd *cobra.Command, args []string)
return err
}
hub, err := cwhub.GetHub()
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
branch := chooseHubBranch()
for _, name := range args {
t := hub.GetItem(it.name, name)
if t == nil {
nearestItem, score := GetDistance(it.name, name)
nearestItem, score := GetDistance(hub, it.name, name)
Suggest(it.name, name, nearestItem.Name, score, ignoreError)
continue
}
if err := hub.InstallItem(name, it.name, force, downloadOnly, hubURLTemplate, branch); err != nil {
if err := hub.InstallItem(name, it.name, force, downloadOnly); err != nil {
if !ignoreError {
return fmt.Errorf("error while installing '%s': %w", name, err)
}
log.Errorf("Error while installing '%s': %s", name, err)
}
}
// XXX: only reload if we installed something
log.Infof(ReloadMessage())
return nil
}
@ -283,17 +269,32 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e
return err
}
hub, err := cwhub.GetHub()
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
if all {
err := hub.RemoveMany(it.name, "", all, purge, force)
items, err := hub.GetInstalledItems(it.name)
if err != nil {
return err
}
removed := 0
for _, item := range items {
didRemove, err := hub.RemoveItem(it.name, item.Name, purge, force)
if err != nil {
return err
}
if didRemove {
removed++
}
}
log.Infof("Removed %d %s", removed, it.name)
if removed > 0 {
log.Infof(ReloadMessage())
}
return nil
}
@ -301,6 +302,7 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e
return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular)
}
removed := 0
for _, name := range args {
if !force {
item := hub.GetItem(it.name, name)
@ -315,10 +317,18 @@ func itemsRemoveRunner(it hubItemType) func(cmd *cobra.Command, args []string) e
}
}
err := hub.RemoveMany(it.name, name, all, purge, force)
didRemove, err := hub.RemoveItem(it.name, name, purge, force)
if err != nil {
return err
}
if didRemove {
log.Infof("Removed %s", name)
removed++
}
}
if removed > 0 {
log.Infof(ReloadMessage())
}
return nil
@ -364,17 +374,32 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string)
return err
}
hub, err := cwhub.GetHub()
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
branch := chooseHubBranch()
if all {
if err := hub.UpgradeConfig(it.name, "", force, hubURLTemplate, branch); err != nil {
items, err := hub.GetInstalledItems(it.name)
if err != nil {
return err
}
updated := 0
for _, item := range items {
didUpdate, err := hub.UpgradeItem(it.name, item.Name, force)
if err != nil {
return err
}
if didUpdate {
updated++
}
}
log.Infof("Updated %d %s", updated, it.name)
if updated > 0 {
log.Infof(ReloadMessage())
}
return nil
}
@ -382,10 +407,19 @@ func itemsUpgradeRunner(it hubItemType) func(cmd *cobra.Command, args []string)
return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular)
}
updated := 0
for _, name := range args {
if err := hub.UpgradeConfig(it.name, name, force, hubURLTemplate, branch); err != nil {
didUpdate, err := hub.UpgradeItem(it.name, name, force)
if err != nil {
return err
}
if didUpdate {
log.Infof("Updated %s", name)
updated++
}
}
if updated > 0 {
log.Infof(ReloadMessage())
}
return nil
@ -434,8 +468,17 @@ func itemsInspectRunner(it hubItemType) func(cmd *cobra.Command, args []string)
return err
}
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
for _, name := range args {
if err = InspectItem(name, it.name, noMetrics); err != nil {
item := hub.GetItem(it.name, name)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", name, it.name)
}
if err = InspectItem(hub, item, noMetrics); err != nil {
return err
}
}
@ -478,7 +521,12 @@ func itemsListRunner(it hubItemType) func(cmd *cobra.Command, args []string) err
return err
}
if err = ListItems(color.Output, []string{it.name}, args, false, true, all); err != nil {
hub, err := require.Hub(csConfig, nil)
if err != nil {
return err
}
if err = ListItems(hub, color.Output, []string{it.name}, args, false, true, all); err != nil {
return err
}

51
cmd/crowdsec-cli/items.go
View file

@ -5,23 +5,22 @@ import (
"encoding/json"
"fmt"
"io"
"slices"
"os"
"sort"
"strings"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v2"
"gopkg.in/yaml.v3"
"slices"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly bool) ([]string, error) {
itemNames := hub.GetItemNames(itemType)
notExist := []string{}
if len(args) > 0 {
installedOnly = false
for _, arg := range args {
if !slices.Contains(itemNames, arg) {
notExist = append(notExist, arg)
@ -35,6 +34,7 @@ func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly b
if len(args) > 0 {
itemNames = args
installedOnly = false
}
if installedOnly {
@ -49,25 +49,19 @@ func selectItems(hub *cwhub.Hub, itemType string, args []string, installedOnly b
return itemNames, nil
}
func ListItems(out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) error {
func ListItems(hub *cwhub.Hub, out io.Writer, itemTypes []string, args []string, showType bool, showHeader bool, all bool) error {
var err error
hub, err := cwhub.GetHub()
if err != nil {
return err
}
items := make(map[string][]string)
for _, itemType := range itemTypes {
if items[itemType], err = selectItems(hub, itemType, args, !all); err != nil {
return err
}
}
if csConfig.Cscli.Output == "human" {
for _, itemType := range itemTypes {
listHubItemTable(out, "\n"+strings.ToUpper(itemType), itemType, items[itemType])
listHubItemTable(hub, out, "\n"+strings.ToUpper(itemType), itemType, items[itemType])
}
} else if csConfig.Cscli.Output == "json" {
type itemHubStatus struct {
@ -114,7 +108,6 @@ func ListItems(out io.Writer, itemTypes []string, args []string, showType bool,
if err != nil {
log.Fatalf("failed to write header: %s", err)
}
}
for _, itemType := range itemTypes {
for _, itemName := range items[itemType] {
@ -143,40 +136,34 @@ func ListItems(out io.Writer, itemTypes []string, args []string, showType bool,
return nil
}
func InspectItem(name string, itemType string, noMetrics bool) error {
hub, err := cwhub.GetHub()
if err != nil {
return err
}
hubItem := hub.GetItem(itemType, name)
if hubItem == nil {
return fmt.Errorf("can't find '%s' in %s", name, itemType)
}
var b []byte
func InspectItem(hub *cwhub.Hub, item *cwhub.Item, noMetrics bool) error {
var (
b []byte
err error
)
switch csConfig.Cscli.Output {
case "human", "raw":
b, err = yaml.Marshal(*hubItem)
enc := yaml.NewEncoder(os.Stdout)
enc.SetIndent(2)
err = enc.Encode(item)
if err != nil {
return fmt.Errorf("unable to marshal item: %s", err)
return fmt.Errorf("unable to encode item: %s", err)
}
case "json":
b, err = json.MarshalIndent(*hubItem, "", " ")
b, err = json.MarshalIndent(*item, "", " ")
if err != nil {
return fmt.Errorf("unable to marshal item: %s", err)
}
fmt.Printf("%s", string(b))
}
fmt.Printf("%s", string(b))
if noMetrics || csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
return nil
}
fmt.Printf("\nCurrent metrics: \n")
ShowMetrics(hub, hubItem)
ShowMetrics(hub, item)
return nil
}

6
cmd/crowdsec-cli/lapi.go
View file

@ -38,7 +38,7 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
log.Fatalf("parsing api url ('%s'): %s", apiurl, err)
}
hub, err := require.Hub(csConfig)
hub, err := require.Hub(csConfig, nil)
if err != nil {
log.Fatal(err)
}
@ -339,12 +339,12 @@ cscli lapi context detect crowdsecurity/sshd-logs
log.Fatalf("Failed to init expr helpers : %s", err)
}
_, err = require.Hub(csConfig)
hub, err := require.Hub(csConfig, nil)
if err != nil {
log.Fatal(err)
}
csParsers := parser.NewParsers()
csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(csConfig, csParsers); err != nil {
log.Fatalf("unable to load parsers: %s", err)
}

9
cmd/crowdsec-cli/main.go
View file

@ -30,6 +30,9 @@ var OutputColor string
var mergedConfig string
// flagBranch overrides the value in csConfig.Cscli.HubBranch
var flagBranch = ""
func initConfig() {
var err error
if trace_lvl {
@ -65,8 +68,8 @@ func initConfig() {
log.Fatalf("missing 'cscli' configuration in '%s', exiting", ConfigFilePath)
}
if hubBranch == "" && csConfig.Cscli.HubBranch != "" {
hubBranch = csConfig.Cscli.HubBranch
if flagBranch != "" {
csConfig.Cscli.HubBranch = flagBranch
}
if OutputFormat != "" {
@ -197,7 +200,7 @@ It is meant to allow you to manage bans, parsers/scenarios/etc, api and generall
rootCmd.PersistentFlags().BoolVar(&err_lvl, "error", false, "Set logging to error")
rootCmd.PersistentFlags().BoolVar(&trace_lvl, "trace", false, "Set logging to trace")
rootCmd.PersistentFlags().StringVar(&hubBranch, "branch", "", "Override hub branch on github")
rootCmd.PersistentFlags().StringVar(&flagBranch, "branch", "", "Override hub branch on github")
if err := rootCmd.PersistentFlags().MarkHidden("branch"); err != nil {
log.Fatalf("failed to hide flag: %s", err)
}

61
cmd/crowdsec-cli/require/branch.go Normal file
View file

@ -0,0 +1,61 @@
package require
// Set the appropriate hub branch according to config settings and crowdsec version
import (
log "github.com/sirupsen/logrus"
"golang.org/x/mod/semver"
"github.com/crowdsecurity/crowdsec/pkg/cwversion"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
func chooseBranch(cfg *csconfig.Config, logger *log.Logger) string {
if cfg.Cscli.HubBranch != "" {
logger.Debugf("Hub override from config: branch '%s'", cfg.Cscli.HubBranch)
return cfg.Cscli.HubBranch
}
latest, err := cwversion.Latest()
if err != nil {
logger.Warningf("Unable to retrieve latest crowdsec version: %s, using hub branch 'master'", err)
return "master"
}
csVersion := cwversion.VersionStrip()
if csVersion == latest {
logger.Debugf("Latest crowdsec version (%s), using hub branch 'master'", csVersion)
return "master"
}
// if current version is greater than the latest we are in pre-release
if semver.Compare(csVersion, latest) == 1 {
logger.Debugf("Your current crowdsec version seems to be a pre-release (%s), using hub branch 'master'", csVersion)
return "master"
}
if csVersion == "" {
logger.Warning("Crowdsec version is not set, using hub branch 'master'")
return "master"
}
log.Warnf("A new CrowdSec release is available (%s). "+
"Your version is '%s'. Please update it to use new parsers/scenarios/collections.",
latest, csVersion)
return csVersion
}
// HubBranch sets the branch (in cscli config) and returns its value
// It can be "master", or the branch corresponding to the current crowdsec version, or the value overridden in config/flag
func HubBranch(cfg *csconfig.Config) string {
// XXX: we want to be able to suppress logs
// to avoid being too noisy in some commands
logger := log.StandardLogger()
branch := chooseBranch(cfg, logger)
cfg.Cscli.HubBranch = branch
return branch
}

23
cmd/crowdsec-cli/require/require.go
View file

@ -64,12 +64,29 @@ func Notifications(c *csconfig.Config) error {
return nil
}
func Hub(c *csconfig.Config) (*cwhub.Hub, error) {
if c.Hub == nil {
// RemoteHub returns the configuration required to download hub index and items: url, branch, etc.
func RemoteHub(c *csconfig.Config) *cwhub.RemoteHubCfg {
// set branch in config, and log if necessary
branch := HubBranch(c)
remote := &cwhub.RemoteHubCfg {
Branch: branch,
URLTemplate: "https://hub-cdn.crowdsec.net/%s/%s",
IndexPath: ".index.json",
}
return remote
}
// Hub initializes the hub. If a remote configuration is provided, it can be used to download the index and items.
// If no remote parameter is provided, the hub can only be used for local operations.
func Hub(c *csconfig.Config, remote *cwhub.RemoteHubCfg) (*cwhub.Hub, error) {
local := c.Hub
if local == nil {
return nil, fmt.Errorf("you must configure cli before interacting with hub")
}
hub, err := cwhub.InitHub(c.Hub)
hub, err := cwhub.InitHub(local, remote)
if err != nil {
return nil, fmt.Errorf("failed to read Hub index: '%w'. Run 'sudo cscli hub update' to download the index again", err)
}

11
cmd/crowdsec-cli/setup.go
View file

@ -13,6 +13,8 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/setup"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
)
// NewSetupCmd defines the "cscli setup" command.
@ -298,14 +300,17 @@ func runSetupInstallHub(cmd *cobra.Command, args []string) error {
return err
}
branch := chooseHubBranch()
input, err := os.ReadFile(fromFile)
if err != nil {
return fmt.Errorf("while reading file %s: %w", fromFile, err)
}
if err = setup.InstallHubItems(csConfig, input, dryRun, hubURLTemplate, branch); err != nil {
hub, err := require.Hub(csConfig, require.RemoteHub(csConfig))
if err != nil {
return err
}
if err = setup.InstallHubItems(hub, input, dryRun); err != nil {
return err
}

2
cmd/crowdsec-cli/simulation.go
View file

@ -145,7 +145,7 @@ func NewSimulationEnableCmd() *cobra.Command {
Example: `cscli simulation enable`,
DisableAutoGenTag: true,
Run: func(cmd *cobra.Command, args []string) {
hub, err := require.Hub(csConfig)
hub, err := require.Hub(csConfig, nil)
if err != nil {
log.Fatal(err)
}

14
cmd/crowdsec-cli/support.go
View file

@ -128,10 +128,10 @@ func collectOSInfo() ([]byte, error) {
return w.Bytes(), nil
}
func collectHubItems(itemType string) []byte {
func collectHubItems(hub *cwhub.Hub, itemType string) []byte {
out := bytes.NewBuffer(nil)
log.Infof("Collecting %s list", itemType)
if err := ListItems(out, []string{itemType}, []string{}, false, true, false); err != nil {
if err := ListItems(hub, out, []string{itemType}, []string{}, false, true, false); err != nil {
log.Warnf("could not collect %s list: %s", itemType, err)
}
return out.Bytes()
@ -293,7 +293,7 @@ cscli support dump -f /tmp/crowdsec-support.zip
skipAgent = true
}
hub, err := require.Hub(csConfig)
hub, err := require.Hub(csConfig, nil)
if err != nil {
log.Warn("Could not init hub, running on LAPI ? Hub related information will not be collected")
skipHub = true
@ -332,10 +332,10 @@ cscli support dump -f /tmp/crowdsec-support.zip
infos[SUPPORT_CROWDSEC_CONFIG_PATH] = collectCrowdsecConfig()
if !skipHub {
infos[SUPPORT_PARSERS_PATH] = collectHubItems(cwhub.PARSERS)
infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(cwhub.SCENARIOS)
infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(cwhub.POSTOVERFLOWS)
infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(cwhub.COLLECTIONS)
infos[SUPPORT_PARSERS_PATH] = collectHubItems(hub, cwhub.PARSERS)
infos[SUPPORT_SCENARIOS_PATH] = collectHubItems(hub, cwhub.SCENARIOS)
infos[SUPPORT_POSTOVERFLOWS_PATH] = collectHubItems(hub, cwhub.POSTOVERFLOWS)
infos[SUPPORT_COLLECTIONS_PATH] = collectHubItems(hub, cwhub.COLLECTIONS)
}
if !skipDB {

29
cmd/crowdsec-cli/utils_table.go
View file

@ -3,6 +3,7 @@ package main
import (
"fmt"
"io"
"strconv"
"github.com/aquasecurity/table"
"github.com/enescakir/emoji"
@ -10,14 +11,12 @@ import (
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
func listHubItemTable(out io.Writer, title string, itemType string, itemNames []string) {
func listHubItemTable(hub *cwhub.Hub, out io.Writer, title string, itemType string, itemNames []string) {
t := newLightTable(out)
t.SetHeaders("Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path")
t.SetHeaderAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
t.SetAlignment(table.AlignLeft, table.AlignLeft, table.AlignLeft, table.AlignLeft)
hub, _ := cwhub.GetHub()
for itemName := range itemNames {
item := hub.GetItem(itemType, itemNames[itemName])
status, emo := item.Status()
@ -35,11 +34,11 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
t.SetHeaders("Current Count", "Overflows", "Instantiated", "Poured", "Expired")
t.AddRow(
fmt.Sprintf("%d", metrics["curr_count"]),
fmt.Sprintf("%d", metrics["overflow"]),
fmt.Sprintf("%d", metrics["instantiation"]),
fmt.Sprintf("%d", metrics["pour"]),
fmt.Sprintf("%d", metrics["underflow"]),
strconv.Itoa(metrics["curr_count"]),
strconv.Itoa(metrics["overflow"]),
strconv.Itoa(metrics["instantiation"]),
strconv.Itoa(metrics["pour"]),
strconv.Itoa(metrics["underflow"]),
)
renderTableTitle(out, fmt.Sprintf("\n - (Scenario) %s:", itemName))
@ -47,23 +46,25 @@ func scenarioMetricsTable(out io.Writer, itemName string, metrics map[string]int
}
func parserMetricsTable(out io.Writer, itemName string, metrics map[string]map[string]int) {
skip := true
t := newTable(out)
t.SetHeaders("Parsers", "Hits", "Parsed", "Unparsed")
// don't show table if no hits
showTable := false
for source, stats := range metrics {
if stats["hits"] > 0 {
t.AddRow(
source,
fmt.Sprintf("%d", stats["hits"]),
fmt.Sprintf("%d", stats["parsed"]),
fmt.Sprintf("%d", stats["unparsed"]),
strconv.Itoa(stats["hits"]),
strconv.Itoa(stats["parsed"]),
strconv.Itoa(stats["unparsed"]),
)
skip = false
showTable = true
}
}
if !skip {
if showTable {
renderTableTitle(out, fmt.Sprintf("\n - (Parser) %s:", itemName))
t.Render()
}

4
cmd/crowdsec/crowdsec.go
View file

@ -23,13 +23,13 @@ import (
func initCrowdsec(cConfig *csconfig.Config) (*parser.Parsers, error) {
var err error
hub, err := cwhub.InitHub(cConfig.Hub)
hub, err := cwhub.InitHub(cConfig.Hub, nil)
if err != nil {
return nil, fmt.Errorf("while loading hub index: %w", err)
}
// Start loading configs
csParsers := parser.NewParsers()
csParsers := parser.NewParsers(hub)
if csParsers, err = parser.LoadParsers(cConfig, csParsers); err != nil {
return nil, fmt.Errorf("while loading parsers: %w", err)
}

2
pkg/csconfig/config.go
View file

@ -36,7 +36,7 @@ type Config struct {
PluginConfig *PluginCfg `yaml:"plugin_config,omitempty"`
DisableAPI bool `yaml:"-"`
DisableAgent bool `yaml:"-"`
Hub *HubCfg `yaml:"-"`
Hub *LocalHubCfg `yaml:"-"`
}
func NewConfig(configFile string, disableAgent bool, disableAPI bool, quiet bool) (*Config, string, error) {

4
pkg/csconfig/config_test.go
View file

@ -15,10 +15,10 @@ func TestNormalLoad(t *testing.T) {
require.NoError(t, err)
_, _, err = NewConfig("./testdata/xxx.yaml", false, false, false)
assert.EqualError(t, err, "while reading yaml file: open ./testdata/xxx.yaml: "+cstest.FileNotFoundMessage)
require.EqualError(t, err, "while reading yaml file: open ./testdata/xxx.yaml: "+cstest.FileNotFoundMessage)
_, _, err = NewConfig("./testdata/simulation.yaml", false, false, false)
assert.EqualError(t, err, "./testdata/simulation.yaml: yaml: unmarshal errors:\n line 1: field simulation not found in type csconfig.Config")
require.EqualError(t, err, "./testdata/simulation.yaml: yaml: unmarshal errors:\n line 1: field simulation not found in type csconfig.Config")
}
func TestNewCrowdSecConfig(t *testing.T) {

6
pkg/csconfig/hub.go
View file

@ -1,7 +1,7 @@
package csconfig
// HubCfg holds the configuration for a hub
type HubCfg struct {
// LocalHubCfg holds the configuration for a local hub: where to download etc.
type LocalHubCfg struct {
HubIndexFile string // Path to the local index file
HubDir string // Where the hub items are downloaded
InstallDir string // Where to install items
@ -9,7 +9,7 @@ type HubCfg struct {
}
func (c *Config) loadHub() error {
c.Hub = &HubCfg{
c.Hub = &LocalHubCfg{
HubIndexFile: c.ConfigPaths.HubIndexFile,
HubDir: c.ConfigPaths.HubDir,
InstallDir: c.ConfigPaths.ConfigDir,

4
pkg/csconfig/hub_test.go
View file

@ -12,7 +12,7 @@ func TestLoadHub(t *testing.T) {
tests := []struct {
name string
input *Config
expected *HubCfg
expected *LocalHubCfg
expectedErr string
}{
{
@ -25,7 +25,7 @@ func TestLoadHub(t *testing.T) {
HubIndexFile: "./hub/.index.json",
},
},
expected: &HubCfg{
expected: &LocalHubCfg{
HubDir: "./hub",
HubIndexFile: "./hub/.index.json",
InstallDir: "./testdata",

20
pkg/cwhub/cwhub_test.go
View file

@ -32,23 +32,23 @@ func testHub(t *testing.T, update bool) *Hub {
tmpDir, err := os.MkdirTemp("", "testhub")
require.NoError(t, err)
hubCfg := &csconfig.HubCfg{
local := &csconfig.LocalHubCfg{
HubDir: filepath.Join(tmpDir, "crowdsec", "hub"),
HubIndexFile: filepath.Join(tmpDir, "crowdsec", "hub", ".index.json"),
InstallDir: filepath.Join(tmpDir, "crowdsec"),
InstallDataDir: filepath.Join(tmpDir, "installed-data"),
}
err = os.MkdirAll(hubCfg.HubDir, 0o700)
err = os.MkdirAll(local.HubDir, 0o700)
require.NoError(t, err)
err = os.MkdirAll(hubCfg.InstallDir, 0o700)
err = os.MkdirAll(local.InstallDir, 0o700)
require.NoError(t, err)
err = os.MkdirAll(hubCfg.InstallDataDir, 0o700)
err = os.MkdirAll(local.InstallDataDir, 0o700)
require.NoError(t, err)
index, err := os.Create(hubCfg.HubIndexFile)
index, err := os.Create(local.HubIndexFile)
require.NoError(t, err)
_, err = index.WriteString(`{}`)
@ -62,11 +62,17 @@ func testHub(t *testing.T, update bool) *Hub {
var hub *Hub
remote := &RemoteHubCfg{
Branch: "master",
URLTemplate: mockURLTemplate,
IndexPath: ".index.json",
}
if update {
hub, err = InitHubUpdate(hubCfg, mockURLTemplate, "master", ".index.json")
hub, err = InitHubUpdate(local, remote)
require.NoError(t, err)
} else {
hub, err = InitHub(hubCfg)
hub, err = InitHub(local, remote)
require.NoError(t, err)
}

11
pkg/cwhub/dataset_test.go
View file

@ -6,6 +6,7 @@ import (
"github.com/jarcoal/httpmock"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestDownloadFile(t *testing.T) {
@ -29,21 +30,21 @@ func TestDownloadFile(t *testing.T) {
)
err := downloadFile("https://example.com/xx", examplePath)
assert.NoError(t, err)
require.NoError(t, err)
content, err := os.ReadFile(examplePath)
assert.Equal(t, "example content oneoneone", string(content))
assert.NoError(t, err)
require.NoError(t, err)
//bad uri
err = downloadFile("https://zz.com", examplePath)
assert.Error(t, err)
require.Error(t, err)
//404
err = downloadFile("https://example.com/x", examplePath)
assert.Error(t, err)
require.Error(t, err)
//bad target
err = downloadFile("https://example.com/xx", "")
assert.Error(t, err)
require.Error(t, err)
}

20
pkg/cwhub/enable.go
View file

@ -16,7 +16,7 @@ import (
func (h *Hub) EnableItem(target *Item) error {
var err error
parentDir := filepath.Clean(h.cfg.InstallDir + "/" + target.Type + "/" + target.Stage + "/")
parentDir := filepath.Clean(h.local.InstallDir + "/" + target.Type + "/" + target.Stage + "/")
// create directories if needed
if target.Installed {
@ -24,7 +24,7 @@ func (h *Hub) EnableItem(target *Item) error {
return fmt.Errorf("%s is tainted, won't enable unless --force", target.Name)
}
if target.Local {
if target.IsLocal() {
return fmt.Errorf("%s is local, won't enable", target.Name)
}
@ -65,7 +65,7 @@ func (h *Hub) EnableItem(target *Item) error {
}
// hub.ConfigDir + target.RemotePath
srcPath, err := filepath.Abs(h.cfg.HubDir + "/" + target.RemotePath)
srcPath, err := filepath.Abs(h.local.HubDir + "/" + target.RemotePath)
if err != nil {
return fmt.Errorf("while getting source path: %w", err)
}
@ -87,7 +87,7 @@ func (h *Hub) EnableItem(target *Item) error {
}
func (h *Hub) purgeItem(target Item) (Item, error) {
itempath := h.cfg.HubDir + "/" + target.RemotePath
itempath := h.local.HubDir + "/" + target.RemotePath
// disable hub file
if err := os.Remove(itempath); err != nil {
@ -117,7 +117,7 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error {
return nil
}
if target.Local {
if target.IsLocal() {
return fmt.Errorf("%s isn't managed by hub. Please delete manually", target.Name)
}
@ -128,11 +128,11 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error {
// for a COLLECTIONS, disable sub-items
if target.Type == COLLECTIONS {
for _, sub := range target.SubItems() {
val, ok := h.Items[sub.Type][sub.Name]
val, ok := h.Items[sub.Type][sub.Name]
if !ok {
log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, target.Name)
log.Errorf("Referred %s %s in collection %s doesn't exist.", sub.Type, sub.Name, target.Name)
continue
}
}
// check if the item doesn't belong to another collection before removing it
toRemove := true
@ -155,7 +155,7 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error {
}
}
syml, err := filepath.Abs(h.cfg.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName)
syml, err := filepath.Abs(h.local.InstallDir + "/" + target.Type + "/" + target.Stage + "/" + target.FileName)
if err != nil {
return err
}
@ -178,7 +178,7 @@ func (h *Hub) DisableItem(target *Item, purge bool, force bool) error {
return fmt.Errorf("while reading symlink: %w", err)
}
absPath, err := filepath.Abs(h.cfg.HubDir + "/" + target.RemotePath)
absPath, err := filepath.Abs(h.local.HubDir + "/" + target.RemotePath)
if err != nil {
return fmt.Errorf("while abs path: %w", err)
}

4
pkg/cwhub/enable_test.go
View file

@ -10,7 +10,7 @@ import (
func testInstall(hub *Hub, t *testing.T, item Item) {
// Install the parser
err := hub.DownloadLatest(&item, false, false, mockURLTemplate, "master")
err := hub.DownloadLatest(&item, false, false)
require.NoError(t, err, "failed to download %s", item.Name)
_, err = hub.LocalSync()
@ -51,7 +51,7 @@ func testUpdate(hub *Hub, t *testing.T, item Item) {
assert.False(t, hub.Items[item.Type][item.Name].UpToDate, "%s should not be up-to-date", item.Name)
// Update it + check status
err := hub.DownloadLatest(&item, true, true, mockURLTemplate, "master")
err := hub.DownloadLatest(&item, true, true)
require.NoError(t, err, "failed to update %s", item.Name)
// Local sync and check status

207
pkg/cwhub/helpers.go
View file

@ -7,6 +7,7 @@ package cwhub
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"net/http"
@ -19,7 +20,7 @@ import (
)
// InstallItem installs an item from the hub
func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool, hubURLTemplate, branch string) error {
func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly bool) error {
item := h.GetItem(itemType, name)
if item == nil {
return fmt.Errorf("unable to retrieve item: %s", name)
@ -33,7 +34,7 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly
}
}
err := h.DownloadLatest(item, force, true, hubURLTemplate, branch)
err := h.DownloadLatest(item, force, true)
if err != nil {
return fmt.Errorf("while downloading %s: %w", item.Name, err)
}
@ -43,7 +44,7 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly
}
if downloadOnly {
log.Infof("Downloaded %s to %s", item.Name, filepath.Join(h.cfg.HubDir, item.RemotePath))
log.Infof("Downloaded %s to %s", item.Name, filepath.Join(h.local.HubDir, item.RemotePath))
return nil
}
@ -61,131 +62,97 @@ func (h *Hub) InstallItem(name string, itemType string, force bool, downloadOnly
return nil
}
// RemoveItem removes one - or all - the items from the hub
func (h *Hub) RemoveMany(itemType string, name string, all bool, purge bool, forceAction bool) error {
if name != "" {
item := h.GetItem(itemType, name)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", name, itemType)
}
// RemoveItem disables one item, optionally removing the downloaded content
func (h *Hub) RemoveItem(itemType string, name string, purge bool, forceAction bool) (bool, error) {
removed := false
err := h.DisableItem(item, purge, forceAction)
if err != nil {
return fmt.Errorf("unable to disable %s: %w", item.Name, err)
}
if err = h.AddItem(*item); err != nil {
return fmt.Errorf("unable to add %s: %w", item.Name, err)
}
return nil
item := h.GetItem(itemType, name)
if item == nil {
return false, fmt.Errorf("can't find '%s' in %s", name, itemType)
}
if !all {
return fmt.Errorf("removing item: no item specified")
if !item.Downloaded {
log.Infof("removing %s: not downloaded -- no removal required", item.Name)
return false, nil
}
disabled := 0
// remove all
for _, v := range h.GetItemMap(itemType) {
if !v.Installed {
continue
}
err := h.DisableItem(&v, purge, forceAction)
if err != nil {
return fmt.Errorf("unable to disable %s: %w", v.Name, err)
}
if err := h.AddItem(v); err != nil {
return fmt.Errorf("unable to add %s: %w", v.Name, err)
}
disabled++
if !item.Installed && !purge {
log.Infof("removing %s: already uninstalled", item.Name)
return false, nil
}
log.Infof("Disabled %d items", disabled)
if err := h.DisableItem(item, purge, forceAction); err != nil {
return false, fmt.Errorf("unable to disable %s: %w", item.Name, err)
}
return nil
// XXX: should take the value from DisableItem
removed = true
if err := h.AddItem(*item); err != nil {
return false, fmt.Errorf("unable to refresh item state %s: %w", item.Name, err)
}
return removed, nil
}
// UpgradeConfig upgrades an item from the hub
func (h *Hub) UpgradeConfig(itemType string, name string, force bool, hubURLTemplate, branch string) error {
updated := 0
found := false
// UpgradeItem upgrades an item from the hub
func (h *Hub) UpgradeItem(itemType string, name string, force bool) (bool, error) {
updated := false
for _, v := range h.GetItemMap(itemType) {
if name != "" && name != v.Name {
continue
item := h.GetItem(itemType, name)
if item == nil {
return false, fmt.Errorf("can't find '%s' in %s", name, itemType)
}
if !item.Downloaded {
return false, fmt.Errorf("can't upgrade %s: not installed", item.Name)
}
if !item.Installed {
return false, fmt.Errorf("can't upgrade %s: downloaded but not installed", item.Name)
}
if item.UpToDate {
log.Infof("%s: up-to-date", item.Name)
if err := h.DownloadDataIfNeeded(*item, force); err != nil {
return false, fmt.Errorf("%s: download failed: %w", item.Name, err)
}
if !v.Installed {
log.Tracef("skip %s, not installed", v.Name)
continue
}
if !v.Downloaded {
log.Warningf("%s: not downloaded, please install.", v.Name)
continue
}
found = true
if v.UpToDate {
log.Infof("%s: up-to-date", v.Name)
if err := h.DownloadDataIfNeeded(v, force); err != nil {
return fmt.Errorf("%s: download failed: %w", v.Name, err)
}
if !force {
continue
}
}
if err := h.DownloadLatest(&v, force, true, hubURLTemplate, branch); err != nil {
return fmt.Errorf("%s: download failed: %w", v.Name, err)
}
if !v.UpToDate {
if v.Tainted {
log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, v.Name)
} else if v.Local {
log.Infof("%v %s is local", emoji.Prohibited, v.Name)
}
} else {
// this is used while scripting to know if the hub has been upgraded
// and a configuration reload is required
fmt.Printf("updated %s\n", v.Name)
log.Infof("%v %s : updated", emoji.Package, v.Name)
updated++
}
if err := h.AddItem(v); err != nil {
return fmt.Errorf("unable to add %s: %w", v.Name, err)
if !force {
// no upgrade needed
return false, nil
}
}
if !found && name == "" {
log.Infof("No %s installed, nothing to upgrade", itemType)
} else if !found {
log.Errorf("can't find '%s' in %s", name, itemType)
} else if updated == 0 && found {
if name == "" {
log.Infof("All %s are already up-to-date", itemType)
} else {
log.Infof("Item '%s' is up-to-date", name)
}
} else if updated != 0 {
log.Infof("Upgraded %d items", updated)
if err := h.DownloadLatest(item, force, true); err != nil {
return false, fmt.Errorf("%s: download failed: %w", item.Name, err)
}
return nil
if !item.UpToDate {
if item.Tainted {
log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, item.Name)
} else if item.IsLocal() {
log.Infof("%v %s is local", emoji.Prohibited, item.Name)
}
} else {
// a check on stdout is used while scripting to know if the hub has been upgraded
// and a configuration reload is required
// TODO: use a better way to communicate this
fmt.Printf("updated %s\n", item.Name)
log.Infof("%v %s: updated", emoji.Package, item.Name)
updated = true
}
if err := h.AddItem(*item); err != nil {
return false, fmt.Errorf("unable to refresh item state %s: %w", item.Name, err)
}
return updated, nil
}
// DownloadLatest will download the latest version of Item to the tdir directory
func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool, hubURLTemplate, branch string) error {
func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool) error {
var err error
log.Debugf("Downloading %s %s", target.Type, target.Name)
@ -196,7 +163,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool, hubU
return nil
}
return h.DownloadItem(target, overwrite, hubURLTemplate, branch)
return h.DownloadItem(target, overwrite)
}
// collection
@ -216,7 +183,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool, hubU
if sub.Type == COLLECTIONS {
log.Tracef("collection, recurse")
err = h.DownloadLatest(&val, overwrite, updateOnly, hubURLTemplate, branch)
err = h.DownloadLatest(&val, overwrite, updateOnly)
if err != nil {
return fmt.Errorf("while downloading %s: %w", val.Name, err)
}
@ -224,7 +191,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool, hubU
downloaded := val.Downloaded
err = h.DownloadItem(&val, overwrite, hubURLTemplate, branch)
err = h.DownloadItem(&val, overwrite)
if err != nil {
return fmt.Errorf("while downloading %s: %w", val.Name, err)
}
@ -240,7 +207,7 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool, hubU
h.Items[sub.Type][sub.Name] = val
}
err = h.DownloadItem(target, overwrite, hubURLTemplate, branch)
err = h.DownloadItem(target, overwrite)
if err != nil {
return fmt.Errorf("failed to download item: %w", err)
}
@ -248,9 +215,13 @@ func (h *Hub) DownloadLatest(target *Item, overwrite bool, updateOnly bool, hubU
return nil
}
func (h *Hub) DownloadItem(target *Item, overwrite bool, hubURLTemplate, branch string) error {
url := fmt.Sprintf(hubURLTemplate, branch, target.RemotePath)
tdir := h.cfg.HubDir
func (h *Hub) DownloadItem(target *Item, overwrite bool) error {
url, err := h.remote.urlTo(target.RemotePath)
if err != nil {
return fmt.Errorf("failed to build hub item request: %w", err)
}
tdir := h.local.HubDir
// if user didn't --force, don't overwrite local, tainted, up-to-date files
if !overwrite {
@ -291,7 +262,7 @@ func (h *Hub) DownloadItem(target *Item, overwrite bool, hubURLTemplate, branch
return fmt.Errorf("while hashing %s: %w", target.Name, err)
}
meow := fmt.Sprintf("%x", hash.Sum(nil))
meow := hex.EncodeToString(hash.Sum(nil))
if meow != target.Versions[target.Version].Digest {
log.Errorf("Downloaded version doesn't match index, please 'hub update'")
log.Debugf("got %s, expected %s", meow, target.Versions[target.Version].Digest)
@ -347,7 +318,7 @@ func (h *Hub) DownloadItem(target *Item, overwrite bool, hubURLTemplate, branch
target.Tainted = false
target.UpToDate = true
if err = downloadData(h.cfg.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
if err = downloadData(h.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
return fmt.Errorf("while downloading data for %s: %w", target.FileName, err)
}
@ -358,7 +329,7 @@ func (h *Hub) DownloadItem(target *Item, overwrite bool, hubURLTemplate, branch
// DownloadDataIfNeeded downloads the data files for an item
func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error {
itemFilePath := fmt.Sprintf("%s/%s/%s/%s", h.cfg.InstallDir, target.Type, target.Stage, target.FileName)
itemFilePath := fmt.Sprintf("%s/%s/%s/%s", h.local.InstallDir, target.Type, target.Stage, target.FileName)
itemFile, err := os.Open(itemFilePath)
if err != nil {
@ -367,7 +338,7 @@ func (h *Hub) DownloadDataIfNeeded(target Item, force bool) error {
defer itemFile.Close()
if err = downloadData(h.cfg.InstallDataDir, force, itemFile); err != nil {
if err = downloadData(h.local.InstallDataDir, force, itemFile); err != nil {
return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
}

67
pkg/cwhub/helpers_test.go
View file

@ -10,14 +10,14 @@ import (
// Download index, install collection. Add scenario to collection (hub-side), update index, upgrade collection
// We expect the new scenario to be installed
func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
hub := envSetup(t)
// fresh install of collection
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false, mockURLTemplate, "master"))
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false))
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
@ -33,18 +33,25 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
// collection receives an update. It now adds new scenario "crowdsecurity/barfoo_scenario"
pushUpdateToCollectionInHub()
hub, err := InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json")
remote := &RemoteHubCfg{
URLTemplate: mockURLTemplate,
Branch: "master",
IndexPath: ".index.json",
}
hub, err := InitHubUpdate(hub.local, remote)
require.NoError(t, err, "failed to download index: %s", err)
hub = getHubOrFail(t, hub.cfg)
hub = getHubOrFail(t, hub.local, remote)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false, mockURLTemplate, "master")
didUpdate, err := hub.UpgradeItem(COLLECTIONS, "crowdsecurity/test_collection", false)
require.NoError(t, err)
require.True(t, didUpdate)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Downloaded)
@ -53,7 +60,7 @@ func TestUpgradeConfigNewScenarioInCollection(t *testing.T) {
// Install a collection, disable a scenario.
// Upgrade should install should not enable/download the disabled scenario.
func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
hub := envSetup(t)
// fresh install of collection
@ -61,7 +68,7 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false, mockURLTemplate, "master"))
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false))
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
@ -70,10 +77,17 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
err := hub.RemoveMany(SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
didRemove, err := hub.RemoveItem(SCENARIOS, "crowdsecurity/foobar_scenario", false, false)
require.NoError(t, err)
require.True(t, didRemove)
hub = getHubOrFail(t, hub.cfg)
remote := &RemoteHubCfg{
URLTemplate: mockURLTemplate,
Branch: "master",
IndexPath: ".index.json",
}
hub = getHubOrFail(t, hub.local, remote)
// scenario referenced by collection was deleted hence, collection should be tainted
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Tainted)
@ -81,19 +95,20 @@ func TestUpgradeConfigInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].UpToDate)
hub, err = InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json")
hub, err = InitHubUpdate(hub.local, remote)
require.NoError(t, err, "failed to download index: %s", err)
err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false, mockURLTemplate, "master")
didUpdate, err := hub.UpgradeItem(COLLECTIONS, "crowdsecurity/test_collection", false)
require.NoError(t, err)
require.False(t, didUpdate)
hub = getHubOrFail(t, hub.cfg)
hub = getHubOrFail(t, hub.local, remote)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
}
// getHubOrFail refreshes the hub state (load index, sync) and returns the singleton, or fails the test
func getHubOrFail(t *testing.T, hubCfg *csconfig.HubCfg) *Hub {
hub, err := InitHub(hubCfg)
func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCfg) *Hub {
hub, err := InitHub(local, remote)
require.NoError(t, err, "failed to load hub index")
return hub
@ -102,7 +117,7 @@ func getHubOrFail(t *testing.T, hubCfg *csconfig.HubCfg) *Hub {
// Install a collection. Disable a referenced scenario. Publish new version of collection with new scenario
// Upgrade should not enable/download the disabled scenario.
// Upgrade should install and enable the newly added scenario.
func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) {
func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) {
hub := envSetup(t)
// fresh install of collection
@ -110,7 +125,7 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *
require.False(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false, mockURLTemplate, "master"))
require.NoError(t, hub.InstallItem("crowdsecurity/test_collection", COLLECTIONS, false, false))
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Downloaded)
require.True(t, hub.Items[COLLECTIONS]["crowdsecurity/test_collection"].Installed)
@ -119,10 +134,17 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
assertCollectionDepsInstalled(t, "crowdsecurity/test_collection")
err := hub.RemoveMany(SCENARIOS, "crowdsecurity/foobar_scenario", false, false, false)
didRemove, err := hub.RemoveItem(SCENARIOS, "crowdsecurity/foobar_scenario", false, false)
require.NoError(t, err)
require.True(t, didRemove)
hub = getHubOrFail(t, hub.cfg)
remote := &RemoteHubCfg{
URLTemplate: mockURLTemplate,
Branch: "master",
IndexPath: ".index.json",
}
hub = getHubOrFail(t, hub.local, remote)
// scenario referenced by collection was deleted hence, collection should be tainted
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Downloaded) // this fails
@ -136,16 +158,17 @@ func TestUpgradeConfigNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *
// we just removed. Nor should it install the newly added scenario
pushUpdateToCollectionInHub()
hub, err = InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json")
hub, err = InitHubUpdate(hub.local, remote)
require.NoError(t, err, "failed to download index: %s", err)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
hub = getHubOrFail(t, hub.cfg)
hub = getHubOrFail(t, hub.local, remote)
err = hub.UpgradeConfig(COLLECTIONS, "crowdsecurity/test_collection", false, mockURLTemplate, "master")
didUpdate, err := hub.UpgradeItem(COLLECTIONS, "crowdsecurity/test_collection", false)
require.NoError(t, err)
require.True(t, didUpdate)
hub = getHubOrFail(t, hub.cfg)
hub = getHubOrFail(t, hub.local, remote)
require.False(t, hub.Items[SCENARIOS]["crowdsecurity/foobar_scenario"].Installed)
require.True(t, hub.Items[SCENARIOS]["crowdsecurity/barfoo_scenario"].Installed)
}

57
pkg/cwhub/hub.go
View file

@ -17,10 +17,17 @@ import (
// const HubIndexFile = ".index.json"
// Hub represents the runtime status of the hub (parsed items, etc.)
// RemoteHubCfg contains where to find the remote hub, which branch etc.
type RemoteHubCfg struct {
Branch string
URLTemplate string
IndexPath string
}
type Hub struct {
Items HubItems
cfg *csconfig.HubCfg
local *csconfig.LocalHubCfg
remote *RemoteHubCfg
skippedLocal int
skippedTainted int
}
@ -32,6 +39,7 @@ var (
// GetHub returns the hub singleton
// it returns an error if it's not initialized to avoid nil dereference
// XXX: convenience function that we should get rid of at some point
func GetHub() (*Hub, error) {
if theHub == nil {
return nil, fmt.Errorf("hub not initialized")
@ -41,14 +49,14 @@ func GetHub() (*Hub, error) {
}
// InitHub initializes the Hub, syncs the local state and returns the singleton for immediate use
func InitHub(cfg *csconfig.HubCfg) (*Hub, error) {
if cfg == nil {
return nil, fmt.Errorf("no configuration found for hub")
func InitHub(local *csconfig.LocalHubCfg, remote *RemoteHubCfg) (*Hub, error) {
if local == nil {
return nil, fmt.Errorf("no hub configuration found")
}
log.Debugf("loading hub idx %s", cfg.HubIndexFile)
log.Debugf("loading hub idx %s", local.HubIndexFile)
bidx, err := os.ReadFile(cfg.HubIndexFile)
bidx, err := os.ReadFile(local.HubIndexFile)
if err != nil {
return nil, fmt.Errorf("unable to read index file: %w", err)
}
@ -64,8 +72,9 @@ func InitHub(cfg *csconfig.HubCfg) (*Hub, error) {
}
theHub = &Hub{
Items: ret,
cfg: cfg,
Items: ret,
local: local,
remote: remote,
}
_, err = theHub.LocalSync()
@ -78,12 +87,12 @@ func InitHub(cfg *csconfig.HubCfg) (*Hub, error) {
// InitHubUpdate is like InitHub but downloads and updates the index instead of reading from the disk
// It is used to inizialize the hub when there is no index file yet
func InitHubUpdate(cfg *csconfig.HubCfg, urlTemplate, branch, remotePath string) (*Hub, error) {
if cfg == nil {
func InitHubUpdate(local *csconfig.LocalHubCfg, remote *RemoteHubCfg) (*Hub, error) {
if local == nil {
return nil, fmt.Errorf("no configuration found for hub")
}
bidx, err := DownloadIndex(cfg.HubIndexFile, urlTemplate, branch, remotePath)
bidx, err := remote.DownloadIndex(local.HubIndexFile)
if err != nil {
return nil, fmt.Errorf("failed to download index: %w", err)
}
@ -96,8 +105,9 @@ func InitHubUpdate(cfg *csconfig.HubCfg, urlTemplate, branch, remotePath string)
}
theHub = &Hub{
Items: ret,
cfg: cfg,
Items: ret,
local: local,
remote: remote,
}
if _, err := theHub.LocalSync(); err != nil {
@ -107,10 +117,20 @@ func InitHubUpdate(cfg *csconfig.HubCfg, urlTemplate, branch, remotePath string)
return theHub, nil
}
func (r RemoteHubCfg) urlTo(remotePath string) (string, error) {
if fmt.Sprintf(r.URLTemplate, "%s", "%s") != r.URLTemplate {
return "", fmt.Errorf("invalid URL template '%s'", r.URLTemplate)
}
return fmt.Sprintf(r.URLTemplate, r.Branch, remotePath), nil
}
// DownloadIndex downloads the latest version of the index and returns the content
func DownloadIndex(localPath, hubURLTemplate, branch, remotePath string) ([]byte, error) {
url := fmt.Sprintf(hubURLTemplate, branch, remotePath)
log.Debugf("fetching index from branch %s (%s)", branch, url)
func (r RemoteHubCfg) DownloadIndex(localPath string) ([]byte, error) {
url, err := r.urlTo(r.IndexPath)
if err != nil {
return nil, fmt.Errorf("failed to build hub index request: %w", err)
}
req, err := http.NewRequest(http.MethodGet, url, nil)
if err != nil {
@ -212,14 +232,17 @@ func ParseIndex(buff []byte) (HubItems, error) {
// ItemStats returns total counts of the hub items
func (h *Hub) ItemStats() []string {
loaded := ""
for _, itemType := range ItemTypes {
// ensure the order is always the same
if h.Items[itemType] == nil {
continue
}
if len(h.Items[itemType]) == 0 {
continue
}
loaded += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType)
}

39
pkg/cwhub/hub_test.go
View file

@ -13,7 +13,13 @@ import (
func TestInitHubUpdate(t *testing.T) {
hub := envSetup(t)
_, err := InitHubUpdate(hub.cfg, mockURLTemplate, "master", ".index.json")
remote := &RemoteHubCfg {
URLTemplate: mockURLTemplate,
Branch: "master",
IndexPath: ".index.json",
}
_, err := InitHubUpdate(hub.local, remote)
require.NoError(t, err)
_, err = GetHub()
@ -31,23 +37,44 @@ func TestDownloadIndex(t *testing.T) {
os.Remove(tmpIndex.Name())
})
ret, err := DownloadIndex(tmpIndex.Name(), "x", "", "")
cstest.RequireErrorContains(t, err, "failed to build request for hub index: parse ")
hub := envSetup(t)
hub.remote = &RemoteHubCfg {
URLTemplate: "x",
Branch: "",
IndexPath: "",
}
ret, err := hub.remote.DownloadIndex(tmpIndex.Name())
cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'")
fmt.Printf("->%+v", ret)
// bad domain
fmt.Println("Test 'bad domain'")
ret, err = DownloadIndex(tmpIndex.Name(), "https://baddomain/%s/%s", "master", ".index.json")
cstest.RequireErrorContains(t, err, "failed http request for hub index: Get")
hub.remote = &RemoteHubCfg {
URLTemplate: "https://baddomain/%s/%s",
Branch: "master",
IndexPath: ".index.json",
}
ret, err = hub.remote.DownloadIndex(tmpIndex.Name())
// XXX: this is not failing
// cstest.RequireErrorContains(t, err, "failed http request for hub index: Get")
fmt.Printf("->%+v", ret)
// bad target path
fmt.Println("Test 'bad target path'")
ret, err = DownloadIndex("/does/not/exist/index.json", mockURLTemplate, "master", ".index.json")
hub.remote = &RemoteHubCfg {
URLTemplate: mockURLTemplate,
Branch: "master",
IndexPath: ".index.json",
}
ret, err = hub.remote.DownloadIndex("/does/not/exist/index.json")
cstest.RequireErrorContains(t, err, "while opening hub index file: open /does/not/exist/index.json:")
fmt.Printf("->%+v", ret)

58
pkg/cwhub/items.go
View file

@ -1,6 +1,7 @@
package cwhub
import (
"encoding/json"
"fmt"
"github.com/enescakir/emoji"
@ -49,11 +50,10 @@ type Item struct {
LocalPath string `json:"local_path,omitempty" yaml:"local_path,omitempty"` // the local path relative to ${CFG_DIR}
LocalVersion string `json:"local_version,omitempty"`
LocalHash string `json:"local_hash,omitempty"` // the local meow
Installed bool `json:"installed,omitempty"`
Installed bool `json:"installed,omitempty"` // XXX: should we remove omitempty from bool fields?
Downloaded bool `json:"downloaded,omitempty"`
UpToDate bool `json:"up_to_date,omitempty"`
Tainted bool `json:"tainted,omitempty"` // has it been locally modified?
Local bool `json:"local,omitempty"` // if it's a non versioned control one
// if it's a collection, it can have sub items
Parsers []string `json:"parsers,omitempty" yaml:"parsers,omitempty"`
@ -62,35 +62,73 @@ type Item struct {
Collections []string `json:"collections,omitempty" yaml:"collections,omitempty"`
}
type SubItem struct {
Type string
Name string
}
func (i *Item) IsLocal() bool {
return i.Installed && !i.Downloaded
}
// MarshalJSON is used to add the "local" field to the json output
// (i.e. with cscli ... inspect -o json)
// It must not use a pointer receiver
func (i Item) MarshalJSON() ([]byte, error) {
type Alias Item
return json.Marshal(&struct {
Alias
Local bool `json:"local"` // XXX: omitempty?
}{
Alias: Alias(i),
Local: i.IsLocal(),
})
}
// MarshalYAML is used to add the "local" field to the yaml output
// (i.e. with cscli ... inspect -o raw)
// It must not use a pointer receiver
func (i Item) MarshalYAML() (interface{}, error) {
type Alias Item
return &struct {
Alias `yaml:",inline"`
Local bool `yaml:"local"`
}{
Alias: Alias(i),
Local: i.IsLocal(),
}, nil
}
// SubItems returns the list of sub items for a given item (typically a collection)
func (i *Item) SubItems() []SubItem {
sub := make([]SubItem,
len(i.Parsers) +
len(i.PostOverflows) +
len(i.Scenarios) +
len(i.Collections))
len(i.Parsers)+
len(i.PostOverflows)+
len(i.Scenarios)+
len(i.Collections))
n := 0
for _, name := range i.Parsers {
sub[n] = SubItem{Type: PARSERS, Name: name}
n++
}
for _, name := range i.PostOverflows {
sub[n] = SubItem{Type: POSTOVERFLOWS, Name: name}
n++
}
for _, name := range i.Scenarios {
sub[n] = SubItem{Type: SCENARIOS, Name: name}
n++
}
for _, name := range i.Collections {
sub[n] = SubItem{Type: COLLECTIONS, Name: name}
n++
}
return sub
}
@ -106,7 +144,7 @@ func (i *Item) Status() (string, emoji.Emoji) {
}
managed := true
if i.Local {
if i.IsLocal() {
managed = false
status += ",local"
}
@ -115,7 +153,7 @@ func (i *Item) Status() (string, emoji.Emoji) {
if i.Tainted {
warning = true
status += ",tainted"
} else if !i.UpToDate && !i.Local {
} else if !i.UpToDate && !i.IsLocal() {
warning = true
status += ",update-available"
}
@ -201,7 +239,7 @@ func (h *Hub) AddItem(item Item) error {
func (h *Hub) GetInstalledItems(itemType string) ([]Item, error) {
items, ok := h.Items[itemType]
if !ok {
return nil, fmt.Errorf("no %s in hubIdx", itemType)
return nil, fmt.Errorf("no %s in the hub index", itemType)
}
retItems := make([]Item, 0)

8
pkg/cwhub/items_test.go
View file

@ -22,19 +22,19 @@ func TestItemStatus(t *testing.T) {
item.Installed = true
item.UpToDate = false
item.Local = false
item.Tainted = false
item.Downloaded = true
txt, _ := item.Status()
require.Equal(t, "enabled,update-available", txt)
item.Installed = false
item.Installed = true
item.UpToDate = false
item.Local = true
item.Tainted = false
item.Downloaded = false
txt, _ = item.Status()
require.Equal(t, "disabled,local", txt)
require.Equal(t, "enabled,local", txt)
}
stats := hub.ItemStats()

4
pkg/cwhub/leakybucket.go
View file

@ -37,13 +37,15 @@ func itemKey(itemPath string) (string, error) {
return fmt.Sprintf("%s/%s", author, fname), nil
}
// GetItemByPath retrieves the item from hubIdx based on the path. To achieve this it will resolve symlink to find associated hub item.
// GetItemByPath retrieves the item from the hub index based on its path.
// To achieve this it resolves a symlink to find the associated hub item.
func (h *Hub) GetItemByPath(itemType string, itemPath string) (*Item, error) {
itemKey, err := itemKey(itemPath)
if err != nil {
return nil, err
}
// XXX: use GetItem()
m := h.GetItemMap(itemType)
if m == nil {
return nil, fmt.Errorf("item type %s doesn't exist", itemType)

22
pkg/cwhub/sync.go
View file

@ -2,6 +2,7 @@ package cwhub
import (
"crypto/sha256"
"encoding/hex"
"fmt"
"io"
"os"
@ -50,7 +51,7 @@ func getSHA256(filepath string) (string, error) {
return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err)
}
return fmt.Sprintf("%x", h.Sum(nil)), nil
return hex.EncodeToString(h.Sum(nil)), nil
}
type itemFileInfo struct {
@ -64,8 +65,8 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) {
ret := itemFileInfo{}
inhub := false
hubDir := h.cfg.HubDir
installDir := h.cfg.InstallDir
hubDir := h.local.HubDir
installDir := h.local.InstallDir
subs := strings.Split(path, string(os.PathSeparator))
@ -190,7 +191,6 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
Stage: info.stage,
Installed: true,
Type: info.ftype,
Local: true,
LocalPath: path,
UpToDate: true,
FileName: fileName,
@ -229,7 +229,7 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
continue
}
if path == h.cfg.HubDir+"/"+item.RemotePath {
if path == h.local.HubDir+"/"+item.RemotePath {
log.Tracef("marking %s as downloaded", item.Name)
item.Downloaded = true
}
@ -391,11 +391,11 @@ func (h *Hub) SyncDir(dir string) ([]string, error) {
for _, scan := range ItemTypes {
cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
if err != nil {
log.Errorf("failed %s : %s", cpath, err)
log.Errorf("failed %s: %s", cpath, err)
}
// explicit check for non existing directory, avoid spamming log.Debug
if _, err := os.Stat(cpath); os.IsNotExist(err) {
if _, err = os.Stat(cpath); os.IsNotExist(err) {
log.Tracef("directory %s doesn't exist, skipping", cpath)
continue
}
@ -435,14 +435,14 @@ func (h *Hub) LocalSync() ([]string, error) {
h.skippedLocal = 0
h.skippedTainted = 0
warnings, err := h.SyncDir(h.cfg.InstallDir)
warnings, err := h.SyncDir(h.local.InstallDir)
if err != nil {
return warnings, fmt.Errorf("failed to scan %s: %w", h.cfg.InstallDir, err)
return warnings, fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err)
}
_, err = h.SyncDir(h.cfg.HubDir)
_, err = h.SyncDir(h.local.HubDir)
if err != nil {
return warnings, fmt.Errorf("failed to scan %s: %w", h.cfg.HubDir, err)
return warnings, fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err)
}
return warnings, nil

28
pkg/hubtest/hubtest_item.go
View file

@ -39,7 +39,7 @@ type HubTestItem struct {
RuntimeConfigFilePath string
RuntimeProfileFilePath string
RuntimeSimulationFilePath string
RuntimeHubConfig *csconfig.HubCfg
RuntimeHubConfig *csconfig.LocalHubCfg
ResultsPath string
ParserResultFile string
@ -117,7 +117,7 @@ func NewTest(name string, hubTest *HubTest) (*HubTestItem, error) {
ParserResultFile: filepath.Join(resultPath, ParserResultFileName),
ScenarioResultFile: filepath.Join(resultPath, ScenarioResultFileName),
BucketPourResultFile: filepath.Join(resultPath, BucketPourResultFileName),
RuntimeHubConfig: &csconfig.HubCfg{
RuntimeHubConfig: &csconfig.LocalHubCfg{
HubDir: runtimeHubFolder,
HubIndexFile: hubTest.HubIndexFile,
InstallDir: runtimeFolder,
@ -391,7 +391,7 @@ func (t *HubTestItem) InstallHub() error {
}
// load installed hub
hub, err := cwhub.InitHub(t.RuntimeHubConfig)
hub, err := cwhub.InitHub(t.RuntimeHubConfig, nil)
if err != nil {
log.Fatal(err)
}
@ -451,51 +451,51 @@ func (t *HubTestItem) Run() error {
}
// create runtime folder
if err := os.MkdirAll(t.RuntimePath, os.ModePerm); err != nil {
if err = os.MkdirAll(t.RuntimePath, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimePath, err)
}
// create runtime data folder
if err := os.MkdirAll(t.RuntimeDataPath, os.ModePerm); err != nil {
if err = os.MkdirAll(t.RuntimeDataPath, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeDataPath, err)
}
// create runtime hub folder
if err := os.MkdirAll(t.RuntimeHubPath, os.ModePerm); err != nil {
if err = os.MkdirAll(t.RuntimeHubPath, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %+v", t.RuntimeHubPath, err)
}
if err := Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil {
if err = Copy(t.HubIndexFile, filepath.Join(t.RuntimeHubPath, ".index.json")); err != nil {
return fmt.Errorf("unable to copy .index.json file in '%s': %s", filepath.Join(t.RuntimeHubPath, ".index.json"), err)
}
// create results folder
if err := os.MkdirAll(t.ResultsPath, os.ModePerm); err != nil {
if err = os.MkdirAll(t.ResultsPath, os.ModePerm); err != nil {
return fmt.Errorf("unable to create folder '%s': %+v", t.ResultsPath, err)
}
// copy template config file to runtime folder
if err := Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil {
if err = Copy(t.TemplateConfigPath, t.RuntimeConfigFilePath); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateConfigPath, t.RuntimeConfigFilePath, err)
}
// copy template profile file to runtime folder
if err := Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil {
if err = Copy(t.TemplateProfilePath, t.RuntimeProfileFilePath); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateProfilePath, t.RuntimeProfileFilePath, err)
}
// copy template simulation file to runtime folder
if err := Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil {
if err = Copy(t.TemplateSimulationPath, t.RuntimeSimulationFilePath); err != nil {
return fmt.Errorf("unable to copy '%s' to '%s': %v", t.TemplateSimulationPath, t.RuntimeSimulationFilePath, err)
}
// copy template patterns folder to runtime folder
if err := CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil {
if err = CopyDir(crowdsecPatternsFolder, t.RuntimePatternsPath); err != nil {
return fmt.Errorf("unable to copy 'patterns' from '%s' to '%s': %s", crowdsecPatternsFolder, t.RuntimePatternsPath, err)
}
// install the hub in the runtime folder
if err := t.InstallHub(); err != nil {
if err = t.InstallHub(); err != nil {
return fmt.Errorf("unable to install hub in '%s': %s", t.RuntimeHubPath, err)
}
@ -503,7 +503,7 @@ func (t *HubTestItem) Run() error {
logType := t.Config.LogType
dsn := fmt.Sprintf("file://%s", logFile)
if err := os.Chdir(testPath); err != nil {
if err = os.Chdir(testPath); err != nil {
return fmt.Errorf("can't 'cd' to '%s': %s", testPath, err)
}

4
pkg/leakybucket/buckets_test.go
View file

@ -38,12 +38,12 @@ func TestBucket(t *testing.T) {
testdata := "./tests"
hubCfg := &csconfig.HubCfg{
hubCfg := &csconfig.LocalHubCfg{
HubDir: filepath.Join(testdata, "hub"),
HubIndexFile: filepath.Join(testdata, "hub", "index.json"),
}
_, err := cwhub.InitHub(hubCfg)
_, err := cwhub.InitHub(hubCfg, nil)
if err != nil {
t.Fatalf("failed to init hub : %s", err)
}

5
pkg/parser/unix_parser.go
View file

@ -57,7 +57,7 @@ func Init(c map[string]interface{}) (*UnixParserCtx, error) {
// Return new parsers
// nodes and povfwnodes are already initialized in parser.LoadStages
func NewParsers() *Parsers {
func NewParsers(hub *cwhub.Hub) *Parsers {
parsers := &Parsers{
Ctx: &UnixParserCtx{},
Povfwctx: &UnixParserCtx{},
@ -65,9 +65,6 @@ func NewParsers() *Parsers {
PovfwStageFiles: make([]Stagefile, 0),
}
// XXX: handle error
hub, _ := cwhub.GetHub()
for _, itemType := range []string{cwhub.PARSERS, cwhub.POSTOVERFLOWS} {
for _, hubParserItem := range hub.GetItemMap(itemType) {
if hubParserItem.Installed {

16
pkg/setup/install.go
View file

@ -10,7 +10,6 @@ import (
goccyyaml "github.com/goccy/go-yaml"
"gopkg.in/yaml.v3"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)
@ -46,17 +45,12 @@ func decodeSetup(input []byte, fancyErrors bool) (Setup, error) {
}
// InstallHubItems installs the objects recommended in a setup file.
func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool, hubURLTemplate, branch string) error {
func InstallHubItems(hub *cwhub.Hub, input []byte, dryRun bool) error {
setupEnvelope, err := decodeSetup(input, false)
if err != nil {
return err
}
hub, err := cwhub.InitHub(csConfig.Hub)
if err != nil {
return fmt.Errorf("getting hub index: %w", err)
}
for _, setupItem := range setupEnvelope.Setup {
forceAction := false
downloadOnly := false
@ -74,7 +68,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool, hubUR
continue
}
if err := hub.InstallItem(collection, cwhub.COLLECTIONS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil {
if err := hub.InstallItem(collection, cwhub.COLLECTIONS, forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing collection %s: %w", collection, err)
}
}
@ -88,7 +82,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool, hubUR
continue
}
if err := hub.InstallItem(parser, cwhub.PARSERS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil {
if err := hub.InstallItem(parser, cwhub.PARSERS, forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing parser %s: %w", parser, err)
}
}
@ -102,7 +96,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool, hubUR
continue
}
if err := hub.InstallItem(scenario, cwhub.SCENARIOS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil {
if err := hub.InstallItem(scenario, cwhub.SCENARIOS, forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing scenario %s: %w", scenario, err)
}
}
@ -116,7 +110,7 @@ func InstallHubItems(csConfig *csconfig.Config, input []byte, dryRun bool, hubUR
continue
}
if err := hub.InstallItem(postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly, hubURLTemplate, branch); err != nil {
if err := hub.InstallItem(postoverflow, cwhub.POSTOVERFLOWS, forceAction, downloadOnly); err != nil {
return fmt.Errorf("while installing postoverflow %s: %w", postoverflow, err)
}
}

4
test/bats/01_crowdsec.bats
View file

@ -57,12 +57,12 @@ teardown() {
@test "crowdsec - default logging configuration (empty/missing common section)" {
config_set '.common={}'
rune -124 timeout 1s "${CROWDSEC}"
rune -124 timeout 2s "${CROWDSEC}"
refute_output
assert_stderr --partial "Starting processing data"
config_set 'del(.common)'
rune -124 timeout 1s "${CROWDSEC}"
rune -124 timeout 2s "${CROWDSEC}"
refute_output
assert_stderr --partial "Starting processing data"
}

78
test/bats/20_hub.bats Normal file
View file

@ -0,0 +1,78 @@
#!/usr/bin/env bats
# vim: ft=bats:list:ts=8:sts=4:sw=4:et:ai:si:
set -u
setup_file() {
load "../lib/setup_file.sh"
./instance-data load
HUB_DIR=$(config_get '.config_paths.hub_dir')
export HUB_DIR
CONFIG_DIR=$(config_get '.config_paths.config_dir')
export CONFIG_DIR
}
teardown_file() {
load "../lib/teardown_file.sh"
}
setup() {
load "../lib/setup.sh"
load "../lib/bats-file/load.bash"
./instance-data load
hub_purge_all
hub_strip_index
}
teardown() {
:
}
#----------
@test "cscli hub list" {
# no items
rune -0 cscli hub list
assert_output --regexp ".*COLLECTIONS.*PARSERS.*SCENARIOS.*POSTOVERFLOWS.*"
rune -0 cscli hub list -o json
assert_json '{parsers:[],scenarios:[],collections:[],postoverflows:[]}'
rune -0 cscli hub list -o raw
refute_output
# some items
rune -0 cscli parsers install crowdsecurity/whitelists
rune -0 cscli scenarios install crowdsecurity/telnet-bf
rune -0 cscli hub list
assert_output --regexp ".*COLLECTIONS.*PARSERS.*crowdsecurity/whitelists.*SCENARIOS.*crowdsecurity/telnet-bf.*POSTOVERFLOWS.*"
rune -0 cscli hub list -o json
rune -0 jq -e '(.parsers | length == 1) and (.scenarios | length == 1)' <(output)
rune -0 cscli hub list -o raw
assert_output --partial 'crowdsecurity/whitelists'
assert_output --partial 'crowdsecurity/telnet-bf'
refute_output --partial 'crowdsecurity/linux'
# all items
rune -0 cscli hub list -a
assert_output --regexp ".*COLLECTIONS.*crowdsecurity/linux.*PARSERS.*crowdsecurity/whitelists.*SCENARIOS.*crowdsecurity/telnet-bf.*POSTOVERFLOWS.*"
rune -0 cscli hub list -a -o json
rune -0 jq -e '(.parsers | length > 1) and (.scenarios | length > 1)' <(output)
rune -0 cscli hub list -a -o raw
assert_output --partial 'crowdsecurity/whitelists'
assert_output --partial 'crowdsecurity/telnet-bf'
assert_output --partial 'crowdsecurity/linux'
}
@test "cscli hub update" {
#XXX: todo
:
}
@test "cscli hub upgrade" {
#XXX: todo
:
}
@test "cscli hub upgrade --force" {
#XXX: todo
:
}

64
test/bats/20_hub_collections.bats
View file

@ -5,6 +5,7 @@ set -u
setup_file() {
load "../lib/setup_file.sh"
./instance-data load
HUB_DIR=$(config_get '.config_paths.hub_dir')
export HUB_DIR
CONFIG_DIR=$(config_get '.config_paths.config_dir')
@ -19,9 +20,8 @@ setup() {
load "../lib/setup.sh"
load "../lib/bats-file/load.bash"
./instance-data load
hub_uninstall_all
hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)')
echo "$hub_min" >"$HUB_DIR/.index.json"
hub_purge_all
hub_strip_index
}
teardown() {
@ -75,6 +75,8 @@ teardown() {
rune -0 cscli collections list -o raw -a
rune -0 grep -vc 'name,status,version,description' <(output)
assert_output "$expected"
# XXX: check alphabetical order in human, json, raw
}
@ -159,6 +161,7 @@ teardown() {
@test "cscli collections inspect [collection]..." {
rune -1 cscli collections inspect
assert_stderr --partial 'requires at least 1 arg(s), only received 0'
# required for metrics
./instance-crowdsec start
rune -1 cscli collections inspect blahblah/blahblah
@ -220,12 +223,17 @@ teardown() {
@test "cscli collections remove [collection]..." {
rune -1 cscli collections remove
assert_stderr --partial "specify at least one collection to remove or '--all'"
rune -1 cscli collections remove blahblah/blahblah
assert_stderr --partial "can't find 'blahblah/blahblah' in collections"
# XXX: we can however remove a real item if it's not installed, or already removed
rune -0 cscli collections remove crowdsecurity/sshd
assert_stderr --partial 'removing crowdsecurity/sshd: not downloaded -- no removal required'
rune -0 cscli collections install crowdsecurity/sshd --download-only
rune -0 cscli collections remove crowdsecurity/sshd
assert_stderr --partial 'removing crowdsecurity/sshd: already uninstalled'
rune -0 cscli collections remove crowdsecurity/sshd --purge
assert_stderr --partial 'Removed source file [crowdsecurity/sshd]'
# install, then remove, check files
rune -0 cscli collections install crowdsecurity/sshd
@ -258,8 +266,8 @@ teardown() {
assert_output "0"
}
@test "cscli parsers remove [parser]... --force" {
# remove a parser that belongs to a collection
@test "cscli collections remove [collections]... --force" {
# remove a collections that belongs to a collection
rune -0 cscli collections install crowdsecurity/linux
rune -0 cscli collections remove crowdsecurity/sshd
assert_stderr --partial "crowdsecurity/sshd belongs to collections: [crowdsecurity/linux]"
@ -269,26 +277,24 @@ teardown() {
@test "cscli collections upgrade [collection]..." {
rune -1 cscli collections upgrade
assert_stderr --partial "specify at least one collection to upgrade or '--all'"
# XXX: should this return 1 instead of log.Error?
rune -0 cscli collections upgrade blahblah/blahblah
rune -1 cscli collections upgrade blahblah/blahblah
assert_stderr --partial "can't find 'blahblah/blahblah' in collections"
rune -1 cscli collections upgrade crowdsecurity/exim
assert_stderr --partial "can't upgrade crowdsecurity/exim: not installed"
rune -0 cscli collections install crowdsecurity/exim --download-only
rune -1 cscli collections upgrade crowdsecurity/exim
assert_stderr --partial "can't upgrade crowdsecurity/exim: downloaded but not installed"
# XXX: same message if the item exists but is not installed, this is confusing
rune -0 cscli collections upgrade crowdsecurity/sshd
assert_stderr --partial "can't find 'crowdsecurity/sshd' in collections"
# hash of the string "v0.0"
sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d"
# hash of an empty file
sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
# add version 0.0 to the hub
new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {collections:{"crowdsecurity/sshd":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}')
# add version 0.0 to all collections
new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$HUB_DIR/.index.json" '.collections |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})')
echo "$new_hub" >"$HUB_DIR/.index.json"
rune -0 cscli collections install crowdsecurity/sshd
# bring the file to v0.0
truncate -s 0 "$CONFIG_DIR/collections/sshd.yaml"
echo "v0.0" > "$CONFIG_DIR/collections/sshd.yaml"
rune -0 cscli collections inspect crowdsecurity/sshd -o json
rune -0 jq -e '.local_version=="0.0"' <(output)
@ -312,16 +318,20 @@ teardown() {
# multiple items
rune -0 cscli collections install crowdsecurity/smb
echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml"
echo "dirty" >"$CONFIG_DIR/collections/smb.yaml"
echo "v0.0" >"$CONFIG_DIR/collections/sshd.yaml"
echo "v0.0" >"$CONFIG_DIR/collections/smb.yaml"
rune -0 cscli collections list -o json
rune -0 jq -e '[.collections[].local_version]==["?","?"]' <(output)
rune -0 jq -e '[.collections[].local_version]==["0.0","0.0"]' <(output)
rune -0 cscli collections upgrade crowdsecurity/sshd crowdsecurity/smb
rune -0 jq -e '[.collections[].local_version]==[.collections[].version]' <(output)
rune -0 cscli collections list -o json
rune -0 jq -e 'any(.collections[].local_version; .=="0.0") | not' <(output)
# upgrade all
echo "dirty" >"$CONFIG_DIR/collections/sshd.yaml"
echo "dirty" >"$CONFIG_DIR/collections/smb.yaml"
echo "v0.0" >"$CONFIG_DIR/collections/sshd.yaml"
echo "v0.0" >"$CONFIG_DIR/collections/smb.yaml"
rune -0 cscli collections list -o json
rune -0 jq -e '[.collections[].local_version]==["0.0","0.0"]' <(output)
rune -0 cscli collections upgrade --all
rune -0 jq -e '[.collections[].local_version]==[.collections[].version]' <(output)
rune -0 cscli collections list -o json
rune -0 jq -e 'any(.collections[].local_version; .=="0.0") | not' <(output)
}

6
test/bats/20_hub_collections_dep.bats
View file

@ -5,6 +5,7 @@ set -u
setup_file() {
load "../lib/setup_file.sh"
./instance-data load
HUB_DIR=$(config_get '.config_paths.hub_dir')
export HUB_DIR
CONFIG_DIR=$(config_get '.config_paths.config_dir')
@ -19,9 +20,8 @@ setup() {
load "../lib/setup.sh"
load "../lib/bats-file/load.bash"
./instance-data load
hub_uninstall_all
hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)')
echo "$hub_min" >"$HUB_DIR/.index.json"
hub_purge_all
hub_strip_index
}
teardown() {

153
test/bats/20_hub_parsers.bats
View file

@ -5,6 +5,7 @@ set -u
setup_file() {
load "../lib/setup_file.sh"
./instance-data load
HUB_DIR=$(config_get '.config_paths.hub_dir')
export HUB_DIR
CONFIG_DIR=$(config_get '.config_paths.config_dir')
@ -19,11 +20,8 @@ setup() {
load "../lib/setup.sh"
load "../lib/bats-file/load.bash"
./instance-data load
hub_uninstall_all
# XXX: remove all "content" fields from the index, to make sure
# XXX: we don't rely on it in any way
hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)')
echo "$hub_min" >"$HUB_DIR/.index.json"
hub_purge_all
hub_strip_index
}
teardown() {
@ -77,6 +75,8 @@ teardown() {
rune -0 cscli parsers list -o raw -a
rune -0 grep -vc 'name,status,version,description' <(output)
assert_output "$expected"
# XXX: check alphabetical order in human, json, raw
}
@test "cscli parsers list [parser]..." {
@ -164,6 +164,7 @@ teardown() {
@test "cscli parsers inspect [parser]..." {
rune -1 cscli parsers inspect
assert_stderr --partial 'requires at least 1 arg(s), only received 0'
# required for metrics
./instance-crowdsec start
rune -1 cscli parsers inspect blahblah/blahblah
@ -224,18 +225,20 @@ teardown() {
assert_output "0"
}
@test "cscli parsers remove [parser]..." {
@test "foo cscli parsers remove [parser]..." {
rune -1 cscli parsers remove
assert_stderr --partial "specify at least one parser to remove or '--all'"
rune -1 cscli parsers remove blahblah/blahblah
assert_stderr --partial "can't find 'blahblah/blahblah' in parsers"
# XXX: we can however remove a real item if it's not installed, or already removed
rune -0 cscli parsers remove crowdsecurity/whitelists
assert_stderr --partial 'removing crowdsecurity/whitelists: not downloaded -- no removal required'
# XXX: have the --force ignore uninstalled items
# XXX: maybe also with --purge
rune -0 cscli parsers install crowdsecurity/whitelists --download-only
rune -0 cscli parsers remove crowdsecurity/whitelists
assert_stderr --partial 'removing crowdsecurity/whitelists: already uninstalled'
rune -0 cscli parsers remove crowdsecurity/whitelists --purge
assert_stderr --partial 'Removed source file [crowdsecurity/whitelists]'
# install, then remove, check files
rune -0 cscli parsers install crowdsecurity/whitelists
@ -279,26 +282,24 @@ teardown() {
@test "cscli parsers upgrade [parser]..." {
rune -1 cscli parsers upgrade
assert_stderr --partial "specify at least one parser to upgrade or '--all'"
# XXX: should this return 1 instead of log.Error?
rune -0 cscli parsers upgrade blahblah/blahblah
rune -1 cscli parsers upgrade blahblah/blahblah
assert_stderr --partial "can't find 'blahblah/blahblah' in parsers"
rune -1 cscli parsers upgrade crowdsecurity/pam-logs
assert_stderr --partial "can't upgrade crowdsecurity/pam-logs: not installed"
rune -0 cscli parsers install crowdsecurity/pam-logs --download-only
rune -1 cscli parsers upgrade crowdsecurity/pam-logs
assert_stderr --partial "can't upgrade crowdsecurity/pam-logs: downloaded but not installed"
# XXX: same message if the item exists but is not installed, this is confusing
rune -0 cscli parsers upgrade crowdsecurity/whitelists
assert_stderr --partial "can't find 'crowdsecurity/whitelists' in parsers"
# hash of the string "v0.0"
sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d"
# hash of an empty file
sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
# add version 0.0 to the hub
new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {parsers:{"crowdsecurity/whitelists":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}')
# add version 0.0 to all parsers
new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$HUB_DIR/.index.json" '.parsers |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})')
echo "$new_hub" >"$HUB_DIR/.index.json"
rune -0 cscli parsers install crowdsecurity/whitelists
# bring the file to v0.0
truncate -s 0 "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
echo "v0.0" > "$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
rune -0 cscli parsers inspect crowdsecurity/whitelists -o json
rune -0 jq -e '.local_version=="0.0"' <(output)
@ -322,104 +323,20 @@ teardown() {
# multiple items
rune -0 cscli parsers install crowdsecurity/windows-auth
echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
echo "dirty" >"$CONFIG_DIR/parsers/s01-parse/windows-auth.yaml"
echo "v0.0" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
echo "v0.0" >"$CONFIG_DIR/parsers/s01-parse/windows-auth.yaml"
rune -0 cscli parsers list -o json
rune -0 jq -e '[.parsers[].local_version]==["?","?"]' <(output)
rune -0 jq -e '[.parsers[].local_version]==["0.0","0.0"]' <(output)
rune -0 cscli parsers upgrade crowdsecurity/whitelists crowdsecurity/windows-auth
rune -0 jq -e '[.parsers[].local_version]==[.parsers[].version]' <(output)
rune -0 cscli parsers list -o json
rune -0 jq -e 'any(.parsers[].local_version; .=="0.0") | not' <(output)
# upgrade all
echo "dirty" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
echo "dirty" >"$CONFIG_DIR/parsers/s01-parse/windows-auth.yaml"
echo "v0.0" >"$CONFIG_DIR/parsers/s02-enrich/whitelists.yaml"
echo "v0.0" >"$CONFIG_DIR/parsers/s01-parse/windows-auth.yaml"
rune -0 cscli parsers list -o json
rune -0 jq -e '[.parsers[].local_version]==["0.0","0.0"]' <(output)
rune -0 cscli parsers upgrade --all
rune -0 jq -e '[.parsers[].local_version]==[.parsers[].version]' <(output)
rune -0 cscli parsers list -o json
rune -0 jq -e 'any(.parsers[].local_version; .=="0.0") | not' <(output)
}
#@test "must use --force to remove a collection that belongs to another, which becomes tainted" {
# # we expect no error since we may have multiple collections, some removed and some not
# rune -0 cscli collections remove crowdsecurity/sshd
# assert_stderr --partial "crowdsecurity/sshd belongs to other collections"
# assert_stderr --partial "[crowdsecurity/linux]"
#
# rune -0 cscli collections remove crowdsecurity/sshd --force
# assert_stderr --partial "Removed symlink [crowdsecurity/sshd]"
# rune -0 cscli collections inspect crowdsecurity/linux -o json
# rune -0 jq -r '.tainted' <(output)
# assert_output "true"
#}
#
#@test "can remove a collection" {
# rune -0 cscli collections remove crowdsecurity/linux
# assert_stderr --partial "Removed"
# assert_stderr --regexp ".*for the new configuration to be effective."
# rune -0 cscli collections inspect crowdsecurity/linux -o human --no-metrics
# assert_line 'installed: false'
#}
#
#@test "collections delete is an alias for collections remove" {
# rune -0 cscli collections delete crowdsecurity/linux
# assert_stderr --partial "Removed"
# assert_stderr --regexp ".*for the new configuration to be effective."
#}
#
#@test "removing a collection that does not exist is noop" {
# rune -0 cscli collections remove crowdsecurity/apache2
# refute_stderr --partial "Removed"
# assert_stderr --regexp ".*for the new configuration to be effective."
#}
#
#@test "can remove a removed collection" {
# rune -0 cscli collections install crowdsecurity/mysql
# rune -0 cscli collections remove crowdsecurity/mysql
# assert_stderr --partial "Removed"
# rune -0 cscli collections remove crowdsecurity/mysql
# refute_stderr --partial "Removed"
#}
#
#@test "can remove all collections" {
# # we may have this too, from package installs
# rune cscli parsers delete crowdsecurity/whitelists
# rune -0 cscli collections remove --all
# assert_stderr --partial "Removed symlink [crowdsecurity/sshd]"
# assert_stderr --partial "Removed symlink [crowdsecurity/linux]"
# rune -0 cscli hub list -o json
# assert_json '{collections:[],parsers:[],postoverflows:[],scenarios:[]}'
# rune -0 cscli collections remove --all
# assert_stderr --partial 'Disabled 0 items'
#}
#
#@test "a taint bubbles up to the top collection" {
# coll=crowdsecurity/nginx
# subcoll=crowdsecurity/base-http-scenarios
# scenario=crowdsecurity/http-crawl-non_statics
#
# # install a collection with dependencies
# rune -0 cscli collections install "$coll"
#
# # the collection, subcollection and scenario are installed and not tainted
# # we have to default to false because tainted is (as of 1.4.6) returned
# # only when true
# rune -0 cscli collections inspect "$coll" -o json
# rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output)
# rune -0 cscli collections inspect "$subcoll" -o json
# rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output)
# rune -0 cscli scenarios inspect "$scenario" -o json
# rune -0 jq -e '(.installed,.tainted|false)==(true,false)' <(output)
#
# # we taint the scenario
# HUB_DIR=$(config_get '.config_paths.hub_dir')
# yq e '.description="I am tainted"' -i "$HUB_DIR/scenarios/$scenario.yaml"
#
# # the collection, subcollection and scenario are now tainted
# rune -0 cscli scenarios inspect "$scenario" -o json
# rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output)
# rune -0 cscli collections inspect "$subcoll" -o json
# rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output)
# rune -0 cscli collections inspect "$coll" -o json
# rune -0 jq -e '(.installed,.tainted)==(true,true)' <(output)
#}
#
## TODO test download-only

64
test/bats/20_hub_postoverflows.bats
View file

@ -5,6 +5,7 @@ set -u
setup_file() {
load "../lib/setup_file.sh"
./instance-data load
HUB_DIR=$(config_get '.config_paths.hub_dir')
export HUB_DIR
CONFIG_DIR=$(config_get '.config_paths.config_dir')
@ -19,9 +20,8 @@ setup() {
load "../lib/setup.sh"
load "../lib/bats-file/load.bash"
./instance-data load
hub_uninstall_all
hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)')
echo "$hub_min" >"$HUB_DIR/.index.json"
hub_purge_all
hub_strip_index
}
teardown() {
@ -75,6 +75,8 @@ teardown() {
rune -0 cscli postoverflows list -o raw -a
rune -0 grep -vc 'name,status,version,description' <(output)
assert_output "$expected"
# XXX: check alphabetical order in human, json, raw
}
@ -159,6 +161,7 @@ teardown() {
@test "cscli postoverflows inspect [scenario]..." {
rune -1 cscli postoverflows inspect
assert_stderr --partial 'requires at least 1 arg(s), only received 0'
# required for metrics
./instance-crowdsec start
rune -1 cscli postoverflows inspect blahblah/blahblah
@ -222,12 +225,17 @@ teardown() {
@test "cscli postoverflows remove [postoverflow]..." {
rune -1 cscli postoverflows remove
assert_stderr --partial "specify at least one postoverflow to remove or '--all'"
rune -1 cscli postoverflows remove blahblah/blahblah
assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows"
# XXX: we can however remove a real item if it's not installed, or already removed
rune -0 cscli postoverflows remove crowdsecurity/rdns
assert_stderr --partial 'removing crowdsecurity/rdns: not downloaded -- no removal required'
rune -0 cscli postoverflows install crowdsecurity/rdns --download-only
rune -0 cscli postoverflows remove crowdsecurity/rdns
assert_stderr --partial 'removing crowdsecurity/rdns: already uninstalled'
rune -0 cscli postoverflows remove crowdsecurity/rdns --purge
assert_stderr --partial 'Removed source file [crowdsecurity/rdns]'
# install, then remove, check files
rune -0 cscli postoverflows install crowdsecurity/rdns
@ -260,8 +268,8 @@ teardown() {
assert_output "0"
}
@test "cscli postoverflows remove [parser]... --force" {
# remove a parser that belongs to a collection
@test "cscli postoverflows remove [postoverflow]... --force" {
# remove a postoverflow that belongs to a collection
rune -0 cscli collections install crowdsecurity/auditd
rune -0 cscli postoverflows remove crowdsecurity/auditd-whitelisted-process
assert_stderr --partial "crowdsecurity/auditd-whitelisted-process belongs to collections: [crowdsecurity/auditd]"
@ -271,26 +279,24 @@ teardown() {
@test "cscli postoverflows upgrade [postoverflow]..." {
rune -1 cscli postoverflows upgrade
assert_stderr --partial "specify at least one postoverflow to upgrade or '--all'"
# XXX: should this return 1 instead of log.Error?
rune -0 cscli postoverflows upgrade blahblah/blahblah
rune -1 cscli postoverflows upgrade blahblah/blahblah
assert_stderr --partial "can't find 'blahblah/blahblah' in postoverflows"
rune -1 cscli postoverflows upgrade crowdsecurity/discord-crawler-whitelist
assert_stderr --partial "can't upgrade crowdsecurity/discord-crawler-whitelist: not installed"
rune -0 cscli postoverflows install crowdsecurity/discord-crawler-whitelist --download-only
rune -1 cscli postoverflows upgrade crowdsecurity/discord-crawler-whitelist
assert_stderr --partial "can't upgrade crowdsecurity/discord-crawler-whitelist: downloaded but not installed"
# XXX: same message if the item exists but is not installed, this is confusing
rune -0 cscli postoverflows upgrade crowdsecurity/rdns
assert_stderr --partial "can't find 'crowdsecurity/rdns' in postoverflows"
# hash of the string "v0.0"
sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d"
# hash of an empty file
sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
# add version 0.0 to the hub
new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {postoverflows:{"crowdsecurity/rdns":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}')
# add version 0.0 to all postoverflows
new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$HUB_DIR/.index.json" '.postoverflows |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})')
echo "$new_hub" >"$HUB_DIR/.index.json"
rune -0 cscli postoverflows install crowdsecurity/rdns
# bring the file to v0.0
truncate -s 0 "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
echo "v0.0" > "$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
rune -0 cscli postoverflows inspect crowdsecurity/rdns -o json
rune -0 jq -e '.local_version=="0.0"' <(output)
@ -314,16 +320,20 @@ teardown() {
# multiple items
rune -0 cscli postoverflows install crowdsecurity/cdn-whitelist
echo "dirty" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
echo "dirty" >"$CONFIG_DIR/postoverflows/s01-whitelist/cdn-whitelist.yaml"
echo "v0.0" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
echo "v0.0" >"$CONFIG_DIR/postoverflows/s01-whitelist/cdn-whitelist.yaml"
rune -0 cscli postoverflows list -o json
rune -0 jq -e '[.postoverflows[].local_version]==["?","?"]' <(output)
rune -0 jq -e '[.postoverflows[].local_version]==["0.0","0.0"]' <(output)
rune -0 cscli postoverflows upgrade crowdsecurity/rdns crowdsecurity/cdn-whitelist
rune -0 jq -e '[.postoverflows[].local_version]==[.postoverflows[].version]' <(output)
rune -0 cscli postoverflows list -o json
rune -0 jq -e 'any(.postoverflows[].local_version; .=="0.0") | not' <(output)
# upgrade all
echo "dirty" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
echo "dirty" >"$CONFIG_DIR/postoverflows/s01-whitelist/cdn-whitelist.yaml"
echo "v0.0" >"$CONFIG_DIR/postoverflows/s00-enrich/rdns.yaml"
echo "v0.0" >"$CONFIG_DIR/postoverflows/s01-whitelist/cdn-whitelist.yaml"
rune -0 cscli postoverflows list -o json
rune -0 jq -e '[.postoverflows[].local_version]==["0.0","0.0"]' <(output)
rune -0 cscli postoverflows upgrade --all
rune -0 jq -e '[.postoverflows[].local_version]==[.postoverflows[].version]' <(output)
rune -0 cscli postoverflows list -o json
rune -0 jq -e 'any(.postoverflows[].local_version; .=="0.0") | not' <(output)
}

60
test/bats/20_hub_scenarios.bats
View file

@ -5,6 +5,7 @@ set -u
setup_file() {
load "../lib/setup_file.sh"
./instance-data load
HUB_DIR=$(config_get '.config_paths.hub_dir')
export HUB_DIR
CONFIG_DIR=$(config_get '.config_paths.config_dir')
@ -19,9 +20,8 @@ setup() {
load "../lib/setup.sh"
load "../lib/bats-file/load.bash"
./instance-data load
hub_uninstall_all
hub_min=$(jq <"$HUB_DIR/.index.json" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)')
echo "$hub_min" >"$HUB_DIR/.index.json"
hub_purge_all
hub_strip_index
}
teardown() {
@ -75,6 +75,8 @@ teardown() {
rune -0 cscli scenarios list -o raw -a
rune -0 grep -vc 'name,status,version,description' <(output)
assert_output "$expected"
# XXX: check alphabetical order in human, json, raw
}
@test "cscli scenarios list [scenario]..." {
@ -161,6 +163,7 @@ teardown() {
@test "cscli scenarios inspect [scenario]..." {
rune -1 cscli scenarios inspect
assert_stderr --partial 'requires at least 1 arg(s), only received 0'
# required for metrics
./instance-crowdsec start
rune -1 cscli scenarios inspect blahblah/blahblah
@ -222,12 +225,17 @@ teardown() {
@test "cscli scenarios remove [scenario]..." {
rune -1 cscli scenarios remove
assert_stderr --partial "specify at least one scenario to remove or '--all'"
rune -1 cscli scenarios remove blahblah/blahblah
assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios"
# XXX: we can however remove a real item if it's not installed, or already removed
rune -0 cscli scenarios remove crowdsecurity/ssh-bf
assert_stderr --partial 'removing crowdsecurity/ssh-bf: not downloaded -- no removal required'
rune -0 cscli scenarios install crowdsecurity/ssh-bf --download-only
rune -0 cscli scenarios remove crowdsecurity/ssh-bf
assert_stderr --partial 'removing crowdsecurity/ssh-bf: already uninstalled'
rune -0 cscli scenarios remove crowdsecurity/ssh-bf --purge
assert_stderr --partial 'Removed source file [crowdsecurity/ssh-bf]'
# install, then remove, check files
rune -0 cscli scenarios install crowdsecurity/ssh-bf
@ -271,26 +279,24 @@ teardown() {
@test "cscli scenarios upgrade [scenario]..." {
rune -1 cscli scenarios upgrade
assert_stderr --partial "specify at least one scenario to upgrade or '--all'"
# XXX: should this return 1 instead of log.Error?
rune -0 cscli scenarios upgrade blahblah/blahblah
rune -1 cscli scenarios upgrade blahblah/blahblah
assert_stderr --partial "can't find 'blahblah/blahblah' in scenarios"
rune -1 cscli scenarios upgrade crowdsecurity/vsftpd-bf
assert_stderr --partial "can't upgrade crowdsecurity/vsftpd-bf: not installed"
rune -0 cscli scenarios install crowdsecurity/vsftpd-bf --download-only
rune -1 cscli scenarios upgrade crowdsecurity/vsftpd-bf
assert_stderr --partial "can't upgrade crowdsecurity/vsftpd-bf: downloaded but not installed"
# XXX: same message if the item exists but is not installed, this is confusing
rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf
assert_stderr --partial "can't find 'crowdsecurity/ssh-bf' in scenarios"
# hash of the string "v0.0"
sha256_0_0="dfebecf42784a31aa3d009dbcec0c657154a034b45f49cf22a895373f6dbf63d"
# hash of an empty file
sha256_empty="e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
# add version 0.0 to the hub
new_hub=$(jq --arg DIGEST "$sha256_empty" <"$HUB_DIR/.index.json" '. * {scenarios:{"crowdsecurity/ssh-bf":{"versions":{"0.0":{"digest":$DIGEST, "deprecated": false}}}}}')
# add version 0.0 to all scenarios
new_hub=$(jq --arg DIGEST "$sha256_0_0" <"$HUB_DIR/.index.json" '.scenarios |= with_entries(.value.versions["0.0"] = {"digest": $DIGEST, "deprecated": false})')
echo "$new_hub" >"$HUB_DIR/.index.json"
rune -0 cscli scenarios install crowdsecurity/ssh-bf
# bring the file to v0.0
truncate -s 0 "$CONFIG_DIR/scenarios/ssh-bf.yaml"
echo "v0.0" > "$CONFIG_DIR/scenarios/ssh-bf.yaml"
rune -0 cscli scenarios inspect crowdsecurity/ssh-bf -o json
rune -0 jq -e '.local_version=="0.0"' <(output)
@ -314,16 +320,20 @@ teardown() {
# multiple items
rune -0 cscli scenarios install crowdsecurity/telnet-bf
echo "dirty" >"$CONFIG_DIR/scenarios/ssh-bf.yaml"
echo "dirty" >"$CONFIG_DIR/scenarios/telnet-bf.yaml"
echo "v0.0" >"$CONFIG_DIR/scenarios/ssh-bf.yaml"
echo "v0.0" >"$CONFIG_DIR/scenarios/telnet-bf.yaml"
rune -0 cscli scenarios list -o json
rune -0 jq -e '[.scenarios[].local_version]==["?","?"]' <(output)
rune -0 jq -e '[.scenarios[].local_version]==["0.0","0.0"]' <(output)
rune -0 cscli scenarios upgrade crowdsecurity/ssh-bf crowdsecurity/telnet-bf
rune -0 jq -e '[.scenarios[].local_version]==[.scenarios[].version]' <(output)
rune -0 cscli scenarios list -o json
rune -0 jq -e 'any(.scenarios[].local_version; .=="0.0") | not' <(output)
# upgrade all
echo "dirty" >"$CONFIG_DIR/scenarios/ssh-bf.yaml"
echo "dirty" >"$CONFIG_DIR/scenarios/telnet-bf.yaml"
echo "v0.0" >"$CONFIG_DIR/scenarios/ssh-bf.yaml"
echo "v0.0" >"$CONFIG_DIR/scenarios/telnet-bf.yaml"
rune -0 cscli scenarios list -o json
rune -0 jq -e '[.scenarios[].local_version]==["0.0","0.0"]' <(output)
rune -0 cscli scenarios upgrade --all
rune -0 jq -e '[.scenarios[].local_version]==[.scenarios[].version]' <(output)
rune -0 cscli scenarios list -o json
rune -0 jq -e 'any(.scenarios[].local_version; .=="0.0") | not' <(output)
}

22
test/lib/setup_file.sh
View file

@ -238,11 +238,29 @@ assert_stderr_line() {
}
export -f assert_stderr_line
hub_uninstall_all() {
# remove all installed items and data
hub_purge_all() {
local CONFIG_DIR
CONFIG_DIR=$(dirname "$CONFIG_YAML")
rm -rf "$CONFIG_DIR"/collections/* "$CONFIG_DIR"/parsers/*/* "$CONFIG_DIR"/scenarios/* "$CONFIG_DIR"/postoverflows/*
rm -rf "$CONFIG_DIR"/hub/collections/* "$CONFIG_DIR"/hub/parsers/*/* "$CONFIG_DIR"/hub/scenarios/* "$CONFIG_DIR"/hub/postoverflows/*
local DATA_DIR
DATA_DIR=$(config_get .config_paths.data_dir)
# should remove everything except the db (find $DATA_DIR -not -name "crowdsec.db*" -delete),
# but don't play with fire if there is a misconfiguration
rm -rfv "$DATA_DIR"/GeoLite*
}
export -f hub_uninstall_all
export -f hub_purge_all
# remove unused data from the index, to make sure we don't rely on it in any way
hub_strip_index() {
local INDEX
INDEX=$(config_get .config_paths.index_path)
local hub_min
hub_min=$(jq <"$INDEX" 'del(..|.content?) | del(..|.long_description?) | del(..|.deprecated?) | del (..|.labels?)')
echo "$hub_min" >"$INDEX"
}
export -f hub_strip_index
# remove color and style sequences from stdin
plaintext() {