beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Add "taintedBy" and "--diff" flag to cscli... inspect (#2665)

Browse source 
* "cscli inspect" reports tainted sub-items
* cscli... inspect --diff
* unified diff
* option --diff --rev
* tainted message
* correctly report multiple taint reasons
This commit is contained in:
mmetc 2023-12-15 15:27:22 +01:00 committed by GitHub
parent bc3a179af9
commit a79fcaf378
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 349 additions and 128 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmd/crowdsec-cli/hub.go
View file

@ -149,6 +149,7 @@ func (cli cliHub) upgrade(cmd *cobra.Command, args []string) error {
updated := 0
log.Infof("Upgrading %s", itemType)
for _, item := range items {
didUpdate, err := item.Upgrade(force)
if err != nil {
@ -158,6 +159,7 @@ func (cli cliHub) upgrade(cmd *cobra.Command, args []string) error {
updated++
}
}
log.Infof("Upgraded %d %s", updated, itemType)
}

3
cmd/crowdsec-cli/hubappsec.go
View file

@ -49,16 +49,19 @@ cscli appsec-configs list crowdsecurity/vpatch`,
func NewCLIAppsecRule() *cliItem {
inspectDetail := func(item *cwhub.Item) error {
appsecRule := appsec.AppsecCollectionConfig{}
yamlContent, err := os.ReadFile(item.State.LocalPath)
if err != nil {
return fmt.Errorf("unable to read file %s : %s", item.State.LocalPath, err)
}
if err := yaml.Unmarshal(yamlContent, &appsecRule); err != nil {
return fmt.Errorf("unable to unmarshal yaml file %s : %s", item.State.LocalPath, err)
}
for _, ruleType := range appsec_rule.SupportedTypes() {
fmt.Printf("\n%s format:\n", cases.Title(language.Und, cases.NoLower).String(ruleType))
for _, rule := range appsecRule.Rules {
convertedRule, _, err := rule.Convert(ruleType, appsecRule.Name)
if err != nil {

18
cmd/crowdsec-cli/hubtest.go
View file

@ -41,7 +41,7 @@ func (cli cliHubTest) NewCommand() *cobra.Command {
Long: "Run functional tests on hub configurations (parsers, scenarios, collections...)",
Args: cobra.ExactArgs(0),
DisableAutoGenTag: true,
PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
var err error
HubTest, err = hubtest.NewHubTest(hubPath, crowdsecPath, cscliPath, false)
if err != nil {
@ -94,7 +94,7 @@ cscli hubtest create my-nginx-custom-test --type nginx
cscli hubtest create my-scenario-test --parsers crowdsecurity/nginx --scenarios crowdsecurity/http-probing`,
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, args []string) error {
testName := args[0]
testPath := filepath.Join(hubPtr.HubTestPath, testName)
if _, err := os.Stat(testPath); os.IsExist(err) {
@ -262,7 +262,7 @@ func (cli cliHubTest) NewRunCmd() *cobra.Command {
return nil
},
PersistentPostRunE: func(cmd *cobra.Command, args []string) error {
PersistentPostRunE: func(_ *cobra.Command, _ []string) error {
success := true
testResult := make(map[string]bool)
for _, test := range hubPtr.Tests {
@ -388,7 +388,7 @@ func (cli cliHubTest) NewCleanCmd() *cobra.Command {
Short: "clean [test_name]",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, args []string) error {
for _, testName := range args {
test, err := hubPtr.LoadTestItem(testName)
if err != nil {
@ -412,7 +412,7 @@ func (cli cliHubTest) NewInfoCmd() *cobra.Command {
Short: "info [test_name]",
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, args []string) error {
for _, testName := range args {
test, err := hubPtr.LoadTestItem(testName)
if err != nil {
@ -444,7 +444,7 @@ func (cli cliHubTest) NewListCmd() *cobra.Command {
Use: "list",
Short: "list",
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
if err := hubPtr.LoadAllTests(); err != nil {
return fmt.Errorf("unable to load all tests: %s", err)
}
@ -479,7 +479,7 @@ func (cli cliHubTest) NewCoverageCmd() *cobra.Command {
Use: "coverage",
Short: "coverage",
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, _ []string) error {
//for this one we explicitly don't do for appsec
if err := HubTest.LoadAllTests(); err != nil {
return fmt.Errorf("unable to load all tests: %+v", err)
@ -617,7 +617,7 @@ func (cli cliHubTest) NewEvalCmd() *cobra.Command {
Short: "eval [test_name]",
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, args []string) error {
for _, testName := range args {
test, err := hubPtr.LoadTestItem(testName)
if err != nil {
@ -652,7 +652,7 @@ func (cli cliHubTest) NewExplainCmd() *cobra.Command {
Short: "explain [test_name]",
Args: cobra.ExactArgs(1),
DisableAutoGenTag: true,
RunE: func(cmd *cobra.Command, args []string) error {
RunE: func(_ *cobra.Command, args []string) error {
for _, testName := range args {
test, err := HubTest.LoadTestItem(testName)
if err != nil {

5
cmd/crowdsec-cli/item_metrics.go
View file

@ -34,8 +34,7 @@ func ShowMetrics(hubItem *cwhub.Item) error {
}
case cwhub.APPSEC_RULES:
log.Error("FIXME: not implemented yet")
default:
// no metrics for this item type
default: // no metrics for this item type
}
return nil
}
@ -222,6 +221,7 @@ var ranges = []unit{
func formatNumber(num int) string {
goodUnit := unit{}
for _, u := range ranges {
if int64(num) >= u.value {
goodUnit = u
@ -234,5 +234,6 @@ func formatNumber(num int) string {
}
res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
}

270
cmd/crowdsec-cli/itemcli.go
View file

@ -2,8 +2,13 @@ package main
import (
"fmt"
"os"
"strings"
"github.com/fatih/color"
"github.com/hexops/gotextdiff"
"github.com/hexops/gotextdiff/myers"
"github.com/hexops/gotextdiff/span"
log "github.com/sirupsen/logrus"
"github.com/spf13/cobra"
@ -35,27 +40,27 @@ type cliItem struct {
listHelp cliHelp
}
func (it cliItem) NewCommand() *cobra.Command {
func (cli cliItem) NewCommand() *cobra.Command {
cmd := &cobra.Command{
Use: coalesce.String(it.help.use, fmt.Sprintf("%s <action> [item]...", it.name)),
Short: coalesce.String(it.help.short, fmt.Sprintf("Manage hub %s", it.name)),
Long: it.help.long,
Example: it.help.example,
Use: coalesce.String(cli.help.use, fmt.Sprintf("%s <action> [item]...", cli.name)),
Short: coalesce.String(cli.help.short, fmt.Sprintf("Manage hub %s", cli.name)),
Long: cli.help.long,
Example: cli.help.example,
Args: cobra.MinimumNArgs(1),
Aliases: []string{it.singular},
Aliases: []string{cli.singular},
DisableAutoGenTag: true,
}
cmd.AddCommand(it.NewInstallCmd())
cmd.AddCommand(it.NewRemoveCmd())
cmd.AddCommand(it.NewUpgradeCmd())
cmd.AddCommand(it.NewInspectCmd())
cmd.AddCommand(it.NewListCmd())
cmd.AddCommand(cli.NewInstallCmd())
cmd.AddCommand(cli.NewRemoveCmd())
cmd.AddCommand(cli.NewUpgradeCmd())
cmd.AddCommand(cli.NewInspectCmd())
cmd.AddCommand(cli.NewListCmd())
return cmd
}
func (it cliItem) Install(cmd *cobra.Command, args []string) error {
func (cli cliItem) Install(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
downloadOnly, err := flags.GetBool("download-only")
@ -79,9 +84,9 @@ func (it cliItem) Install(cmd *cobra.Command, args []string) error {
}
for _, name := range args {
item := hub.GetItem(it.name, name)
item := hub.GetItem(cli.name, name)
if item == nil {
msg := suggestNearestMessage(hub, it.name, name)
msg := suggestNearestMessage(hub, cli.name, name)
if !ignoreError {
return fmt.Errorf(msg)
}
@ -103,24 +108,24 @@ func (it cliItem) Install(cmd *cobra.Command, args []string) error {
return nil
}
func (it cliItem) NewInstallCmd() *cobra.Command {
func (cli cliItem) NewInstallCmd() *cobra.Command {
cmd := &cobra.Command{
Use: coalesce.String(it.installHelp.use, "install [item]..."),
Short: coalesce.String(it.installHelp.short, fmt.Sprintf("Install given %s", it.oneOrMore)),
Long: coalesce.String(it.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", it.name)),
Example: it.installHelp.example,
Use: coalesce.String(cli.installHelp.use, "install [item]..."),
Short: coalesce.String(cli.installHelp.short, fmt.Sprintf("Install given %s", cli.oneOrMore)),
Long: coalesce.String(cli.installHelp.long, fmt.Sprintf("Fetch and install one or more %s from the hub", cli.name)),
Example: cli.installHelp.example,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compAllItems(it.name, args, toComplete)
return compAllItems(cli.name, args, toComplete)
},
RunE: it.Install,
RunE: cli.Install,
}
flags := cmd.Flags()
flags.BoolP("download-only", "d", false, "Only download packages, don't enable")
flags.Bool("force", false, "Force install: overwrite tainted and outdated files")
flags.Bool("ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", it.name))
flags.Bool("ignore", false, fmt.Sprintf("Ignore errors when installing multiple %s", cli.name))
return cmd
}
@ -138,7 +143,7 @@ func istalledParentNames(item *cwhub.Item) []string {
return ret
}
func (it cliItem) Remove(cmd *cobra.Command, args []string) error {
func (cli cliItem) Remove(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
purge, err := flags.GetBool("purge")
@ -167,7 +172,7 @@ func (it cliItem) Remove(cmd *cobra.Command, args []string) error {
getter = hub.GetAllItems
}
items, err := getter(it.name)
items, err := getter(cli.name)
if err != nil {
return err
}
@ -185,7 +190,7 @@ func (it cliItem) Remove(cmd *cobra.Command, args []string) error {
}
}
log.Infof("Removed %d %s", removed, it.name)
log.Infof("Removed %d %s", removed, cli.name)
if removed > 0 {
log.Infof(ReloadMessage())
}
@ -194,22 +199,23 @@ func (it cliItem) Remove(cmd *cobra.Command, args []string) error {
}
if len(args) == 0 {
return fmt.Errorf("specify at least one %s to remove or '--all'", it.singular)
return fmt.Errorf("specify at least one %s to remove or '--all'", cli.singular)
}
removed := 0
for _, itemName := range args {
item := hub.GetItem(it.name, itemName)
item := hub.GetItem(cli.name, itemName)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
return fmt.Errorf("can't find '%s' in %s", itemName, cli.name)
}
parents := istalledParentNames(item)
if !force && len(parents) > 0 {
log.Warningf("%s belongs to collections: %s", item.Name, parents)
log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, it.singular)
log.Warningf("Run 'sudo cscli %s remove %s --force' if you want to force remove this %s", item.Type, item.Name, cli.singular)
continue
}
@ -224,7 +230,7 @@ func (it cliItem) Remove(cmd *cobra.Command, args []string) error {
}
}
log.Infof("Removed %d %s", removed, it.name)
log.Infof("Removed %d %s", removed, cli.name)
if removed > 0 {
log.Infof(ReloadMessage())
}
@ -232,29 +238,29 @@ func (it cliItem) Remove(cmd *cobra.Command, args []string) error {
return nil
}
func (it cliItem) NewRemoveCmd() *cobra.Command {
func (cli cliItem) NewRemoveCmd() *cobra.Command {
cmd := &cobra.Command{
Use: coalesce.String(it.removeHelp.use, "remove [item]..."),
Short: coalesce.String(it.removeHelp.short, fmt.Sprintf("Remove given %s", it.oneOrMore)),
Long: coalesce.String(it.removeHelp.long, fmt.Sprintf("Remove one or more %s", it.name)),
Example: it.removeHelp.example,
Use: coalesce.String(cli.removeHelp.use, "remove [item]..."),
Short: coalesce.String(cli.removeHelp.short, fmt.Sprintf("Remove given %s", cli.oneOrMore)),
Long: coalesce.String(cli.removeHelp.long, fmt.Sprintf("Remove one or more %s", cli.name)),
Example: cli.removeHelp.example,
Aliases: []string{"delete"},
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(it.name, args, toComplete)
return compInstalledItems(cli.name, args, toComplete)
},
RunE: it.Remove,
RunE: cli.Remove,
}
flags := cmd.Flags()
flags.Bool("purge", false, "Delete source file too")
flags.Bool("force", false, "Force remove: remove tainted and outdated files")
flags.Bool("all", false, fmt.Sprintf("Remove all the %s", it.name))
flags.Bool("all", false, fmt.Sprintf("Remove all the %s", cli.name))
return cmd
}
func (it cliItem) Upgrade(cmd *cobra.Command, args []string) error {
func (cli cliItem) Upgrade(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
force, err := flags.GetBool("force")
@ -273,7 +279,7 @@ func (it cliItem) Upgrade(cmd *cobra.Command, args []string) error {
}
if all {
items, err := hub.GetInstalledItems(it.name)
items, err := hub.GetInstalledItems(cli.name)
if err != nil {
return err
}
@ -290,7 +296,7 @@ func (it cliItem) Upgrade(cmd *cobra.Command, args []string) error {
}
}
log.Infof("Updated %d %s", updated, it.name)
log.Infof("Updated %d %s", updated, cli.name)
if updated > 0 {
log.Infof(ReloadMessage())
@ -300,15 +306,15 @@ func (it cliItem) Upgrade(cmd *cobra.Command, args []string) error {
}
if len(args) == 0 {
return fmt.Errorf("specify at least one %s to upgrade or '--all'", it.singular)
return fmt.Errorf("specify at least one %s to upgrade or '--all'", cli.singular)
}
updated := 0
for _, itemName := range args {
item := hub.GetItem(it.name, itemName)
item := hub.GetItem(cli.name, itemName)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", itemName, it.name)
return fmt.Errorf("can't find '%s' in %s", itemName, cli.name)
}
didUpdate, err := item.Upgrade(force)
@ -328,27 +334,27 @@ func (it cliItem) Upgrade(cmd *cobra.Command, args []string) error {
return nil
}
func (it cliItem) NewUpgradeCmd() *cobra.Command {
func (cli cliItem) NewUpgradeCmd() *cobra.Command {
cmd := &cobra.Command{
Use: coalesce.String(it.upgradeHelp.use, "upgrade [item]..."),
Short: coalesce.String(it.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", it.oneOrMore)),
Long: coalesce.String(it.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", it.name)),
Example: it.upgradeHelp.example,
Use: coalesce.String(cli.upgradeHelp.use, "upgrade [item]..."),
Short: coalesce.String(cli.upgradeHelp.short, fmt.Sprintf("Upgrade given %s", cli.oneOrMore)),
Long: coalesce.String(cli.upgradeHelp.long, fmt.Sprintf("Fetch and upgrade one or more %s from the hub", cli.name)),
Example: cli.upgradeHelp.example,
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(it.name, args, toComplete)
return compInstalledItems(cli.name, args, toComplete)
},
RunE: it.Upgrade,
RunE: cli.Upgrade,
}
flags := cmd.Flags()
flags.BoolP("all", "a", false, fmt.Sprintf("Upgrade all the %s", it.name))
flags.BoolP("all", "a", false, fmt.Sprintf("Upgrade all the %s", cli.name))
flags.Bool("force", false, "Force upgrade: overwrite tainted and outdated files")
return cmd
}
func (it cliItem) Inspect(cmd *cobra.Command, args []string) error {
func (cli cliItem) Inspect(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
url, err := flags.GetString("url")
@ -360,27 +366,50 @@ func (it cliItem) Inspect(cmd *cobra.Command, args []string) error {
csConfig.Cscli.PrometheusUrl = url
}
diff, err := flags.GetBool("diff")
if err != nil {
return err
}
rev, err := flags.GetBool("rev")
if err != nil {
return err
}
noMetrics, err := flags.GetBool("no-metrics")
if err != nil {
return err
}
hub, err := require.Hub(csConfig, nil)
remote := (*cwhub.RemoteHubCfg)(nil)
if diff {
remote = require.RemoteHub(csConfig)
}
hub, err := require.Hub(csConfig, remote)
if err != nil {
return err
}
for _, name := range args {
item := hub.GetItem(it.name, name)
item := hub.GetItem(cli.name, name)
if item == nil {
return fmt.Errorf("can't find '%s' in %s", name, it.name)
return fmt.Errorf("can't find '%s' in %s", name, cli.name)
}
if diff {
fmt.Println(cli.whyTainted(hub, item, rev))
continue
}
if err = InspectItem(item, !noMetrics); err != nil {
return err
}
if it.inspectDetail != nil {
if err = it.inspectDetail(item); err != nil {
if cli.inspectDetail != nil {
if err = cli.inspectDetail(item); err != nil {
return err
}
}
@ -389,28 +418,49 @@ func (it cliItem) Inspect(cmd *cobra.Command, args []string) error {
return nil
}
func (it cliItem) NewInspectCmd() *cobra.Command {
func (cli cliItem) NewInspectCmd() *cobra.Command {
cmd := &cobra.Command{
Use: coalesce.String(it.inspectHelp.use, "inspect [item]..."),
Short: coalesce.String(it.inspectHelp.short, fmt.Sprintf("Inspect given %s", it.oneOrMore)),
Long: coalesce.String(it.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", it.name)),
Example: it.inspectHelp.example,
Use: coalesce.String(cli.inspectHelp.use, "inspect [item]..."),
Short: coalesce.String(cli.inspectHelp.short, fmt.Sprintf("Inspect given %s", cli.oneOrMore)),
Long: coalesce.String(cli.inspectHelp.long, fmt.Sprintf("Inspect the state of one or more %s", cli.name)),
Example: cli.inspectHelp.example,
Args: cobra.MinimumNArgs(1),
DisableAutoGenTag: true,
ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
return compInstalledItems(it.name, args, toComplete)
return compInstalledItems(cli.name, args, toComplete)
},
RunE: it.Inspect,
PreRunE: func(cmd *cobra.Command, _ []string) error {
flags := cmd.Flags()
diff, err := flags.GetBool("diff")
if err != nil {
return err
}
rev, err := flags.GetBool("rev")
if err != nil {
return err
}
if rev && !diff {
return fmt.Errorf("--rev can only be used with --diff")
}
return nil
},
RunE: cli.Inspect,
}
flags := cmd.Flags()
flags.StringP("url", "u", "", "Prometheus url")
flags.Bool("diff", false, "Show diff with latest version (for tainted items)")
flags.Bool("rev", false, "Reverse diff output")
flags.Bool("no-metrics", false, "Don't show metrics (when cscli.output=human)")
return cmd
}
func (it cliItem) List(cmd *cobra.Command, args []string) error {
func (cli cliItem) List(cmd *cobra.Command, args []string) error {
flags := cmd.Flags()
all, err := flags.GetBool("all")
@ -425,26 +475,26 @@ func (it cliItem) List(cmd *cobra.Command, args []string) error {
items := make(map[string][]*cwhub.Item)
items[it.name], err = selectItems(hub, it.name, args, !all)
items[cli.name], err = selectItems(hub, cli.name, args, !all)
if err != nil {
return err
}
if err = listItems(color.Output, []string{it.name}, items, false); err != nil {
if err = listItems(color.Output, []string{cli.name}, items, false); err != nil {
return err
}
return nil
}
func (it cliItem) NewListCmd() *cobra.Command {
func (cli cliItem) NewListCmd() *cobra.Command {
cmd := &cobra.Command{
Use: coalesce.String(it.listHelp.use, "list [item... | -a]"),
Short: coalesce.String(it.listHelp.short, fmt.Sprintf("List %s", it.oneOrMore)),
Long: coalesce.String(it.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", it.name)),
Example: it.listHelp.example,
Use: coalesce.String(cli.listHelp.use, "list [item... | -a]"),
Short: coalesce.String(cli.listHelp.short, fmt.Sprintf("List %s", cli.oneOrMore)),
Long: coalesce.String(cli.listHelp.long, fmt.Sprintf("List of installed/available/specified %s", cli.name)),
Example: cli.listHelp.example,
DisableAutoGenTag: true,
RunE: it.List,
RunE: cli.List,
}
flags := cmd.Flags()
@ -452,3 +502,75 @@ func (it cliItem) NewListCmd() *cobra.Command {
return cmd
}
// return the diff between the installed version and the latest version
func (cli cliItem) itemDiff(item *cwhub.Item, reverse bool) (string, error) {
if !item.State.Installed {
return "", fmt.Errorf("'%s' is not installed", item.FQName())
}
latestContent, remoteURL, err := item.FetchLatest()
if err != nil {
return "", err
}
localContent, err := os.ReadFile(item.State.LocalPath)
if err != nil {
return "", fmt.Errorf("while reading %s: %w", item.State.LocalPath, err)
}
file1 := item.State.LocalPath
file2 := remoteURL
content1 := string(localContent)
content2 := string(latestContent)
if reverse {
file1, file2 = file2, file1
content1, content2 = content2, content1
}
edits := myers.ComputeEdits(span.URIFromPath(file1), content1, content2)
diff := gotextdiff.ToUnified(file1, file2, content1, edits)
return fmt.Sprintf("%s", diff), nil
}
func (cli cliItem) whyTainted(hub *cwhub.Hub, item *cwhub.Item, reverse bool) string {
if !item.State.Installed {
return fmt.Sprintf("# %s is not installed", item.FQName())
}
if !item.State.Tainted {
return fmt.Sprintf("# %s is not tainted", item.FQName())
}
if len(item.State.TaintedBy) == 0 {
return fmt.Sprintf("# %s is tainted but we don't know why. please report this as a bug", item.FQName())
}
ret := []string{
fmt.Sprintf("# Let's see why %s is tainted.", item.FQName()),
}
for _, fqsub := range item.State.TaintedBy {
ret = append(ret, fmt.Sprintf("\n-> %s\n", fqsub))
sub, err := hub.GetItemFQ(fqsub)
if err != nil {
ret = append(ret, err.Error())
}
diff, err := cli.itemDiff(sub, reverse)
if err != nil {
ret = append(ret, err.Error())
}
if diff != "" {
ret = append(ret, diff)
} else if len(sub.State.TaintedBy) > 0 {
taintList := strings.Join(sub.State.TaintedBy, ", ")
ret = append(ret, fmt.Sprintf("# %s is tainted by %s", sub.FQName(), taintList))
}
}
return strings.Join(ret, "\n")
}

6
cmd/crowdsec-cli/items.go
View file

@ -57,6 +57,7 @@ func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item
switch csConfig.Cscli.Output {
case "human":
nothingToDisplay := true
for _, itemType := range itemTypes {
if omitIfEmpty && len(items[itemType]) == 0 {
continue
@ -64,6 +65,7 @@ func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item
listHubItemTable(out, "\n"+strings.ToUpper(itemType), items[itemType])
nothingToDisplay = false
}
if nothingToDisplay {
fmt.Println("No items to display")
}
@ -84,14 +86,14 @@ func listItems(out io.Writer, itemTypes []string, items map[string][]*cwhub.Item
for i, item := range items[itemType] {
status := item.State.Text()
status_emo := item.State.Emoji()
statusEmo := item.State.Emoji()
hubStatus[itemType][i] = itemHubStatus{
Name: item.Name,
LocalVersion: item.State.LocalVersion,
LocalPath: item.State.LocalPath,
Description: item.Description,
Status: status,
UTF8Status: fmt.Sprintf("%v %s", status_emo, status),
UTF8Status: fmt.Sprintf("%v %s", statusEmo, status),
}
}
}

9
go.mod
View file

@ -24,6 +24,7 @@ require (
github.com/buger/jsonparser v1.1.1
github.com/c-robinson/iplib v1.0.3
github.com/cespare/xxhash/v2 v2.2.0
github.com/crowdsecurity/coraza/v3 v3.0.0-20231213144607-41d5358da94f
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26
github.com/crowdsecurity/go-cs-lib v0.0.5
github.com/crowdsecurity/grokky v0.2.1
@ -53,6 +54,7 @@ require (
github.com/hashicorp/go-hclog v1.5.0
github.com/hashicorp/go-plugin v1.4.10
github.com/hashicorp/go-version v1.2.1
github.com/hexops/gotextdiff v1.0.3
github.com/ivanpirog/coloredcobra v1.0.1
github.com/jackc/pgx/v4 v4.14.1
github.com/jarcoal/httpmock v1.1.0
@ -83,16 +85,12 @@ require (
golang.org/x/crypto v0.16.0
golang.org/x/mod v0.11.0
golang.org/x/sys v0.15.0
golang.org/x/text v0.14.0
google.golang.org/grpc v1.56.3
google.golang.org/protobuf v1.31.0
gopkg.in/natefinch/lumberjack.v2 v2.2.1
gopkg.in/tomb.v2 v2.0.0-20161208151619-d5d1b5820637
gopkg.in/yaml.v2 v2.4.0
)
require (
github.com/crowdsecurity/coraza/v3 v3.0.0-20231213144607-41d5358da94f
golang.org/x/text v0.14.0
gopkg.in/yaml.v3 v3.0.1
gotest.tools/v3 v3.5.0
k8s.io/apiserver v0.28.4
@ -183,6 +181,7 @@ require (
github.com/rivo/uniseg v0.2.0 // indirect
github.com/robfig/cron/v3 v3.0.1 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sergi/go-diff v1.3.1 // indirect
github.com/shoenig/go-m1cpu v0.1.6 // indirect
github.com/shopspring/decimal v1.2.0 // indirect
github.com/spf13/cast v1.3.1 // indirect

8
go.sum
View file

@ -98,8 +98,6 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3
github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/creack/pty v1.1.18 h1:n56/Zwd5o6whRC5PMGretI4IdRLlmBXYNjScPaBgsbY=
github.com/creack/pty v1.1.18/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/crowdsecurity/coraza/v3 v3.0.0-20231206171741-c5b03c916879 h1:dhAc0AelASC3BbfuLURJeai1LYgFNgpMds0KPd9whbo=
github.com/crowdsecurity/coraza/v3 v3.0.0-20231206171741-c5b03c916879/go.mod h1:jNww1Y9SujXQc89zDR+XOb70bkC7mZ6ep7iKhUBBsiI=
github.com/crowdsecurity/coraza/v3 v3.0.0-20231213144607-41d5358da94f h1:FkOB9aDw0xzDd14pTarGRLsUNAymONq3dc7zhvsXElg=
github.com/crowdsecurity/coraza/v3 v3.0.0-20231213144607-41d5358da94f/go.mod h1:TrU7Li+z2RHNrPy0TKJ6R65V6Yzpan2sTIRryJJyJso=
github.com/crowdsecurity/dlog v0.0.0-20170105205344-4fb5f8204f26 h1:r97WNVC30Uen+7WnLs4xDScS/Ex988+id2k6mDf8psU=
@ -348,6 +346,8 @@ github.com/hashicorp/hcl/v2 v2.13.0 h1:0Apadu1w6M11dyGFxWnmhhcMjkbAiKCv7G1r/2QgC
github.com/hashicorp/hcl/v2 v2.13.0/go.mod h1:e4z5nxYlWNPdDSNYX+ph14EvWYMFm3eP0zIUqPc2jr0=
github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb h1:b5rjCoWHc7eqmAS4/qyk21ZsHyb6Mxv/jykxvNTkU4M=
github.com/hashicorp/yamux v0.0.0-20180604194846-3520598351bb/go.mod h1:+NfK9FKeTrX5uv1uIXGdwYDTeHna2qgaIlx54MXqjAM=
github.com/hexops/gotextdiff v1.0.3 h1:gitA9+qJrrTCsiCl7+kh75nPqQt1cx4ZkudSTLoUqJM=
github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSow5/V2vxeg=
github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog=
github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68=
github.com/huandu/xstrings v1.3.3 h1:/Gcsuc1x8JVbJ9/rlye4xZnVAbEkGauT8lbebqcQws4=
@ -612,8 +612,9 @@ github.com/sanity-io/litter v1.5.5/go.mod h1:9gzJgR2i4ZpjZHsKvUXIRQVk7P+yM3e+jAF
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
github.com/segmentio/kafka-go v0.4.45 h1:prqrZp1mMId4kI6pyPolkLsH6sWOUmDxmmucbL4WS6E=
github.com/segmentio/kafka-go v0.4.45/go.mod h1:HjF6XbOKh0Pjlkr5GVZxt6CsjjwnmhVOfURM5KMd8qg=
github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8=
github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I=
github.com/shirou/gopsutil/v3 v3.23.5 h1:5SgDCeQ0KW0S4N0znjeM/eFHXXOKyv2dVNgRq/c9P6Y=
github.com/shirou/gopsutil/v3 v3.23.5/go.mod h1:Ng3Maa27Q2KARVJ0SPZF5NdrQSC3XHKP8IIWrHgMeLY=
github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
@ -916,6 +917,7 @@ google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqw
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=

22
pkg/cwhub/hub.go
View file

@ -179,6 +179,28 @@ func (h *Hub) GetItem(itemType string, itemName string) *Item {
return h.GetItemMap(itemType)[itemName]
}
// GetItemFQ returns an item from hub based on its type and name (type:author/name).
func (h *Hub) GetItemFQ(itemFQName string) (*Item, error) {
// type and name are separated by a colon
parts := strings.Split(itemFQName, ":")
if len(parts) != 2 {
return nil, fmt.Errorf("invalid item name %s", itemFQName)
}
m := h.GetItemMap(parts[0])
if m == nil {
return nil, fmt.Errorf("invalid item type %s", parts[0])
}
i := m[parts[1]]
if i == nil {
return nil, fmt.Errorf("item %s not found", parts[1])
}
return i, nil
}
// GetItemNames returns a slice of (full) item names for a given type
// (eg. for collections: crowdsecurity/apache2 crowdsecurity/nginx).
func (h *Hub) GetItemNames(itemType string) []string {

35
pkg/cwhub/item.go
View file

@ -8,6 +8,7 @@ import (
"github.com/Masterminds/semver/v3"
"github.com/enescakir/emoji"
log "github.com/sirupsen/logrus"
"slices"
)
const (
@ -53,6 +54,7 @@ type ItemState struct {
Downloaded bool `json:"downloaded"`
UpToDate bool `json:"up_to_date"`
Tainted bool `json:"tainted"`
TaintedBy []string `json:"tainted_by,omitempty" yaml:"tainted_by,omitempty"`
BelongsToCollections []string `json:"belongs_to_collections,omitempty" yaml:"belongs_to_collections,omitempty"`
}
@ -406,3 +408,36 @@ func (i *Item) versionStatus() int {
func (i *Item) validPath(dirName, fileName string) bool {
return (dirName+"/"+fileName == i.Name+".yaml") || (dirName+"/"+fileName == i.Name+".yml")
}
// FQName returns the fully qualified name of the item (ie. parsers:crowdsecurity/apache2-logs).
func (i *Item) FQName () string {
return fmt.Sprintf("%s:%s", i.Type, i.Name)
}
// addTaint marks the item as tainted, and propagates the taint to the ancestors.
// sub: the sub-item that caused the taint. May be the item itself!
func (i *Item) addTaint(sub *Item) {
i.State.Tainted = true
taintedBy := sub.FQName()
idx, ok := slices.BinarySearch(i.State.TaintedBy, taintedBy)
if ok {
return
}
// insert the taintedBy in the slice
i.State.TaintedBy = append(i.State.TaintedBy, "")
copy(i.State.TaintedBy[idx+1:], i.State.TaintedBy[idx:])
i.State.TaintedBy[idx] = taintedBy
log.Debugf("%s is tainted by %s", i.Name, taintedBy)
// propagate the taint to the ancestors
for _, ancestor := range i.Ancestors() {
ancestor.addTaint(sub)
}
}

2
pkg/cwhub/iteminstall_test.go
View file

@ -106,6 +106,7 @@ func TestInstallParser(t *testing.T) {
testTaint(hub, t, it)
testUpdate(hub, t, it)
testDisable(hub, t, it)
break
}
}
@ -128,6 +129,7 @@ func TestInstallCollection(t *testing.T) {
testTaint(hub, t, it)
testUpdate(hub, t, it)
testDisable(hub, t, it)
break
}
}

3
pkg/cwhub/itemremove.go
View file

@ -45,14 +45,15 @@ func (i *Item) disable(purge bool, force bool) (bool, error) {
link, _ := i.installPath()
return false, fmt.Errorf("link %s does not exist (override with --force or --purge)", link)
}
didRemove = false
} else if err != nil {
return false, err
}
i.State.Installed = false
didPurge := false
if purge {
if didPurge, err = i.purge(); err != nil {
return didRemove, err

22
pkg/cwhub/itemupgrade.go
View file

@ -115,31 +115,31 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
return ret, nil
}
// fetch downloads the item from the hub, verifies the hash and returns the content.
func (i *Item) fetch() ([]byte, error) {
// FetchLatest downloads the latest item from the hub, verifies the hash and returns the content and the used url.
func (i *Item) FetchLatest() ([]byte, string, error) {
url, err := i.hub.remote.urlTo(i.RemotePath)
if err != nil {
return nil, fmt.Errorf("failed to build hub item request: %w", err)
return nil, "", fmt.Errorf("failed to build request: %w", err)
}
resp, err := hubClient.Get(url)
if err != nil {
return nil, fmt.Errorf("while downloading %s: %w", url, err)
return nil, "", err
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("bad http code %d for %s", resp.StatusCode, url)
return nil, "", fmt.Errorf("bad http code %d", resp.StatusCode)
}
body, err := io.ReadAll(resp.Body)
if err != nil {
return nil, fmt.Errorf("while downloading %s: %w", url, err)
return nil, "", err
}
hash := sha256.New()
if _, err = hash.Write(body); err != nil {
return nil, fmt.Errorf("while hashing %s: %w", i.Name, err)
return nil, "", fmt.Errorf("while hashing %s: %w", i.Name, err)
}
meow := hex.EncodeToString(hash.Sum(nil))
@ -147,10 +147,10 @@ func (i *Item) fetch() ([]byte, error) {
log.Errorf("Downloaded version doesn't match index, please 'hub update'")
log.Debugf("got %s, expected %s", meow, i.Versions[i.Version].Digest)
return nil, fmt.Errorf("invalid download hash for %s", i.Name)
return nil, "", fmt.Errorf("invalid download hash for %s", i.Name)
}
return body, nil
return body, url, nil
}
// download downloads the item from the hub and writes it to the hub directory.
@ -171,9 +171,9 @@ func (i *Item) download(overwrite bool) (string, error) {
}
}
body, err := i.fetch()
body, url, err := i.FetchLatest()
if err != nil {
return "", err
return "", fmt.Errorf("while downloading %s: %w", url, err)
}
// all good, install

2
pkg/cwhub/itemupgrade_test.go
View file

@ -189,7 +189,7 @@ func assertCollectionDepsInstalled(t *testing.T, hub *Hub, collection string) {
t.Helper()
c := hub.GetItem(COLLECTIONS, collection)
require.NoError(t, c.checkSubItemVersions())
require.Empty(t, c.checkSubItemVersions())
}
func pushUpdateToCollectionInHub() {

66
pkg/cwhub/sync.go
View file

@ -7,13 +7,13 @@ import (
"io"
"os"
"path/filepath"
"slices"
"sort"
"strings"
"github.com/Masterminds/semver/v3"
log "github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"
"slices"
)
func isYAMLFileName(path string) bool {
@ -221,10 +221,12 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
if !info.inhub {
log.Tracef("%s is a local file, skip", path)
item, err := newLocalItem(h, path, info)
if err != nil {
return err
}
h.addItem(item)
return nil
@ -295,14 +297,16 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
}
// checkSubItemVersions checks for the presence, taint and version state of sub-items.
func (i *Item) checkSubItemVersions() error {
func (i *Item) checkSubItemVersions() []string {
warn := make([]string, 0)
if !i.HasSubItems() {
return nil
return warn
}
if i.versionStatus() != versionUpToDate {
log.Debugf("%s dependencies not checked: not up-to-date", i.Name)
return nil
return warn
}
// ensure all the sub-items are installed, or tag the parent as tainted
@ -315,33 +319,42 @@ func (i *Item) checkSubItemVersions() error {
continue
}
if err := sub.checkSubItemVersions(); err != nil {
if w := sub.checkSubItemVersions(); len(w) > 0 {
if sub.State.Tainted {
i.State.Tainted = true
i.addTaint(sub)
warn = append(warn, fmt.Sprintf("%s is tainted by %s", i.Name, sub.FQName()))
}
return fmt.Errorf("dependency of %s: sub collection %s is broken: %w", i.Name, sub.Name, err)
warn = append(warn, w...)
continue
}
if sub.State.Tainted {
i.State.Tainted = true
return fmt.Errorf("%s is tainted because %s:%s is tainted", i.Name, sub.Type, sub.Name)
i.addTaint(sub)
warn = append(warn, fmt.Sprintf("%s is tainted by %s", i.Name, sub.FQName()))
continue
}
if !sub.State.Installed && i.State.Installed {
i.State.Tainted = true
return fmt.Errorf("%s is tainted because %s:%s is missing", i.Name, sub.Type, sub.Name)
i.addTaint(sub)
warn = append(warn, fmt.Sprintf("%s is tainted by missing %s", i.Name, sub.FQName()))
continue
}
if !sub.State.UpToDate {
i.State.UpToDate = false
return fmt.Errorf("dependency of %s: outdated %s:%s", i.Name, sub.Type, sub.Name)
warn = append(warn, fmt.Sprintf("%s is tainted by outdated %s", i.Name, sub.FQName()))
continue
}
log.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, i.State.Tainted, i.State.UpToDate)
}
return nil
return warn
}
// syncDir scans a directory for items, and updates the Hub state accordingly.
@ -379,6 +392,23 @@ func insertInOrderNoCase(sl []string, value string) []string {
return append(sl[:i], append([]string{value}, sl[i:]...)...)
}
func removeDuplicates(sl []string) []string {
seen := make(map[string]struct{}, len(sl))
j := 0
for _, v := range sl {
if _, ok := seen[v]; ok {
continue
}
seen[v] = struct{}{}
sl[j] = v
j++
}
return sl[:j]
}
// localSync updates the hub state with downloaded, installed and local items.
func (h *Hub) localSync() error {
err := h.syncDir(h.local.InstallDir)
@ -411,8 +441,8 @@ func (h *Hub) localSync() error {
vs := item.versionStatus()
switch vs {
case versionUpToDate: // latest
if err := item.checkSubItemVersions(); err != nil {
warnings = append(warnings, err.Error())
if w := item.checkSubItemVersions(); len(w) > 0 {
warnings = append(warnings, w...)
}
case versionUpdateAvailable: // not up-to-date
warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
@ -420,14 +450,14 @@ func (h *Hub) localSync() error {
warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version))
case versionUnknown:
if !item.State.IsLocal() {
warnings = append(warnings, fmt.Sprintf("collection %s is tainted (latest:%s)", item.Name, item.Version))
warnings = append(warnings, fmt.Sprintf("collection %s is tainted by local changes (latest:%s)", item.Name, item.Version))
}
}
log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.State.LocalVersion, item.Version, item.Versions)
}
h.Warnings = warnings
h.Warnings = removeDuplicates(warnings)
return nil
}
@ -469,7 +499,7 @@ func (i *Item) setVersionState(path string, inhub bool) error {
}
i.State.UpToDate = false
i.State.Tainted = true
i.addTaint(i)
return nil
}

4
test/bats/20_hub.bats
View file

@ -83,7 +83,7 @@ teardown() {
refute_stderr --partial "tainted"
rune -0 truncate -s0 "$CONFIG_DIR/parsers/s01-parse/sshd-logs.yaml"
rune -0 cscli hub list
assert_stderr --partial "crowdsecurity/sshd is tainted because parsers:crowdsecurity/sshd-logs is tainted"
assert_stderr --partial "crowdsecurity/sshd is tainted by parsers:crowdsecurity/sshd-logs"
}
@test "loading hub reports tainted items (subitem is not installed)" {
@ -92,7 +92,7 @@ teardown() {
refute_stderr --partial "tainted"
rune -0 rm "$CONFIG_DIR/parsers/s01-parse/sshd-logs.yaml"
rune -0 cscli hub list
assert_stderr --partial "crowdsecurity/sshd is tainted because parsers:crowdsecurity/sshd-logs is missing"
assert_stderr --partial "crowdsecurity/sshd is tainted by missing parsers:crowdsecurity/sshd-logs"
}
@test "cscli hub update" {