beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

pkg/cwhub: improve error messages (#2712)

Browse source 
* pkg/cwhub: improve error messages
* lint
This commit is contained in:
mmetc 2024-01-11 10:28:58 +01:00 committed by GitHub
parent 0f722916b8
commit 260f5a7992
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 58 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
cmd/crowdsec-cli/hub.go
View file

@ -155,6 +155,7 @@ func (cli cliHub) upgrade(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
if didUpdate {
updated++
}
@ -191,18 +192,21 @@ func (cli cliHub) types(cmd *cobra.Command, args []string) error {
if err != nil {
return err
}
fmt.Print(string(s))
case "json":
jsonStr, err := json.Marshal(cwhub.ItemTypes)
if err != nil {
return err
}
fmt.Println(string(jsonStr))
case "raw":
for _, itemType := range cwhub.ItemTypes {
fmt.Println(itemType)
}
}
return nil
}

5
cmd/crowdsec-cli/hubappsec.go
View file

@ -48,10 +48,11 @@ cscli appsec-configs list crowdsecurity/vpatch`,
func NewCLIAppsecRule() *cliItem {
inspectDetail := func(item *cwhub.Item) error {
//Only show the converted rules in human mode
// Only show the converted rules in human mode
if csConfig.Cscli.Output != "human" {
return nil
}
appsecRule := appsec.AppsecCollectionConfig{}
yamlContent, err := os.ReadFile(item.State.LocalPath)
@ -65,11 +66,13 @@ func NewCLIAppsecRule() *cliItem {
for _, ruleType := range appsec_rule.SupportedTypes() {
fmt.Printf("\n%s format:\n", cases.Title(language.Und, cases.NoLower).String(ruleType))
for _, rule := range appsecRule.Rules {
convertedRule, _, err := rule.Convert(ruleType, appsecRule.Name)
if err != nil {
return fmt.Errorf("unable to convert rule %s : %s", rule.Name, err)
}
fmt.Println(convertedRule)
}

2
cmd/crowdsec-cli/item_suggest.go
View file

@ -2,11 +2,11 @@ package main
import (
"fmt"
"slices"
"strings"
"github.com/agext/levenshtein"
"github.com/spf13/cobra"
"slices"
"github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/require"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"

2
cmd/crowdsec-cli/items.go
View file

@ -7,10 +7,10 @@ import (
"io"
"os"
"path/filepath"
"slices"
"strings"
"gopkg.in/yaml.v3"
"slices"
"github.com/crowdsecurity/crowdsec/pkg/cwhub"
)

3
pkg/cwhub/dataset.go
View file

@ -56,6 +56,7 @@ func downloadFile(url string, destPath string) error {
// if the remote has no modification date, but local file has been modified > a week ago, update.
func needsUpdate(destPath string, url string, logger *logrus.Logger) bool {
fileInfo, err := os.Stat(destPath)
switch {
case os.IsNotExist(err):
return true
@ -89,6 +90,7 @@ func needsUpdate(destPath string, url string, logger *logrus.Logger) bool {
if localIsOld {
logger.Infof("no last modified date for %s, but local file is older than %s", url, shelfLife)
}
return localIsOld
}
@ -129,6 +131,7 @@ func downloadDataSet(dataFolder string, force bool, reader io.Reader, logger *lo
if force || needsUpdate(destPath, dataS.SourceURL, logger) {
logger.Debugf("downloading %s in %s", dataS.SourceURL, destPath)
if err := downloadFile(dataS.SourceURL, destPath); err != nil {
return fmt.Errorf("while getting data: %w", err)
}

6
pkg/cwhub/hub.go
View file

@ -7,10 +7,10 @@ import (
"io"
"os"
"path"
"slices"
"strings"
"github.com/sirupsen/logrus"
"slices"
"github.com/crowdsecurity/crowdsec/pkg/csconfig"
)
@ -97,6 +97,10 @@ func (h *Hub) parseIndex() error {
item.FileName = path.Base(item.RemotePath)
item.logMissingSubItems()
if item.latestHash() == "" {
h.logger.Errorf("invalid hub item %s: latest version missing from index", item.FQName())
}
}
}

1
pkg/cwhub/hub_test.go
View file

@ -12,7 +12,6 @@ import (
func TestInitHubUpdate(t *testing.T) {
hub := envSetup(t)
remote := &RemoteHubCfg{
URLTemplate: mockURLTemplate,
Branch: "master",

14
pkg/cwhub/item.go
View file

@ -4,10 +4,10 @@ import (
"encoding/json"
"fmt"
"path/filepath"
"slices"
"github.com/Masterminds/semver/v3"
"github.com/enescakir/emoji"
"slices"
)
const (
@ -440,3 +440,15 @@ func (i *Item) addTaint(sub *Item) {
ancestor.addTaint(sub)
}
}
// latestHash() returns the hash of the latest version of the item.
// if it's missing, the index file has been manually modified or got corrupted.
func (i *Item) latestHash() string {
for k, v := range i.Versions {
if k == i.Version {
return v.Digest
}
}
return ""
}

2
pkg/cwhub/iteminstall.go
View file

@ -50,7 +50,7 @@ func (i *Item) Install(force bool, downloadOnly bool) error {
filePath, err := i.downloadLatest(force, true)
if err != nil {
return fmt.Errorf("while downloading %s: %w", i.Name, err)
return err
}
if downloadOnly {

1
pkg/cwhub/itemremove.go
View file

@ -3,7 +3,6 @@ package cwhub
import (
"fmt"
"os"
"slices"
)

20
pkg/cwhub/itemupgrade.go
View file

@ -6,6 +6,7 @@ import (
"bytes"
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"io"
"net/http"
@ -82,14 +83,14 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
i.hub.logger.Tracef("collection, recurse")
if _, err := sub.downloadLatest(overwrite, updateOnly); err != nil {
return "", fmt.Errorf("while downloading %s: %w", sub.Name, err)
return "", err
}
}
downloaded := sub.State.Downloaded
if _, err := sub.download(overwrite); err != nil {
return "", fmt.Errorf("while downloading %s: %w", sub.Name, err)
return "", err
}
// We need to enable an item when it has been added to a collection since latest release of the collection.
@ -108,7 +109,7 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
ret, err := i.download(overwrite)
if err != nil {
return "", fmt.Errorf("failed to download item: %w", err)
return "", err
}
return ret, nil
@ -116,6 +117,10 @@ func (i *Item) downloadLatest(overwrite bool, updateOnly bool) (string, error) {
// FetchLatest downloads the latest item from the hub, verifies the hash and returns the content and the used url.
func (i *Item) FetchLatest() ([]byte, string, error) {
if i.latestHash() == "" {
return nil, "", errors.New("latest hash missing from index")
}
url, err := i.hub.remote.urlTo(i.RemotePath)
if err != nil {
return nil, "", fmt.Errorf("failed to build request: %w", err)
@ -146,7 +151,7 @@ func (i *Item) FetchLatest() ([]byte, string, error) {
i.hub.logger.Errorf("Downloaded version doesn't match index, please 'hub update'")
i.hub.logger.Debugf("got %s, expected %s", meow, i.Versions[i.Version].Digest)
return nil, "", fmt.Errorf("invalid download hash for %s", i.Name)
return nil, "", fmt.Errorf("invalid download hash")
}
return body, url, nil
@ -180,7 +185,12 @@ func (i *Item) download(overwrite bool) (string, error) {
body, url, err := i.FetchLatest()
if err != nil {
return "", fmt.Errorf("while downloading %s: %w", url, err)
what := i.Name
if url != "" {
what += " from " + url
}
return "", fmt.Errorf("while downloading %s: %w", what, err)
}
// all good, install

3
pkg/cwhub/itemupgrade_test.go
View file

@ -12,7 +12,6 @@ import (
// We expect the new scenario to be installed.
func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
hub := envSetup(t)
item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
// fresh install of collection
@ -65,7 +64,6 @@ func TestUpgradeItemNewScenarioInCollection(t *testing.T) {
// Upgrade should install should not enable/download the disabled scenario.
func TestUpgradeItemInDisabledScenarioShouldNotBeInstalled(t *testing.T) {
hub := envSetup(t)
item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
// fresh install of collection
@ -127,7 +125,6 @@ func getHubOrFail(t *testing.T, local *csconfig.LocalHubCfg, remote *RemoteHubCf
// Upgrade should install and enable the newly added scenario.
func TestUpgradeItemNewScenarioIsInstalledWhenReferencedScenarioIsDisabled(t *testing.T) {
hub := envSetup(t)
item := hub.GetItem(COLLECTIONS, "crowdsecurity/test_collection")
// fresh install of collection

2
pkg/cwhub/sync.go
View file

@ -7,13 +7,13 @@ import (
"io"
"os"
"path/filepath"
"slices"
"sort"
"strings"
"github.com/Masterminds/semver/v3"
"github.com/sirupsen/logrus"
"gopkg.in/yaml.v3"
"slices"
)
func isYAMLFileName(path string) bool {

10
test/bats/20_hub.bats
View file

@ -69,6 +69,16 @@ teardown() {
assert_output --partial 'crowdsecurity/iptables'
}
@test "cscli hub list (invalid index)" {
new_hub=$(jq <"$INDEX_PATH" '."appsec-rules"."crowdsecurity/vpatch-laravel-debug-mode".version="999"')
echo "$new_hub" >"$INDEX_PATH"
rune -0 cscli hub list --error
assert_stderr --partial "invalid hub item appsec-rules:crowdsecurity/vpatch-laravel-debug-mode: latest version missing from index"
rune -1 cscli appsec-rules install crowdsecurity/vpatch-laravel-debug-mode --force
assert_stderr --partial "error while installing 'crowdsecurity/vpatch-laravel-debug-mode': while downloading crowdsecurity/vpatch-laravel-debug-mode: latest hash missing from index"
}
@test "missing reference in hub index" {
new_hub=$(jq <"$INDEX_PATH" 'del(.parsers."crowdsecurity/smb-logs") | del (.scenarios."crowdsecurity/mysql-bf")')
echo "$new_hub" >"$INDEX_PATH"