beenull/crowdsec
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Refact cwhub: simplify tree scan and dependency checks (#2600)

Browse source 
* method rename: GetInstalledItemsAsString() -> GetInstalledItemNames()
* use path package
* Comments and method names
* Extract method Item.setVersionState() from Hub.itemVisit()
* refact localSync(), itemVisit() etc.
* fix check for cyclic dependencies, with test
This commit is contained in:
mmetc 2023-11-20 11:41:31 +01:00 committed by GitHub
parent 56ad2bbf98
commit 6b317f0723
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
15 changed files with 254 additions and 207 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
cmd/crowdsec-cli/capi.go
View file

@ -156,7 +156,7 @@ func NewCapiStatusCmd() *cobra.Command {
return err
}
scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("failed to get scenarios: %w", err)
}

2
cmd/crowdsec-cli/console.go
View file

@ -76,7 +76,7 @@ After running this command your will need to validate the enrollment in the weba
return err
}
scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("failed to get installed scenarios: %s", err)
}

2
cmd/crowdsec-cli/item_suggest.go
View file

@ -61,7 +61,7 @@ func compInstalledItems(itemType string, args []string, toComplete string) ([]st
return nil, cobra.ShellCompDirectiveDefault
}
items, err := hub.GetInstalledItemsAsString(itemType)
items, err := hub.GetInstalledItemNames(itemType)
if err != nil {
cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
return nil, cobra.ShellCompDirectiveDefault

2
cmd/crowdsec-cli/lapi.go
View file

@ -43,7 +43,7 @@ func runLapiStatus(cmd *cobra.Command, args []string) error {
log.Fatal(err)
}
scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
log.Fatalf("failed to get scenarios : %s", err)
}

2
cmd/crowdsec-cli/support.go
View file

@ -174,7 +174,7 @@ func collectAPIStatus(login string, password string, endpoint string, prefix str
if err != nil {
return []byte(fmt.Sprintf("cannot parse API URL: %s", err))
}
scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return []byte(fmt.Sprintf("could not collect scenarios: %s", err))
}

4
cmd/crowdsec/output.go
View file

@ -71,7 +71,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
var cache []types.RuntimeAlert
var cacheMutex sync.Mutex
scenarios, err := hub.GetInstalledItemsAsString(cwhub.SCENARIOS)
scenarios, err := hub.GetInstalledItemNames(cwhub.SCENARIOS)
if err != nil {
return fmt.Errorf("loading list of installed hub scenarios: %w", err)
}
@ -94,7 +94,7 @@ func runOutput(input chan types.Event, overflow chan types.Event, buckets *leaky
URL: apiURL,
PapiURL: papiURL,
VersionPrefix: "v1",
UpdateScenario: func() ([]string, error) {return hub.GetInstalledItemsAsString(cwhub.SCENARIOS)},
UpdateScenario: func() ([]string, error) {return hub.GetInstalledItemNames(cwhub.SCENARIOS)},
})
if err != nil {
return fmt.Errorf("new client api: %w", err)

6
pkg/cwhub/dataset.go
View file

@ -18,6 +18,7 @@ type DataSet struct {
Data []types.DataSource `yaml:"data,omitempty"`
}
// downloadFile downloads a file and writes it to disk, with no hash verification
func downloadFile(url string, destPath string) error {
log.Debugf("downloading %s in %s", url, destPath)
@ -37,6 +38,7 @@ func downloadFile(url string, destPath string) error {
}
defer file.Close()
// avoid reading the whole file in memory
_, err = io.Copy(file, resp.Body)
if err != nil {
return err
@ -49,8 +51,8 @@ func downloadFile(url string, destPath string) error {
return nil
}
// downloadData downloads the data files for an item
func downloadData(dataFolder string, force bool, reader io.Reader) error {
// downloadDataSet downloads all the data files for an item
func downloadDataSet(dataFolder string, force bool, reader io.Reader) error {
dec := yaml.NewDecoder(reader)
for {

14
pkg/cwhub/enable.go
View file

@ -10,14 +10,15 @@ import (
log "github.com/sirupsen/logrus"
)
// installLink returns the location of the symlink to the actual config file (eg. /etc/crowdsec/collections/xyz.yaml)
func (i *Item) installLink() string {
// installLink returns the location of the symlink to the downloaded config file
// (eg. /etc/crowdsec/collections/xyz.yaml)
func (i *Item) installLinkPath() string {
return filepath.Join(i.hub.local.InstallDir, i.Type, i.Stage, i.FileName)
}
// makeLink creates a symlink between the actual config file at hub.HubDir and hub.ConfigDir
func (i *Item) createInstallLink() error {
dest, err := filepath.Abs(i.installLink())
dest, err := filepath.Abs(i.installLinkPath())
if err != nil {
return err
}
@ -102,8 +103,9 @@ func (i *Item) purge() error {
return nil
}
// removeInstallLink removes the symlink to the downloaded content
func (i *Item) removeInstallLink() error {
syml, err := filepath.Abs(i.installLink())
syml, err := filepath.Abs(i.installLinkPath())
if err != nil {
return err
}
@ -143,13 +145,13 @@ func (i *Item) removeInstallLink() error {
return nil
}
// disable removes the symlink to the downloaded content, also removes the content if purge is true
// disable removes the install link, and optionally the downloaded content
func (i *Item) disable(purge bool, force bool) error {
// XXX: should return the number of disabled/purged items to inform the upper layer whether to reload or not
err := i.removeInstallLink()
if os.IsNotExist(err) {
if !purge && !force {
return fmt.Errorf("link %s does not exist (override with --force or --purge)", i.installLink())
return fmt.Errorf("link %s does not exist (override with --force or --purge)", i.installLinkPath())
}
} else if err != nil {
return err

62
pkg/cwhub/helpers.go
View file

@ -52,20 +52,46 @@ func (i *Item) Install(force bool, downloadOnly bool) error {
return nil
}
// allDependencies return a list of all dependencies and sub-dependencies of the item
func (i *Item) allDependencies() []*Item {
var deps []*Item
// allDependencies returns a list of all (direct or indirect) dependencies of the item
func (i *Item) allDependencies() ([]*Item, error) {
var collectSubItems func(item *Item, visited map[*Item]bool, result *[]*Item) error
for _, dep := range i.SubItems() {
if dep == i {
log.Errorf("circular dependency detected: %s depends on %s", dep.Name, i.Name)
continue
collectSubItems = func(item *Item, visited map[*Item]bool, result *[]*Item) error {
if item == nil {
return nil
}
deps = append(deps, dep.allDependencies()...)
if visited[item] {
return nil
}
visited[item] = true
for _, subItem := range item.SubItems() {
if subItem == i {
return fmt.Errorf("circular dependency detected: %s depends on %s", item.Name, i.Name)
}
*result = append(*result, subItem)
err := collectSubItems(subItem, visited, result)
if err != nil {
return err
}
}
return nil
}
return append(deps, i)
ret := []*Item{}
visited := map[*Item]bool{}
err := collectSubItems(i, visited, &ret)
if err != nil {
return nil, err
}
return ret, nil
}
// Remove disables the item, optionally removing the downloaded content
@ -85,15 +111,18 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) {
removed := false
allDeps := i.allDependencies()
allDeps, err := i.allDependencies()
if err != nil {
return false, err
}
for _, sub := range i.SubItems() {
if !sub.Installed {
continue
}
// if the other collection(s) are direct or indirect dependencies of the current one, it's good to go
// log parent collections
// if the sub depends on a collection that is not a direct or indirect dependency
// of the current item, it is not removed
for _, subParent := range sub.parentCollections() {
if subParent == i {
continue
@ -113,8 +142,7 @@ func (i *Item) Remove(purge bool, force bool) (bool, error) {
removed = removed || subRemoved
}
err := i.disable(purge, force)
if err != nil {
if err = i.disable(purge, force); err != nil {
return false, fmt.Errorf("while removing %s: %w", i.Name, err)
}
@ -171,7 +199,7 @@ func (i *Item) Upgrade(force bool) (bool, error) {
return updated, nil
}
// downloadLatest will download the latest version of Item to the tdir directory
// downloadLatest downloads the latest version of the item to the hub directory
func (i *Item) downloadLatest(overwrite bool, updateOnly bool) error {
// XXX: should return the path of the downloaded file (taken from download())
log.Debugf("Downloading %s %s", i.Type, i.Name)
@ -314,7 +342,7 @@ func (i *Item) download(overwrite bool) error {
i.Tainted = false
i.UpToDate = true
if err = downloadData(i.hub.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
if err = downloadDataSet(i.hub.local.InstallDataDir, overwrite, bytes.NewReader(body)); err != nil {
return fmt.Errorf("while downloading data for %s: %w", i.FileName, err)
}
@ -332,7 +360,7 @@ func (i *Item) DownloadDataIfNeeded(force bool) error {
defer itemFile.Close()
if err = downloadData(i.hub.local.InstallDataDir, force, itemFile); err != nil {
if err = downloadDataSet(i.hub.local.InstallDataDir, force, itemFile); err != nil {
return fmt.Errorf("while downloading data for %s: %w", itemFilePath, err)
}

31
pkg/cwhub/hub.go
View file

@ -4,6 +4,7 @@ import (
"encoding/json"
"fmt"
"os"
"path"
"strings"
log "github.com/sirupsen/logrus"
@ -12,12 +13,10 @@ import (
)
type Hub struct {
Items HubItems
local *csconfig.LocalHubCfg
remote *RemoteHubCfg
skippedLocal int
skippedTainted int
Warnings []string
Items HubItems
local *csconfig.LocalHubCfg
remote *RemoteHubCfg
Warnings []string
}
func (h *Hub) GetDataDir() string {
@ -82,8 +81,7 @@ func (h *Hub) parseIndex() error {
}
item.Type = itemType
x := strings.Split(item.RemotePath, "/")
item.FileName = x[len(x)-1]
item.FileName = path.Base(item.RemotePath)
item.logMissingSubItems()
}
@ -95,6 +93,8 @@ func (h *Hub) parseIndex() error {
// ItemStats returns total counts of the hub items
func (h *Hub) ItemStats() []string {
loaded := ""
local := 0
tainted := 0
for _, itemType := range ItemTypes {
if len(h.Items[itemType]) == 0 {
@ -102,11 +102,20 @@ func (h *Hub) ItemStats() []string {
}
loaded += fmt.Sprintf("%d %s, ", len(h.Items[itemType]), itemType)
for _, item := range h.Items[itemType] {
if item.IsLocal() {
local++
}
if item.Tainted {
tainted++
}
}
}
loaded = strings.Trim(loaded, ", ")
if loaded == "" {
// empty hub
loaded = "0 items"
}
@ -114,8 +123,8 @@ func (h *Hub) ItemStats() []string {
fmt.Sprintf("Loaded: %s", loaded),
}
if h.skippedLocal > 0 || h.skippedTainted > 0 {
ret = append(ret, fmt.Sprintf("Unmanaged items: %d local, %d tainted", h.skippedLocal, h.skippedTainted))
if local > 0 || tainted > 0 {
ret = append(ret, fmt.Sprintf("Unmanaged items: %d local, %d tainted", local, tainted))
}
return ret

9
pkg/cwhub/items.go
View file

@ -186,6 +186,7 @@ func (i *Item) logMissingSubItems() {
}
}
// parentCollections returns the list of items (collections) that have this item as a direct dependency
func (i *Item) parentCollections() []*Item {
ret := make([]*Item, 0)
@ -281,8 +282,8 @@ func (h *Hub) GetItem(itemType string, itemName string) *Item {
return h.GetItemMap(itemType)[itemName]
}
// GetItemNames returns the list of item (full) names for a given type
// ie. for parsers: crowdsecurity/apache2 crowdsecurity/nginx
// GetItemNames returns the list of (full) item names for a given type
// ie. for collections: crowdsecurity/apache2 crowdsecurity/nginx
// The names can be used to retrieve the item with GetItem()
func (h *Hub) GetItemNames(itemType string) []string {
m := h.GetItemMap(itemType)
@ -335,8 +336,8 @@ func (h *Hub) GetInstalledItems(itemType string) ([]*Item, error) {
return retItems, nil
}
// GetInstalledItemsAsString returns the names of the installed items
func (h *Hub) GetInstalledItemsAsString(itemType string) ([]string, error) {
// GetInstalledItemNames returns the names of the installed items
func (h *Hub) GetInstalledItemNames(itemType string) ([]string, error) {
items, err := h.GetInstalledItems(itemType)
if err != nil {
return nil, err

5
pkg/cwhub/items_test.go
View file

@ -36,7 +36,10 @@ func TestItemStatus(t *testing.T) {
}
stats := hub.ItemStats()
require.Equal(t, []string{"Loaded: 2 parsers, 1 scenarios, 3 collections"}, stats)
require.Equal(t, []string{
"Loaded: 2 parsers, 1 scenarios, 3 collections",
"Unmanaged items: 3 local, 0 tainted",
}, stats)
}
func TestGetters(t *testing.T) {

1
pkg/cwhub/remote.go
View file

@ -17,6 +17,7 @@ type RemoteHubCfg struct {
IndexPath string
}
// urlTo builds the URL to download a file from the remote hub
func (r *RemoteHubCfg) urlTo(remotePath string) (string, error) {
if r == nil {
return "", ErrNilRemoteHub

309
pkg/cwhub/sync.go
View file

@ -19,21 +19,18 @@ func isYAMLFileName(path string) bool {
return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml")
}
func handleSymlink(path string) (string, error) {
// linkTarget returns the target of a symlink, or empty string if it's dangling
func linkTarget(path string) (string, error) {
hubpath, err := os.Readlink(path)
if err != nil {
return "", fmt.Errorf("unable to read symlink of %s", path)
return "", fmt.Errorf("unable to read symlink: %s", path)
}
// the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
log.Tracef("symlink %s -> %s", path, hubpath)
_, err = os.Lstat(hubpath)
if os.IsNotExist(err) {
log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
// remove the symlink
if err = os.Remove(path); err != nil {
return "", fmt.Errorf("failed to unlink %s: %w", path, err)
}
// ignore this file
log.Infof("link target does not exist: %s -> %s", path, hubpath)
return "", nil
}
@ -57,15 +54,15 @@ func getSHA256(filepath string) (string, error) {
}
type itemFileInfo struct {
inhub bool
fname string
stage string
ftype string
fauthor string
}
func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) {
ret := itemFileInfo{}
inhub := false
func (h *Hub) getItemFileInfo(path string) (*itemFileInfo, error) {
var ret *itemFileInfo
hubDir := h.local.HubDir
installDir := h.local.InstallDir
@ -78,37 +75,41 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) {
if strings.HasPrefix(path, hubDir) {
log.Tracef("in hub dir")
inhub = true
//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
//.../hub/scenarios/crowdsec/ssh_bf.yaml
//.../hub/profiles/crowdsec/linux.yaml
if len(subs) < 4 {
return itemFileInfo{}, false, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
}
ret.fname = subs[len(subs)-1]
ret.fauthor = subs[len(subs)-2]
ret.stage = subs[len(subs)-3]
ret.ftype = subs[len(subs)-4]
ret = &itemFileInfo{
inhub: true,
fname: subs[len(subs)-1],
fauthor: subs[len(subs)-2],
stage: subs[len(subs)-3],
ftype: subs[len(subs)-4],
}
} else if strings.HasPrefix(path, installDir) { // we're in install /etc/crowdsec/<type>/...
log.Tracef("in install dir")
if len(subs) < 3 {
return itemFileInfo{}, false, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs))
}
///.../config/parser/stage/file.yaml
///.../config/postoverflow/stage/file.yaml
///.../config/scenarios/scenar.yaml
///.../config/collections/linux.yaml //file is empty
ret.fname = subs[len(subs)-1]
ret.stage = subs[len(subs)-2]
ret.ftype = subs[len(subs)-3]
ret.fauthor = ""
ret = &itemFileInfo{
inhub: false,
fname: subs[len(subs)-1],
stage: subs[len(subs)-2],
ftype: subs[len(subs)-3],
fauthor: "",
}
} else {
return itemFileInfo{}, false, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubDir, installDir)
return nil, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubDir, installDir)
}
log.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype)
// log.Infof("%s -> name:%s stage:%s", path, fname, stage)
if ret.stage == SCENARIOS {
ret.ftype = SCENARIOS
@ -118,15 +119,15 @@ func (h *Hub) getItemInfo(path string) (itemFileInfo, bool, error) {
ret.stage = ""
} else if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS {
// it's a PARSER / POSTOVERFLOW with a stage
return itemFileInfo{}, inhub, fmt.Errorf("unknown configuration type for file '%s'", path)
return nil, fmt.Errorf("unknown configuration type for file '%s'", path)
}
log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype)
return ret, inhub, nil
return ret, nil
}
// sortedVersions returns the input data, sorted in reverse order by semver
// sortedVersions returns the input data, sorted in reverse order (new, old) by semver
func sortedVersions(raw []string) ([]string, error) {
vs := make([]*semver.Version, len(raw))
@ -149,8 +150,22 @@ func sortedVersions(raw []string) ([]string, error) {
return ret, nil
}
func newLocalItem(h *Hub, path string, info *itemFileInfo) *Item {
_, fileName := filepath.Split(path)
return &Item{
hub: h,
Name: info.fname,
Stage: info.stage,
Installed: true,
Type: info.ftype,
LocalPath: path,
UpToDate: true,
FileName: fileName,
}
}
func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
local := false
hubpath := ""
if err != nil {
@ -159,89 +174,59 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
return nil
}
// only happens if the current working directory was removed (!)
path, err = filepath.Abs(path)
if err != nil {
return err
}
// we only care about files
if f == nil || f.IsDir() {
// we only care about YAML files
if f == nil || f.IsDir() || !isYAMLFileName(f.Name()) {
return nil
}
if !isYAMLFileName(f.Name()) {
return nil
}
info, inhub, err := h.getItemInfo(path)
info, err := h.getItemFileInfo(path)
if err != nil {
return err
}
/*
we can encounter 'collections' in the form of a symlink:
/etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml
when the collection is installed, both files are created
*/
// non symlinks are local user files or hub files
if f.Type()&os.ModeSymlink == 0 {
local = true
log.Tracef("%s is not a symlink", path)
log.Tracef("%s isn't a symlink", path)
if !info.inhub {
log.Tracef("%s is a local file, skip", path)
h.Items[info.ftype][info.fname] = newLocalItem(h, path, info)
return nil
}
} else {
hubpath, err = handleSymlink(path)
hubpath, err = linkTarget(path)
if err != nil {
return err
}
log.Tracef("%s points to %s", path, hubpath)
if hubpath == "" {
// ignore this file
// target does not exist, the user might have removed the file
// or switched to a hub branch without it
return nil
}
}
// if it's not a symlink and not in hub, it's a local file, don't bother
if local && !inhub {
log.Tracef("%s is a local file, skip", path)
h.skippedLocal++
_, fileName := filepath.Split(path)
h.Items[info.ftype][info.fname] = &Item{
hub: h,
Name: info.fname,
Stage: info.stage,
Installed: true,
Type: info.ftype,
LocalPath: path,
UpToDate: true,
FileName: fileName,
}
return nil
}
// try to find which configuration item it is
log.Tracef("check [%s] of %s", info.fname, info.ftype)
match := false
for name, item := range h.Items[info.ftype] {
log.Tracef("check [%s] vs [%s]: %s", info.fname, item.RemotePath, info.ftype+"/"+info.stage+"/"+info.fname+".yaml")
if info.fname != item.FileName {
log.Tracef("%s != %s (filename)", info.fname, item.FileName)
continue
}
// wrong stage
if item.Stage != info.stage {
continue
}
// if we are walking hub dir, just mark present files as downloaded
if inhub {
if info.inhub {
// wrong author
if info.fauthor != item.Author {
continue
@ -262,66 +247,9 @@ func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error {
continue
}
sha, err := getSHA256(path)
err := item.setVersionState(path, info.inhub)
if err != nil {
log.Fatalf("Failed to get sha of %s: %v", path, err)
}
// let's reverse sort the versions to deal with hash collisions (#154)
versions := make([]string, 0, len(item.Versions))
for k := range item.Versions {
versions = append(versions, k)
}
versions, err = sortedVersions(versions)
if err != nil {
return fmt.Errorf("while syncing %s %s: %w", info.ftype, info.fname, err)
}
for _, version := range versions {
if item.Versions[version].Digest != sha {
continue
}
// we got an exact match, update struct
item.Downloaded = true
item.LocalHash = sha
if !inhub {
log.Tracef("found exact match for %s, version is %s, latest is %s", item.Name, version, item.Version)
item.LocalPath = path
item.LocalVersion = version
item.Tainted = false
// if we're walking the hub, present file doesn't means installed file
item.Installed = true
}
if version == item.Version {
log.Tracef("%s is up-to-date", item.Name)
item.UpToDate = true
}
match = true
break
}
if !match {
log.Tracef("got tainted match for %s: %s", item.Name, path)
h.skippedTainted++
// the file and the stage is right, but the hash is wrong, it has been tainted by user
if !inhub {
item.LocalPath = path
item.Installed = true
}
item.UpToDate = false
item.LocalVersion = "?"
item.Tainted = true
item.LocalHash = sha
return err
}
h.Items[info.ftype][name] = item
@ -365,11 +293,13 @@ func (h *Hub) checkSubItems(v *Item) error {
if sub.Tainted {
v.Tainted = true
// XXX: improve msg
return fmt.Errorf("tainted %s %s, tainted", sub.Type, sub.Name)
}
if !sub.Installed && v.Installed {
v.Tainted = true
// XXX: improve msg
return fmt.Errorf("missing %s %s, tainted", sub.Type, sub.Name)
}
@ -388,9 +318,8 @@ func (h *Hub) checkSubItems(v *Item) error {
return nil
}
func (h *Hub) syncDir(dir string) ([]string, error) {
warnings := []string{}
// syncDir scans a directory for items, and updates the Hub state accordingly
func (h *Hub) syncDir(dir string) error {
// For each, scan PARSERS, POSTOVERFLOWS, SCENARIOS and COLLECTIONS last
for _, scan := range ItemTypes {
// cpath: top-level item directory, either downloaded or installed items.
@ -408,11 +337,31 @@ func (h *Hub) syncDir(dir string) ([]string, error) {
}
if err = filepath.WalkDir(cpath, h.itemVisit); err != nil {
return warnings, err
return err
}
}
return nil
}
// localSync updates the hub state with downloaded, installed and local items
func (h *Hub) localSync() error {
err := h.syncDir(h.local.InstallDir)
if err != nil {
return fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err)
}
if err = h.syncDir(h.local.HubDir); err != nil {
return fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err)
}
warnings := make([]string, 0)
for _, item := range h.Items[COLLECTIONS] {
if _, err := item.allDependencies(); err != nil {
return err
}
if !item.Installed {
continue
}
@ -434,27 +383,69 @@ func (h *Hub) syncDir(dir string) ([]string, error) {
log.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.LocalVersion, item.Version, item.Versions)
}
return warnings, nil
}
// Updates the info from HubInit() with the local state
func (h *Hub) localSync() error {
h.skippedLocal = 0
h.skippedTainted = 0
h.Warnings = []string{}
warnings, err := h.syncDir(h.local.InstallDir)
if err != nil {
return fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err)
}
h.Warnings = append(h.Warnings, warnings...)
if warnings, err = h.syncDir(h.local.HubDir); err != nil {
return fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err)
}
h.Warnings = append(h.Warnings, warnings...)
h.Warnings = warnings
return nil
}
func (i *Item) setVersionState(path string, inhub bool) error {
var err error
i.LocalHash, err = getSHA256(path)
if err != nil {
return fmt.Errorf("failed to get sha256 of %s: %w", path, err)
}
// let's reverse sort the versions to deal with hash collisions (#154)
versions := make([]string, 0, len(i.Versions))
for k := range i.Versions {
versions = append(versions, k)
}
versions, err = sortedVersions(versions)
if err != nil {
return fmt.Errorf("while syncing %s %s: %w", i.Type, i.FileName, err)
}
i.LocalVersion = "?"
for _, version := range versions {
if i.Versions[version].Digest == i.LocalHash {
i.LocalVersion = version
break
}
}
if i.LocalVersion == "?" {
log.Tracef("got tainted match for %s: %s", i.Name, path)
if !inhub {
i.LocalPath = path
i.Installed = true
}
i.UpToDate = false
i.Tainted = true
return nil
}
// we got an exact match, update struct
i.Downloaded = true
if !inhub {
log.Tracef("found exact match for %s, version is %s, latest is %s", i.Name, i.LocalVersion, i.Version)
i.LocalPath = path
i.Tainted = false
// if we're walking the hub, present file doesn't means installed file
i.Installed = true
}
if i.LocalVersion == i.Version {
log.Tracef("%s is up-to-date", i.Name)
i.UpToDate = true
}
return nil
}

10
test/bats/20_hub_collections_dep.bats
View file

@ -112,3 +112,13 @@ teardown() {
rune -0 cscli parsers list -o json
rune -0 jq -e '.parsers | length == 0' <(output)
}
@test "cscli collections (dependencies IV: looper)" {
hub_dep=$(jq <"$INDEX_PATH" '. * {collections:{"crowdsecurity/sshd":{collections:["crowdsecurity/linux"]}}}')
echo "$hub_dep" >"$INDEX_PATH"
rune -1 cscli hub list
assert_stderr --partial "circular dependency detected"
rune -1 wait-for "${CROWDSEC}"
assert_stderr --partial "circular dependency detected"
}