Parser visit split (#1242)

* break/cleanup this parser_visit function

* do not delete sub-collections if they belong to > 1 collection, fix #1118

cmd/crowdsec-cli/hub.go

@@ -56,7 +56,7 @@ cscli hub update # Download list of available configurations from the hub
 				log.Info(v)
 			}
 			cwhub.DisplaySummary()
-			ListItems([]string{cwhub.PARSERS, cwhub.COLLECTIONS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW}, args, true, false)
+			ListItems([]string{cwhub.PARSERS, cwhub.COLLECTIONS, cwhub.SCENARIOS, cwhub.PARSERS_OVFLW, cwhub.DATA_FILES}, args, true, false)
 		},
 	}
 	cmdHubList.PersistentFlags().BoolVarP(&all, "all", "a", false, "List disabled items as well")

cmd/crowdsec-cli/utils.go

@@ -250,22 +250,21 @@ func UpgradeConfig(itemType string, name string, force bool) {
 			continue
 		}
 
+		found = true
+		if v.UpToDate || v.Tainted {
+			if v.UpToDate {
+				log.Infof("%s : up-to-date", v.Name)
+			}
+			if err = cwhub.DownloadDataIfNeeded(csConfig.Hub, v, force); err != nil {
+				log.Fatalf("%s : download failed : %v", v.Name, err)
+			}
+		}
+
 		if !v.Downloaded {
 			log.Warningf("%s : not downloaded, please install.", v.Name)
 			continue
 		}
 
-		found = true
-		if v.UpToDate {
-			log.Infof("%s : up-to-date", v.Name)
-
-			if !force {
-				if err = cwhub.DownloadDataIfNeeded(csConfig.Hub, v, false); err != nil {
-					log.Fatalf("%s : download failed : %v", v.Name, err)
-				}
-				continue
-			}
-		}
 		v, err = cwhub.DownloadLatest(csConfig.Hub, v, force, true)
 		if err != nil {
 			log.Fatalf("%s : download failed : %v", v.Name, err)

config/config.yaml

@@ -17,6 +17,7 @@ crowdsec_service:
   acquisition_path: /etc/crowdsec/acquis.yaml
   parser_routines: 1
 cscli:
+  hub_branch: datafiles_in_hub
   output: human
 db_config:
   log_level: info

pkg/cwhub/cwhub.go

@@ -3,6 +3,7 @@ package cwhub
 import (
 	"crypto/sha256"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	//"errors"
@@ -22,11 +23,12 @@ var PARSERS = "parsers"
 var PARSERS_OVFLW = "postoverflows"
 var SCENARIOS = "scenarios"
 var COLLECTIONS = "collections"
-var ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS}
+var DATA_FILES = "data_files"
+var ItemTypes = []string{PARSERS, PARSERS_OVFLW, SCENARIOS, COLLECTIONS, DATA_FILES}
 
 var hubIdx map[string]map[string]Item
 
-var RawFileURLTemplate = "https://hub-cdn.crowdsec.net/%s/%s"
+var RawFileURLTemplate = "https://raw.githubusercontent.com/crowdsecurity/hub/%s/%s"
 var HubBranch = "master"
 var HubIndexFile = ".index.json"
 
@@ -57,7 +59,6 @@ type Item struct {
 	BelongsToCollections []string `yaml:"belongs_to_collections,omitempty" json:"belongs_to_collections,omitempty"` /*if it's part of collections, track name here*/
 
 	/*remote (hub) infos*/
-	RemoteURL  string                 `yaml:"remoteURL,omitempty" json:"remoteURL,omitempty"` //the full remote uri of file in http
 	RemotePath string                 `json:"path,omitempty" yaml:"remote_path,omitempty"` //the path relative to git ie. /parsers/stage/author/file.yaml
 	RemoteHash string                 `yaml:"hash,omitempty" json:"hash,omitempty"`        //the meow
 	Version    string                 `json:"version,omitempty"`                           //the last version
@@ -81,6 +82,55 @@ type Item struct {
 	Collections   []string `yaml:"collections,omitempty" json:"collections,omitempty"`
 }
 
+func (i *Item) compareFile(fname, fstage, fauthor, path string) bool {
+	//wrong filename
+	if fname != i.FileName {
+		log.Tracef("%s != %s (filename)", fname, i.FileName)
+		return false
+	}
+	//wrong stage
+	if fstage != "" && fstage != i.Stage {
+		log.Tracef("%s != %s (stage)", fstage, i.Stage)
+		return false
+	}
+	//wrong author
+	if fauthor != "" && fauthor != i.Author {
+		log.Tracef("%s != %s (author)", fauthor, i.Author)
+		return false
+	}
+	return true
+}
+
+func (i *Item) getVersion(path string) (string, string, bool, error) {
+	//returns version, hash, up-to-date flag, error
+	sha, err := getSHA256(path)
+	if err != nil {
+		return "", "", false, err
+	}
+
+	//let's reverse sort the versions to deal with hash collisions (#154)
+	versions := make([]string, 0, len(i.Versions))
+	for k := range i.Versions {
+		versions = append(versions, k)
+	}
+	sort.Sort(sort.Reverse(sort.StringSlice(versions)))
+
+	for _, version := range versions {
+		val := i.Versions[version]
+		if sha != val.Digest {
+			continue
+		}
+		//if the version is == item.version, it means it's the last available version
+		if version == i.Version {
+			return version, sha, true, nil
+		}
+		//otherwise, return the version nontheless
+		return version, sha, false, nil
+	}
+	//the version is unknown
+	return "", sha, false, nil
+}
+
 func (i *Item) toHubStatus() ItemHubStatus {
 	hubStatus := ItemHubStatus{}
 	hubStatus.Name = i.Name
@@ -201,8 +251,8 @@ func AddItem(itemType string, item Item) error {
 }
 
 func DisplaySummary() {
-	log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]),
-		len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]))
+	log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers, %d data files", len(hubIdx[COLLECTIONS]),
+		len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]), len(hubIdx[DATA_FILES]))
 	if skippedLocal > 0 || skippedTainted > 0 {
 		log.Printf("unmanaged items : %d local, %d tainted", skippedLocal, skippedTainted)
 	}
@@ -266,7 +316,6 @@ func GetUpstreamInstalledScenarios() ([]Item, error) {
 func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus {
 	if _, ok := hubIdx[itemType]; !ok {
 		log.Errorf("type %s doesn't exist", itemType)
-
 		return nil
 	}
 

pkg/cwhub/cwhub_test.go

@@ -214,6 +214,7 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 	if !hubIdx[item.Type][item.Name].Installed {
 		t.Fatalf("install: %s should be install", item.Name)
 	}
+
 }
 
 func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
@@ -332,6 +333,49 @@ func TestInstallParser(t *testing.T) {
 	}
 }
 
+func TestGetItemByPath(t *testing.T) {
+	cfg := test_prepenv()
+
+	if err := GetHubIdx(cfg.Hub); err != nil {
+		t.Fatalf("failed to load hub index")
+	}
+	for _, it := range hubIdx[PARSERS] {
+		//Install the parser
+		item, err := DownloadLatest(cfg.Hub, it, false, false)
+		if err != nil {
+			t.Fatalf("error while downloading %s : %v", item.Name, err)
+		}
+		//Enable the item
+		EnableItem(cfg.Hub, item)
+	}
+	//Sync
+	err, warns := LocalSync(cfg.Hub)
+	if err != nil || len(warns) > 0 {
+		t.Fatalf("unexpected err/warnings : %+v / %+v", err, warns)
+	}
+
+	for _, item := range hubIdx[PARSERS] {
+		//Check that we get the same thing if we get it by the path
+		itemcopy, err := GetItemByPath(item.Type, item.LocalPath)
+		if err != nil {
+			t.Fatalf("error while getting item by path : %v -> %s", err, item.LocalPath)
+		}
+		if itemcopy.Name != item.Name {
+			t.Fatalf("GetItemByPath: %s != %s", itemcopy.Name, item.Name)
+		}
+		if itemcopy.UpToDate != item.UpToDate {
+			t.Fatalf("GetItemByPath: %v != %v", itemcopy.UpToDate, item.UpToDate)
+		}
+		if itemcopy.Installed != item.Installed {
+			t.Fatalf("GetItemByPath: %v != %v", itemcopy.Installed, item.Installed)
+		}
+		if itemcopy.Tainted != item.Tainted {
+			t.Fatalf("GetItemByPath: %v != %v", itemcopy.Tainted, item.Tainted)
+		}
+	}
+
+}
+
 func TestInstallCollection(t *testing.T) {
 
 	/*
@@ -350,6 +394,7 @@ func TestInstallCollection(t *testing.T) {
 	}
 	//map iteration is random by itself
 	for _, it := range hubIdx[COLLECTIONS] {
+
 		testInstallItem(cfg.Hub, t, it)
 		it = hubIdx[COLLECTIONS][it.Name]
 		testTaintItem(cfg.Hub, t, it)
@@ -384,7 +429,7 @@ func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 	log.Printf("---> %s", req.URL.Path)
 
 	/*FAKE PARSER*/
-	if req.URL.Path == "/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml" {
+	if strings.HasSuffix(req.URL.Path, "/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml") {
 		responseBody = `onsuccess: next_stage
 filter: evt.Parsed.program == 'foobar_parser'
 name: crowdsecurity/foobar_parser
@@ -395,7 +440,7 @@ grok:
   apply_on: message
 `
 
-	} else if req.URL.Path == "/master/parsers/s01-parse/crowdsecurity/foobar_subparser.yaml" {
+	} else if strings.HasSuffix(req.URL.Path, "/master/parsers/s01-parse/crowdsecurity/foobar_subparser.yaml") {
 		responseBody = `onsuccess: next_stage
 filter: evt.Parsed.program == 'foobar_parser'
 name: crowdsecurity/foobar_parser
@@ -407,19 +452,19 @@ grok:
 `
 		/*FAKE SCENARIO*/
 
-	} else if req.URL.Path == "/master/scenarios/crowdsecurity/foobar_scenario.yaml" {
+	} else if strings.HasSuffix(req.URL.Path, "/master/scenarios/crowdsecurity/foobar_scenario.yaml") {
 		responseBody = `filter: true
 name: crowdsecurity/foobar_scenario`
 		/*FAKE COLLECTIONS*/
-	} else if req.URL.Path == "/master/collections/crowdsecurity/foobar.yaml" {
+	} else if strings.HasSuffix(req.URL.Path, "/master/collections/crowdsecurity/foobar.yaml") {
 		responseBody = `
 blah: blalala
 qwe: jejwejejw`
-	} else if req.URL.Path == "/master/collections/crowdsecurity/foobar_subcollection.yaml" {
+	} else if strings.HasSuffix(req.URL.Path, "/master/collections/crowdsecurity/foobar_subcollection.yaml") {
 		responseBody = `
 blah: blalala
 qwe: jejwejejw`
-	} else if req.URL.Path == "/master/.index.json" {
+	} else if strings.HasSuffix(req.URL.Path, "/master/.index.json") {
 		responseBody =
 			`{
 				"collections": {

pkg/cwhub/download.go

@@ -3,15 +3,17 @@ package cwhub
 import (
 	"bytes"
 	"crypto/sha256"
+	"io"
 	"path"
 	"path/filepath"
+	"sort"
 
 	//"errors"
 	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
 
 	//"errors"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"net/http"
 	"os"
@@ -20,7 +22,6 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/csconfig"
 	"github.com/crowdsecurity/crowdsec/pkg/types"
 	log "github.com/sirupsen/logrus"
-	"gopkg.in/yaml.v2"
 )
 
 func UpdateHubIdx(hub *csconfig.Hub) error {
@@ -79,7 +80,6 @@ func DownloadHubIdx(hub *csconfig.Hub) ([]byte, error) {
 func DownloadLatest(hub *csconfig.Hub, target Item, overwrite bool, updateOnly bool) (Item, error) {
 	var err error
 
-	log.Debugf("Downloading %s %s", target.Type, target.Name)
 	if target.Type == COLLECTIONS {
 		var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
 		for idx, ptr := range tmp {
@@ -143,10 +143,19 @@ func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error)
 			return target, nil
 		}
 		if target.UpToDate {
-			log.Debugf("%s : up-to-date, not updated", target.Name)
 			//  We still have to check if data files are present
+			log.Debugf("%s : up-to-date, not updated", target.Name)
+			data, err := os.ReadFile(target.LocalPath)
+			if err != nil {
+				return target, errors.Wrapf(err, "reading file '%s'", target.LocalPath)
+			}
+			if err := downloadData(dataFolder, target.Author, overwrite, bytes.NewReader(data)); err != nil {
+				return target, errors.Wrapf(err, "while downloading data for %s", target.FileName)
+			}
+			return target, nil
 		}
 	}
+	log.Debugf("Downloading %s %s", target.Type, target.Name)
 	req, err := http.NewRequest("GET", fmt.Sprintf(RawFileURLTemplate, HubBranch, target.RemotePath), nil)
 	if err != nil {
 		return target, errors.Wrap(err, fmt.Sprintf("while downloading %s", req.URL.String()))
@@ -214,7 +223,7 @@ func DownloadItem(hub *csconfig.Hub, target Item, overwrite bool) (Item, error)
 	target.Tainted = false
 	target.UpToDate = true
 
-	if err = downloadData(dataFolder, overwrite, bytes.NewReader(body)); err != nil {
+	if err = downloadData(dataFolder, target.Author, overwrite, bytes.NewReader(body)); err != nil {
 		return target, errors.Wrapf(err, "while downloading data for %s", target.FileName)
 	}
 
@@ -232,16 +241,15 @@ func DownloadDataIfNeeded(hub *csconfig.Hub, target Item, force bool) error {
 	if itemFile, err = os.Open(itemFilePath); err != nil {
 		return errors.Wrapf(err, "while opening %s", itemFilePath)
 	}
-	if err = downloadData(dataFolder, force, itemFile); err != nil {
+	if err = downloadData(dataFolder, target.Author, force, itemFile); err != nil {
 		return errors.Wrapf(err, "while downloading data for %s", itemFilePath)
 	}
 	return nil
 }
 
-func downloadData(dataFolder string, force bool, reader io.Reader) error {
+func downloadData(dataFolder string, parentItemAuthor string, force bool, reader io.Reader) error {
 	var err error
 	dec := yaml.NewDecoder(reader)
-
 	for {
 		data := &types.DataSet{}
 		err = dec.Decode(data)
@@ -252,20 +260,59 @@ func downloadData(dataFolder string, force bool, reader io.Reader) error {
 			break
 		}
 
-		download := false
-		if !force {
 		for _, dataS := range data.Data {
-				if _, err := os.Stat(path.Join(dataFolder, dataS.DestPath)); os.IsNotExist(err) {
+			download := false
+			dfPath := path.Join(dataFolder, dataS.DestPath)
+			dataFileName := strings.Split(dataS.DestPath, ".")[0]
+			_, downloadFromHub := hubIdx[DATA_FILES][dataFileName]
+			if downloadFromHub {
+				dataS.SourceURL = fmt.Sprintf(RawFileURLTemplate, HubBranch, hubIdx[DATA_FILES][dataFileName].RemotePath)
+			}
+
+			if _, err := os.Stat(dfPath); os.IsNotExist(err) {
 				download = true
+			} else if downloadFromHub {
+				sha, err := getSHA256(dfPath)
+				if err != nil {
+					return err
 				}
+				download = dataFileHasUpdates(sha, dataFileName)
 			}
-		}
+
+			log.Infof("%v has updates=%v", dataFileName, download)
+
 			if download || force {
-			err = types.GetData(data.Data, dataFolder)
+				err = types.GetData(dataS, dataFolder)
 				if err != nil {
 					return errors.Wrap(err, "while getting data")
 				}
 			}
+
 		}
+	}
+
 	return nil
 }
+
+// Checks if the provided data file is latest. Only files which  are available in hub should
+// be checked for.
+func dataFileHasUpdates(fileSha string, dataFileName string) bool {
+	dataItem := hubIdx[DATA_FILES][dataFileName]
+	versions := make([]string, 0, len(dataItem.Versions))
+	for k := range dataItem.Versions {
+		versions = append(versions, k)
+	}
+	sort.Sort(sort.Reverse(sort.StringSlice(versions)))
+	for i, version := range versions {
+		if fileSha != dataItem.Versions[version].Digest {
+			continue
+		}
+		log.Debugf("data file %s matched sha with version %s", dataFileName, version)
+		if i != 0 {
+			log.Debugf("data file %s is outdated, updating to version %s", dataFileName, versions[0])
+			return true
+		}
+		break
+	}
+	return false
+}

pkg/cwhub/download_test.go

@@ -40,3 +40,29 @@ func TestDownloadHubIdx(t *testing.T) {
 	RawFileURLTemplate = back
 	fmt.Printf("->%+v", ret)
 }
+
+func TestDataFileIsLatest(t *testing.T) {
+	dataFileName := "crowdsecurity/sensitive-files"
+	hubIdx = map[string]map[string]Item{
+		"data_files": {
+			"crowdsecurity/sensitive-files": {
+				Versions: map[string]ItemVersion{
+					"0.1": {Digest: "1"},
+					"0.2": {Digest: "2"},
+				},
+			},
+		},
+	}
+	if dataFileHasUpdates("1", dataFileName) {
+		log.Errorf(`expected dataFileIsLatest("1", %s) = true found false `, dataFileName)
+	}
+
+	if !dataFileHasUpdates("2", dataFileName) {
+		log.Errorf(`expected dataFileIsLatest("2", %s) = false found true `, dataFileName)
+	}
+
+	// data file is tainted
+	if dataFileHasUpdates("3", dataFileName) {
+		log.Errorf(`expected dataFileIsLatest("3", %s) = false found true `, dataFileName)
+	}
+}

pkg/cwhub/install.go

@@ -27,17 +27,26 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
 		return target, fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name)
 	}
 
-	/*for a COLLECTIONS, disable sub-items*/
+	/*for a COLLECTIONS, disable sub-items, if they are used _only_ by this item :)*/
 	if target.Type == COLLECTIONS {
 		var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
 		for idx, ptr := range tmp {
 			ptrtype := ItemTypes[idx]
 			for _, p := range ptr {
 				if val, ok := hubIdx[ptrtype][p]; ok {
+
+					if len(val.BelongsToCollections) > 1 {
+						log.Infof("%s belongs to more than one collection, won't disable (%v)", p, val.BelongsToCollections)
+						continue
+					}
+					if len(val.BelongsToCollections) == 1 && val.BelongsToCollections[0] == target.Name {
+						log.Infof("%s belongs exclusively to %s, disabling", p, target.Name)
 						hubIdx[ptrtype][p], err = DisableItem(hub, val, purge, force)
 						if err != nil {
 							return target, errors.Wrap(err, fmt.Sprintf("while disabling %s", p))
 						}
+					}
+
 				} else {
 					log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name)
 				}

pkg/cwhub/loader.go

@@ -5,7 +5,6 @@ import (
 	//"errors"
 	"fmt"
 	"io/ioutil"
-	"sort"
 
 	"github.com/pkg/errors"
 	"golang.org/x/mod/semver"
@@ -20,97 +19,190 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-/*the walk/parser_visit function can't receive extra args*/
-var hubdir, installdir string
+/*the walk function can't receive extra args*/
+var hubdir, installdir, datadir string
 
-func parser_visit(path string, f os.FileInfo, err error) error {
-
-	var target Item
-	var local bool
-	var hubpath string
-	var inhub bool
-	var fname string
-	var ftype string
-	var fauthor string
-	var stage string
-
-	path, err = filepath.Abs(path)
+func visitDiscard(path string, f os.FileInfo) (string, bool, error) {
+	//return path, false, nil
+	path, err := filepath.Abs(path)
 	if err != nil {
-		return err
+		return path, true, err
 	}
 	//we only care about files
 	if f == nil || f.IsDir() {
-		return nil
+		return path, true, nil
 	}
-	//we only care about yaml files
-	if !strings.HasSuffix(f.Name(), ".yaml") && !strings.HasSuffix(f.Name(), ".yml") {
-		return nil
+	return path, false, nil
 }
 
-	subs := strings.Split(path, "/")
+func hubdirVisit(path string, f os.FileInfo, err error) error {
 
-	log.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubdir, installdir)
-	/*we're in hub (~/.hub/hub/)*/
-	if strings.HasPrefix(path, hubdir) {
-		log.Tracef("in hub dir")
-		inhub = true
+	if err != nil {
+		log.Warningf("error visiting %s", err)
+	}
+	allowed_extensions := map[string]bool{".yaml": true, ".yml": true}
+	/*only interested by yaml files */
+	path, discard, err := visitDiscard(path, f)
+	if err != nil {
+		return err
+	}
+	if discard {
+		return nil
+	}
+	if !allowed_extensions[filepath.Ext(path)] {
+		log.Debugf("discarding %s : not a yaml file", path)
+		return nil
+	}
+	//extract components from path :
 	//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
 	//.../hub/scenarios/crowdsec/ssh_bf.yaml
 	//.../hub/profiles/crowdsec/linux.yaml
-		if len(subs) < 4 {
-			log.Fatalf("path is too short : %s (%d)", path, len(subs))
+	path_components := strings.Split(path, string(filepath.Separator))
+
+	if len(path_components) < 4 {
+		log.Fatalf("path is too short : %s (%d)", path, len(path_components))
 	}
-		fname = subs[len(subs)-1]
-		fauthor = subs[len(subs)-2]
-		stage = subs[len(subs)-3]
-		ftype = subs[len(subs)-4]
-	} else if strings.HasPrefix(path, installdir) { /*we're in install /etc/crowdsec/<type>/... */
-		log.Tracef("in install dir")
-		if len(subs) < 3 {
-			log.Fatalf("path is too short : %s (%d)", path, len(subs))
+	fname := path_components[len(path_components)-1]
+	fauthor := path_components[len(path_components)-2]
+	fstage := path_components[len(path_components)-3]
+	ftype := path_components[len(path_components)-4]
+
+	log.Tracef("%s : stage:%s ftype:%s", path, fstage, ftype)
+
+	if ftype == DATA_FILES {
+		return fmt.Errorf("unexpected data file in hub : %s", path)
+	}
+
+	// correct the stage and type for non-stage stuff.
+	if fstage == SCENARIOS {
+		ftype = SCENARIOS
+		fstage = ""
+	} else if fstage == COLLECTIONS {
+		ftype = COLLECTIONS
+		fstage = ""
+	} else if ftype != PARSERS && ftype != PARSERS_OVFLW { /*its a PARSER / PARSER_OVFLW with a stage */
+		return fmt.Errorf("unknown configuration type for file '%s'", path)
+	}
+
+	log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, fstage, ftype)
+
+	//in the hub, we don't expect symlinks
+	if f.Mode()&os.ModeSymlink != 0 {
+		log.Warningf("%s in the hub is a symlink, this isn't expected", path)
+	}
+	//try to find which configuration item it is
+	log.Tracef("check [%s] of %s", fname, ftype)
+
+	for itemName, item := range hubIdx[ftype] {
+		log.Tracef("check [%s] vs [%s] : %s/%s/%s", fname, item.RemotePath, ftype, fstage, fname)
+		if !item.compareFile(fname, fstage, fauthor, path) {
+			continue
+		}
+		//we're in the hub, mark the file as present and downloaded
+		if path == hubdir+"/"+item.RemotePath { //
+			log.Tracef("marking %s as downloaded", item.Name)
+			item.Downloaded = true
+		}
+
+		version, sha, uptodate, err := item.getVersion(path)
+		if err != nil {
+			return errors.Wrapf(err, "while getting version of %s", path)
+		}
+
+		if version == "" {
+			log.Debugf("got tainted match for %s : %s", item.Name, path)
+			skippedTainted += 1
+			item.UpToDate = uptodate
+			item.LocalVersion = "?"
+			item.Tainted = true
+			item.LocalHash = sha
+		} else {
+			item.UpToDate = uptodate
+		}
+		//if it was not present, update the index (it's the first time we're seeing this item. Might be downloaded and not installed)
+		if _, ok := hubIdx[ftype][itemName]; !ok {
+			hubIdx[ftype][itemName] = item
+		}
+		return nil
+	}
+	log.Infof("File %s found in hub directory wasn't found in the hub index, ignoring it", path)
+	return nil
+}
+
+func configdirVisit(path string, f os.FileInfo, err error) error {
+
+	if err != nil {
+		log.Warningf("error visiting %s", err)
+	}
+	allowed_extensions := map[string]bool{".yaml": true, ".yml": true}
+	/*only interested by yaml files */
+	path, discard, err := visitDiscard(path, f)
+	if err != nil {
+		return err
+	}
+	if discard {
+		return nil
+	}
+	if !allowed_extensions[filepath.Ext(path)] {
+		log.Debugf("discarding %s : not a yaml file", path)
+		return nil
+	}
+	path_components := strings.Split(path, string(filepath.Separator))
+
+	if len(path_components) < 3 {
+		log.Fatalf("path is too short : %s (%d)", path, len(path_components))
 	}
 	///.../config/parser/stage/file.yaml
 	///.../config/postoverflow/stage/file.yaml
 	///.../config/scenarios/scenar.yaml
 	///.../config/collections/linux.yaml //file is empty
-		fname = subs[len(subs)-1]
-		stage = subs[len(subs)-2]
-		ftype = subs[len(subs)-3]
-		fauthor = ""
-	} else {
-		return fmt.Errorf("File '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubdir, installdir)
+	fname := path_components[len(path_components)-1]
+	fstage := path_components[len(path_components)-2]
+	ftype := path_components[len(path_components)-3]
+
+	if ftype == DATA_FILES {
+		return fmt.Errorf("unexpected data file in install directory : %s", path)
 	}
 
-	log.Tracef("stage:%s ftype:%s", stage, ftype)
-	//log.Printf("%s -> name:%s stage:%s", path, fname, stage)
-	if stage == SCENARIOS {
+	log.Tracef("stage:%s ftype:%s", fstage, ftype)
+
+	// correct the stage and type for non-stage stuff.
+	if fstage == SCENARIOS {
 		ftype = SCENARIOS
-		stage = ""
-	} else if stage == COLLECTIONS {
+		fstage = ""
+	} else if fstage == COLLECTIONS {
 		ftype = COLLECTIONS
-		stage = ""
-	} else if ftype != PARSERS && ftype != PARSERS_OVFLW /*its a PARSER / PARSER_OVFLW with a stage */ {
+		fstage = ""
+	} else if ftype != PARSERS && ftype != PARSERS_OVFLW { /*its a PARSER / PARSER_OVFLW with a stage */
 		return fmt.Errorf("unknown configuration type for file '%s'", path)
 	}
+	log.Tracef("CORRECTED [%s] in stage [%s] of type [%s]", fname, fstage, ftype)
 
-	log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype)
-
-	/*
-		we can encounter 'collections' in the form of a symlink :
-		/etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml
-		when the collection is installed, both files are created
-	*/
-	//non symlinks are local user files or hub files
+	//non symlinks are local user files or hub files or data files
 	if f.Mode()&os.ModeSymlink == 0 {
-		local = true
 		log.Tracef("%s isn't a symlink", path)
-	} else {
-		hubpath, err = os.Readlink(path)
+		local_item := Item{}
+		log.Tracef("%s is a local file, skip", path)
+		skippedLocal++
+		local_item.Name = fname
+		local_item.Stage = fstage
+		local_item.Installed = true
+		local_item.Type = ftype
+		local_item.Local = true
+		local_item.LocalPath = path
+		local_item.UpToDate = true
+		x := strings.Split(path, string(filepath.Separator))
+		local_item.FileName = x[len(x)-1]
+		hubIdx[ftype][fname] = local_item
+		return nil
+
+	}
+	hubpath, err := os.Readlink(path)
 	if err != nil {
 		return fmt.Errorf("unable to read symlink of %s", path)
 	}
-		//the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
-		_, err := os.Lstat(hubpath)
+	//the symlink target doesn't exist, user might have removed hub directory without deleting /etc/crowdsec/....yaml
+	_, err = os.Lstat(hubpath)
 	if os.IsNotExist(err) {
 		log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
 		//remove the symlink
@@ -119,126 +211,118 @@ func parser_visit(path string, f os.FileInfo, err error) error {
 		}
 		return nil
 	}
-		log.Tracef("%s points to %s", path, hubpath)
+
+	//try to get the matching item version
+	for itemName, item := range hubIdx[ftype] { // eg ftype = "collections", k = crowdsecurity/nginx, v is an Item struct
+		if !item.compareFile(fname, fstage, "", path) {
+			continue
 		}
-
-	//if it's not a symlink and not in hub, it's a local file, don't bother
-	if local && !inhub {
-		log.Tracef("%s is a local file, skip", path)
-		skippedLocal++
-		//	log.Printf("local scenario, skip.")
-		target.Name = fname
-		target.Stage = stage
-		target.Installed = true
-		target.Type = ftype
-		target.Local = true
-		target.LocalPath = path
-		target.UpToDate = true
-		x := strings.Split(path, "/")
-		target.FileName = x[len(x)-1]
-
-		hubIdx[ftype][fname] = target
+		log.Tracef("check [%s] vs [%s] : %s", fname, item.RemotePath, ftype+"/"+fstage+"/"+fname+".yaml")
+		version, sha, uptodate, err := item.getVersion(path)
+		if err != nil {
+			return errors.Wrapf(err, "while getting version of %s", path)
+		}
+		item.LocalPath = path
+		item.Installed = true
+		item.LocalHash = sha
+		log.Debugf("found exact match for %s : version is %s (up-to-date:%t)", path, version, uptodate)
+		/*we found the matching item, update it*/
+		if version != "" {
+			item.LocalVersion = version
+			item.Tainted = false
+			item.Downloaded = true
+			item.UpToDate = uptodate
+		} else {
+			skippedTainted += 1
+			//the file and the stage is right, but the hash is wrong, it has been tainted by user
+			item.UpToDate = false
+			item.LocalVersion = "?"
+			item.Tainted = true
+		}
+		hubIdx[ftype][itemName] = item
 		return nil
 	}
+	log.Warningf("File %s found in install directory wasn't accounted for : not a symlink to hub, not a local file", path)
+	return nil
+}
+
+func datadirVisit(path string, f os.FileInfo, err error) error {
+
+	if err != nil {
+		log.Warningf("error visiting %s", err)
+	}
+
+	/*only interested by yaml files */
+	path, discard, err := visitDiscard(path, f)
+	if err != nil {
+		return err
+	}
+	if discard {
+		return nil
+	}
+
+	path_components := strings.Split(path, string(filepath.Separator))
+
+	if len(path_components) < 2 {
+		log.Fatalf("path is too short : %s (%d)", path, len(path_components))
+	}
+	fname := path_components[len(path_components)-1]
+	fauthor := path_components[len(path_components)-2]
+	ftype := DATA_FILES
+
+	log.Tracef("CORRECTED [%s] by [%s] of type [%s]", fname, fauthor, ftype)
+
+	//non symlinks are local user files or hub files or data files
+	if f.Mode()&os.ModeSymlink != 0 {
+		log.Warningf("%s is a symlink, that's unexpected (but can be ok)", path)
+		final_path, err := os.Readlink(path)
+		if err != nil {
+			return fmt.Errorf("unable to read symlink of %s", path)
+		}
+		//the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
+		_, err = os.Lstat(path)
+		if os.IsNotExist(err) {
+			log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, final_path)
+			if err = os.Remove(path); err != nil {
+				return fmt.Errorf("failed to unlink %s: %+v", path, err)
+			}
+			return nil
+		}
+	}
+
 	//try to find which configuration item it is
 	log.Tracef("check [%s] of %s", fname, ftype)
 
-	match := false
-	for k, v := range hubIdx[ftype] {
-		log.Tracef("check [%s] vs [%s] : %s", fname, v.RemotePath, ftype+"/"+stage+"/"+fname+".yaml")
-		if fname != v.FileName {
-			log.Tracef("%s != %s (filename)", fname, v.FileName)
+	for itemName, item := range hubIdx[ftype] { // eg ftype = "collections", k = crowdsecurity/nginx, v is an Item struct
+		log.Tracef("check [%s] vs [%s]", fname, item.RemotePath)
+		if !item.compareFile(fname, "", fauthor, path) {
 			continue
 		}
-		//wrong stage
-		if v.Stage != stage {
-			continue
-		}
-		/*if we are walking hub dir, just mark present files as downloaded*/
-		if inhub {
-			//wrong author
-			if fauthor != v.Author {
-				continue
-			}
-			//wrong file
-			if v.Name+".yaml" != fauthor+"/"+fname {
-				continue
-			}
-			if path == hubdir+"/"+v.RemotePath {
-				log.Tracef("marking %s as downloaded", v.Name)
-				v.Downloaded = true
-			}
-		} else {
-			//wrong file
-			//<type>/<stage>/<author>/<name>.yaml
-			if !strings.HasSuffix(hubpath, v.RemotePath) {
-				//log.Printf("wrong file %s %s", hubpath, spew.Sdump(v))
 
-				continue
-			}
-		}
-		sha, err := getSHA256(path)
+		version, sha, uptodate, err := item.getVersion(path)
 		if err != nil {
-			log.Fatalf("Failed to get sha of %s : %v", path, err)
+			return errors.Wrapf(err, "while getting version of %s", path)
 		}
-		//let's reverse sort the versions to deal with hash collisions (#154)
-		versions := make([]string, 0, len(v.Versions))
-		for k := range v.Versions {
-			versions = append(versions, k)
-		}
-		sort.Sort(sort.Reverse(sort.StringSlice(versions)))
-
-		for _, version := range versions {
-			val := v.Versions[version]
-			if sha != val.Digest {
-				//log.Printf("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v))
-				continue
-			}
-			/*we got an exact match, update struct*/
-			if !inhub {
-				log.Tracef("found exact match for %s, version is %s, latest is %s", v.Name, version, v.Version)
-				v.LocalPath = path
-				v.LocalVersion = version
-				v.Tainted = false
-				v.Downloaded = true
-				/*if we're walking the hub, present file doesn't means installed file*/
-				v.Installed = true
-				v.LocalHash = sha
-				x := strings.Split(path, "/")
-				target.FileName = x[len(x)-1]
-			}
-			if version == v.Version {
-				log.Tracef("%s is up-to-date", v.Name)
-				v.UpToDate = true
-			}
-			match = true
-			break
-		}
-		if !match {
-			log.Tracef("got tainted match for %s : %s", v.Name, path)
+		item.LocalPath = path
+		item.Installed = true
+		item.LocalHash = sha
+		log.Debugf("DATA [%s] found exact match for %s : version is %s (up-to-date:%t)", itemName, path, version, uptodate)
+		if version == "" {
 			skippedTainted += 1
-			//the file and the stage is right, but the hash is wrong, it has been tainted by user
-			if !inhub {
-				v.LocalPath = path
-				v.Installed = true
-			}
-			v.UpToDate = false
-			v.LocalVersion = "?"
-			v.Tainted = true
-			v.LocalHash = sha
-			x := strings.Split(path, "/")
-			target.FileName = x[len(x)-1]
-
-		}
-		//update the entry if appropriate
-		if _, ok := hubIdx[ftype][k]; !ok {
-			hubIdx[ftype][k] = v
-		} else if !inhub {
-			hubIdx[ftype][k] = v
+			item.UpToDate = uptodate
+			item.LocalVersion = "?"
+			item.Tainted = true
+			item.LocalHash = sha
+		} else {
+			item.UpToDate = uptodate
+			item.LocalVersion = version
+			item.LocalHash = sha
 		}
+		hubIdx[ftype][itemName] = item
 		return nil
 	}
-	log.Infof("Ignoring file %s of type %s", path, ftype)
+	log.Debugf("Ignoring file %s of type %s", path, ftype)
+
 	return nil
 }
 
@@ -267,7 +351,7 @@ func CollecDepsCheck(v *Item) error {
 				if val.Type == COLLECTIONS {
 					log.Tracef("collec, recurse.")
 					if err := CollecDepsCheck(&val); err != nil {
-						return fmt.Errorf("sub collection %s is broken : %s", val.Name, err)
+						return fmt.Errorf("sub collection %s warning : %s", val.Name, err)
 					}
 					hubIdx[ptrtype][p] = val
 				}
@@ -305,23 +389,48 @@ func CollecDepsCheck(v *Item) error {
 func SyncDir(hub *csconfig.Hub, dir string) (error, []string) {
 	hubdir = hub.HubDir
 	installdir = hub.ConfigDir
+	datadir = hub.DataDir
 	warnings := []string{}
 
-	/*For each, scan PARSERS, PARSERS_OVFLW, SCENARIOS and COLLECTIONS last*/
-	for _, scan := range ItemTypes {
-		cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
+	//data_dir is quite simple : there is no collections and such
+	if dir == hub.DataDir {
+		var cpath string
+		var err error
+		cpath, err = filepath.Abs(hub.DataDir)
 		if err != nil {
 			log.Errorf("failed %s : %s", cpath, err)
 		}
-		err = filepath.Walk(cpath, parser_visit)
-		if err != nil {
+		err = filepath.Walk(cpath, datadirVisit)
 		return err, warnings
 	}
 
+	/*For each, scan PARSERS, PARSERS_OVFLW, DATA_FILES, SCENARIOS and COLLECTIONS last*/
+	for _, scan := range ItemTypes {
+		var cpath string
+		var err error
+
+		cpath, err = filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
+		if err != nil {
+			log.Errorf("failed %s : %s", cpath, err)
+		}
+
+		switch dir {
+		case hub.HubDir:
+			err = filepath.Walk(cpath, hubdirVisit)
+		case hub.ConfigDir:
+			err = filepath.Walk(cpath, configdirVisit)
+		default:
+			log.Fatalf("unexpected dir %s", dir)
+		}
+		if err != nil {
+			return err, warnings
+		}
 	}
 
 	for k, v := range hubIdx[COLLECTIONS] {
-		if v.Installed {
+		if !v.Installed {
+			continue
+		}
 		versStat := GetVersionStatus(&v)
 		if versStat == 0 { //latest
 			if err := CollecDepsCheck(&v); err != nil {
@@ -335,7 +444,6 @@ func SyncDir(hub *csconfig.Hub, dir string) (error, []string) {
 		}
 		log.Debugf("installed (%s) - status:%d | installed:%s | latest : %s | full : %+v", v.Name, semver.Compare("v"+v.Version, "v"+v.LocalVersion), v.LocalVersion, v.Version, v.Versions)
 	}
-	}
 	return nil, warnings
 }
 
@@ -352,6 +460,10 @@ func LocalSync(hub *csconfig.Hub) (error, []string) {
 	if err != nil {
 		return fmt.Errorf("failed to scan %s : %s", hub.HubDir, err), warnings
 	}
+	err, _ = SyncDir(hub, hub.DataDir)
+	if err != nil {
+		return fmt.Errorf("failed to scan %s : %s", hub.DataDir, err), warnings
+	}
 	return nil, warnings
 }
 

pkg/parser/enrich_geoip.go

@@ -87,32 +87,54 @@ func GeoIpCity(field string, p *types.Event, ctx interface{}) (map[string]string
 	return ret, nil
 }
 
+/*deal with data versioning without breaking : support crowdsecurity/xxx or xxx*/
+
 func GeoIPCityInit(cfg map[string]string) (interface{}, error) {
 	dbCityReader, err := geoip2.Open(cfg["datadir"] + "/GeoLite2-City.mmdb")
-	if err != nil {
-		log.Debugf("couldn't open geoip : %v", err)
-		return nil, err
-	}
-
+	if err == nil {
 		return dbCityReader, nil
 	}
+	dbCityReader, err = geoip2.Open(cfg["datadir"] + "/crowdsecurity/GeoLite2-City.mmdb")
+	if err == nil {
+		return dbCityReader, nil
+
+	}
+	log.Debugf("couldn't open geoip (%s nor %s) : %v",
+		cfg["datadir"]+"/crowdsecurity/GeoLite2-City.mmdb",
+		cfg["datadir"]+"/GeoLite2-City.mmdb",
+		err)
+	return nil, err
+}
 
 func GeoIPASNInit(cfg map[string]string) (interface{}, error) {
 	dbASReader, err := geoip2.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
-	if err != nil {
-		log.Debugf("couldn't open geoip : %v", err)
-		return nil, err
-	}
-
+	if err == nil {
 		return dbASReader, nil
+
+	}
+	dbASReader, err = geoip2.Open(cfg["datadir"] + "/crowdsecurity/GeoLite2-ASN.mmdb")
+	if err == nil {
+		return dbASReader, nil
+	}
+	log.Debugf("couldn't open geoip (%s nor %s) : %v",
+		cfg["datadir"]+"/crowdsecurity/GeoLite2-ASN.mmdb",
+		cfg["datadir"]+"/GeoLite2-ASN.mmdb",
+		err)
+	return nil, err
 }
 
 func IpToRangeInit(cfg map[string]string) (interface{}, error) {
 	ipToRangeReader, err := maxminddb.Open(cfg["datadir"] + "/GeoLite2-ASN.mmdb")
-	if err != nil {
-		log.Debugf("couldn't open geoip : %v", err)
-		return nil, err
-	}
-
+	if err == nil {
 		return ipToRangeReader, nil
 	}
+	ipToRangeReader, err = maxminddb.Open(cfg["datadir"] + "/crowdsecurity/GeoLite2-ASN.mmdb")
+	if err == nil {
+		return ipToRangeReader, nil
+	}
+	log.Debugf("couldn't open geoip (%s nor %s) : %v",
+		cfg["datadir"]+"/crowdsecurity/GeoLite2-ASN.mmdb",
+		cfg["datadir"]+"/GeoLite2-ASN.mmdb",
+		err)
+	return nil, err
+}

pkg/types/dataset.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"os"
 	"path"
+	"path/filepath"
 
 	log "github.com/sirupsen/logrus"
 )
@@ -42,6 +43,10 @@ func downloadFile(url string, destPath string) error {
 		return fmt.Errorf("download response 'HTTP %d' : %s", resp.StatusCode, string(body))
 	}
 
+	if err := os.MkdirAll(filepath.Dir(destPath), 0755); err != nil {
+		return err
+	}
+
 	file, err := os.OpenFile(destPath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
 	if err != nil {
 		return err
@@ -60,15 +65,13 @@ func downloadFile(url string, destPath string) error {
 	return nil
 }
 
-func GetData(data []*DataSource, dataDir string) error {
-	for _, dataS := range data {
+func GetData(dataS *DataSource, dataDir string) error {
 	destPath := path.Join(dataDir, dataS.DestPath)
 	log.Infof("downloading data '%s' in '%s'", dataS.SourceURL, destPath)
 	err := downloadFile(dataS.SourceURL, destPath)
 	if err != nil {
 		return err
 	}
-	}
 
 	return nil
 }