Parser visit split (#1242)

* break/cleanup this parser_visit function

* do not delete sub-collections if they belong to > 1 collection, fix #1118

pkg/cwhub/cwhub.go

@@ -3,6 +3,7 @@ package cwhub
 import (
 	"crypto/sha256"
 	"path/filepath"
+	"sort"
 	"strings"
 
 	//"errors"
@@ -81,6 +82,55 @@ type Item struct {
 	Collections   []string `yaml:"collections,omitempty" json:"collections,omitempty"`
 }
 
+func (i *Item) compareFile(fname, fstage, fauthor, path string) bool {
+	//wrong filename
+	if fname != i.FileName {
+		log.Tracef("%s != %s (filename)", fname, i.FileName)
+		return false
+	}
+	//wrong stage
+	if fstage != "" && fstage != i.Stage {
+		log.Tracef("%s != %s (stage)", fstage, i.Stage)
+		return false
+	}
+	//wrong author
+	if fauthor != "" && fauthor != i.Author {
+		log.Tracef("%s != %s (author)", fauthor, i.Author)
+		return false
+	}
+	return true
+}
+
+func (i *Item) getVersion(path string) (string, string, bool, error) {
+	//returns version, hash, up-to-date flag, error
+	sha, err := getSHA256(path)
+	if err != nil {
+		return "", "", false, err
+	}
+
+	//let's reverse sort the versions to deal with hash collisions (#154)
+	versions := make([]string, 0, len(i.Versions))
+	for k := range i.Versions {
+		versions = append(versions, k)
+	}
+	sort.Sort(sort.Reverse(sort.StringSlice(versions)))
+
+	for _, version := range versions {
+		val := i.Versions[version]
+		if sha != val.Digest {
+			continue
+		}
+		//if the version is == item.version, it means it's the last available version
+		if version == i.Version {
+			return version, sha, true, nil
+		}
+		//otherwise, return the version nontheless
+		return version, sha, false, nil
+	}
+	//the version is unknown
+	return "", sha, false, nil
+}
+
 func (i *Item) toHubStatus() ItemHubStatus {
 	hubStatus := ItemHubStatus{}
 	hubStatus.Name = i.Name
@@ -201,8 +251,8 @@ func AddItem(itemType string, item Item) error {
 }
 
 func DisplaySummary() {
-	log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers", len(hubIdx[COLLECTIONS]),
+	log.Printf("Loaded %d collecs, %d parsers, %d scenarios, %d post-overflow parsers, %d data files", len(hubIdx[COLLECTIONS]),
-		len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]))
+		len(hubIdx[PARSERS]), len(hubIdx[SCENARIOS]), len(hubIdx[PARSERS_OVFLW]), len(hubIdx[DATA_FILES]))
 	if skippedLocal > 0 || skippedTainted > 0 {
 		log.Printf("unmanaged items : %d local, %d tainted", skippedLocal, skippedTainted)
 	}
@@ -266,7 +316,6 @@ func GetUpstreamInstalledScenarios() ([]Item, error) {
 func GetHubStatusForItemType(itemType string, name string, all bool) []ItemHubStatus {
 	if _, ok := hubIdx[itemType]; !ok {
 		log.Errorf("type %s doesn't exist", itemType)
-
 		return nil
 	}
 

pkg/cwhub/cwhub_test.go

@@ -214,6 +214,7 @@ func testInstallItem(cfg *csconfig.Hub, t *testing.T, item Item) {
 	if !hubIdx[item.Type][item.Name].Installed {
 		t.Fatalf("install: %s should be install", item.Name)
 	}
+
 }
 
 func testTaintItem(cfg *csconfig.Hub, t *testing.T, item Item) {
@@ -332,6 +333,49 @@ func TestInstallParser(t *testing.T) {
 	}
 }
 
+func TestGetItemByPath(t *testing.T) {
+	cfg := test_prepenv()
+
+	if err := GetHubIdx(cfg.Hub); err != nil {
+		t.Fatalf("failed to load hub index")
+	}
+	for _, it := range hubIdx[PARSERS] {
+		//Install the parser
+		item, err := DownloadLatest(cfg.Hub, it, false, false)
+		if err != nil {
+			t.Fatalf("error while downloading %s : %v", item.Name, err)
+		}
+		//Enable the item
+		EnableItem(cfg.Hub, item)
+	}
+	//Sync
+	err, warns := LocalSync(cfg.Hub)
+	if err != nil || len(warns) > 0 {
+		t.Fatalf("unexpected err/warnings : %+v / %+v", err, warns)
+	}
+
+	for _, item := range hubIdx[PARSERS] {
+		//Check that we get the same thing if we get it by the path
+		itemcopy, err := GetItemByPath(item.Type, item.LocalPath)
+		if err != nil {
+			t.Fatalf("error while getting item by path : %v -> %s", err, item.LocalPath)
+		}
+		if itemcopy.Name != item.Name {
+			t.Fatalf("GetItemByPath: %s != %s", itemcopy.Name, item.Name)
+		}
+		if itemcopy.UpToDate != item.UpToDate {
+			t.Fatalf("GetItemByPath: %v != %v", itemcopy.UpToDate, item.UpToDate)
+		}
+		if itemcopy.Installed != item.Installed {
+			t.Fatalf("GetItemByPath: %v != %v", itemcopy.Installed, item.Installed)
+		}
+		if itemcopy.Tainted != item.Tainted {
+			t.Fatalf("GetItemByPath: %v != %v", itemcopy.Tainted, item.Tainted)
+		}
+	}
+
+}
+
 func TestInstallCollection(t *testing.T) {
 
 	/*
@@ -350,6 +394,7 @@ func TestInstallCollection(t *testing.T) {
 	}
 	//map iteration is random by itself
 	for _, it := range hubIdx[COLLECTIONS] {
+
 		testInstallItem(cfg.Hub, t, it)
 		it = hubIdx[COLLECTIONS][it.Name]
 		testTaintItem(cfg.Hub, t, it)
@@ -383,7 +428,6 @@ func (t *mockTransport) RoundTrip(req *http.Request) (*http.Response, error) {
 	responseBody := ""
 	log.Printf("---> %s", req.URL.Path)
 
-
 	/*FAKE PARSER*/
 	if strings.HasSuffix(req.URL.Path, "/master/parsers/s01-parse/crowdsecurity/foobar_parser.yaml") {
 		responseBody = `onsuccess: next_stage

pkg/cwhub/install.go

@@ -27,17 +27,26 @@ func DisableItem(hub *csconfig.Hub, target Item, purge bool, force bool) (Item,
 		return target, fmt.Errorf("%s is tainted, use '--force' to overwrite", target.Name)
 	}
 
-	/*for a COLLECTIONS, disable sub-items*/
+	/*for a COLLECTIONS, disable sub-items, if they are used _only_ by this item :)*/
 	if target.Type == COLLECTIONS {
 		var tmp = [][]string{target.Parsers, target.PostOverflows, target.Scenarios, target.Collections}
 		for idx, ptr := range tmp {
 			ptrtype := ItemTypes[idx]
 			for _, p := range ptr {
 				if val, ok := hubIdx[ptrtype][p]; ok {
+
+					if len(val.BelongsToCollections) > 1 {
+						log.Infof("%s belongs to more than one collection, won't disable (%v)", p, val.BelongsToCollections)
+						continue
+					}
+					if len(val.BelongsToCollections) == 1 && val.BelongsToCollections[0] == target.Name {
+						log.Infof("%s belongs exclusively to %s, disabling", p, target.Name)
 						hubIdx[ptrtype][p], err = DisableItem(hub, val, purge, force)
 						if err != nil {
 							return target, errors.Wrap(err, fmt.Sprintf("while disabling %s", p))
 						}
+					}
+
 				} else {
 					log.Errorf("Referred %s %s in collection %s doesn't exist.", ptrtype, p, target.Name)
 				}

pkg/cwhub/loader.go

@@ -5,7 +5,6 @@ import (
 	//"errors"
 	"fmt"
 	"io/ioutil"
-	"sort"
 
 	"github.com/pkg/errors"
 	"golang.org/x/mod/semver"
@@ -20,128 +19,190 @@ import (
 	log "github.com/sirupsen/logrus"
 )
 
-/*the walk/parser_visit function can't receive extra args*/
+/*the walk function can't receive extra args*/
 var hubdir, installdir, datadir string
 
-// TODO: Break this function into smaller functions.
+func visitDiscard(path string, f os.FileInfo) (string, bool, error) {
-func parser_visit(path string, f os.FileInfo, err error) error {
+	//return path, false, nil
-	var target Item
+	path, err := filepath.Abs(path)
-	var local bool
-	var hubpath string
-	var inhub bool
-	var fname string
-	var ftype string
-	var fauthor string
-	var stage string
-
-	path, err = filepath.Abs(path)
 	if err != nil {
-		return err
+		return path, true, err
 	}
 	//we only care about files
 	if f == nil || f.IsDir() {
+		return path, true, nil
+	}
+	return path, false, nil
+}
+
+func hubdirVisit(path string, f os.FileInfo, err error) error {
+
+	if err != nil {
+		log.Warningf("error visiting %s", err)
+	}
+	allowed_extensions := map[string]bool{".yaml": true, ".yml": true}
+	/*only interested by yaml files */
+	path, discard, err := visitDiscard(path, f)
+	if err != nil {
+		return err
+	}
+	if discard {
 		return nil
 	}
-	// yamls -> collections, parsers, overflows etc. txt, mmdb -> data files
+	if !allowed_extensions[filepath.Ext(path)] {
-	if !strings.HasSuffix(f.Name(), ".yaml") && !strings.HasSuffix(f.Name(), ".yml") && !strings.HasSuffix(f.Name(), ".txt") && !strings.HasSuffix(f.Name(), ".mmdb") {
+		log.Debugf("discarding %s : not a yaml file", path)
 		return nil
 	}
-
+	//extract components from path :
-	subs := strings.Split(path, "/")
-
-	log.Tracef("path:%s, hubdir:%s, installdir:%s datadir%s", path, hubdir, installdir, datadir)
-	/*we're in hub (~/.hub/hub/)*/
-
-	hubDirSetter := func() {
-		log.Tracef("in hub dir")
-		inhub = true
 	//.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml
 	//.../hub/scenarios/crowdsec/ssh_bf.yaml
 	//.../hub/profiles/crowdsec/linux.yaml
-		if len(subs) < 4 {
+	path_components := strings.Split(path, string(filepath.Separator))
-			log.Fatalf("path is too short : %s (%d)", path, len(subs))
+
+	if len(path_components) < 4 {
+		log.Fatalf("path is too short : %s (%d)", path, len(path_components))
 	}
-		fname = subs[len(subs)-1]
+	fname := path_components[len(path_components)-1]
-		fauthor = subs[len(subs)-2]
+	fauthor := path_components[len(path_components)-2]
-		stage = subs[len(subs)-3]
+	fstage := path_components[len(path_components)-3]
-		ftype = subs[len(subs)-4]
+	ftype := path_components[len(path_components)-4]
+
+	log.Tracef("%s : stage:%s ftype:%s", path, fstage, ftype)
+
+	if ftype == DATA_FILES {
+		return fmt.Errorf("unexpected data file in hub : %s", path)
 	}
 
-	dataDirSetter := func() {
+	// correct the stage and type for non-stage stuff.
-		log.Tracef("in data dir")
+	if fstage == SCENARIOS {
-		fauthor = ""
+		ftype = SCENARIOS
-		fname = subs[len(subs)-1]
+		fstage = ""
-		stage = ""
+	} else if fstage == COLLECTIONS {
-		ftype = DATA_FILES
+		ftype = COLLECTIONS
-		fauthor = ""
+		fstage = ""
+	} else if ftype != PARSERS && ftype != PARSERS_OVFLW { /*its a PARSER / PARSER_OVFLW with a stage */
+		return fmt.Errorf("unknown configuration type for file '%s'", path)
 	}
 
-	installDirSetter := func() {
+	log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, fstage, ftype)
-		log.Tracef("in install dir")
+
-		if len(subs) < 3 {
+	//in the hub, we don't expect symlinks
-			log.Fatalf("path is too short : %s (%d)", path, len(subs))
+	if f.Mode()&os.ModeSymlink != 0 {
+		log.Warningf("%s in the hub is a symlink, this isn't expected", path)
+	}
+	//try to find which configuration item it is
+	log.Tracef("check [%s] of %s", fname, ftype)
+
+	for itemName, item := range hubIdx[ftype] {
+		log.Tracef("check [%s] vs [%s] : %s/%s/%s", fname, item.RemotePath, ftype, fstage, fname)
+		if !item.compareFile(fname, fstage, fauthor, path) {
+			continue
+		}
+		//we're in the hub, mark the file as present and downloaded
+		if path == hubdir+"/"+item.RemotePath { //
+			log.Tracef("marking %s as downloaded", item.Name)
+			item.Downloaded = true
+		}
+
+		version, sha, uptodate, err := item.getVersion(path)
+		if err != nil {
+			return errors.Wrapf(err, "while getting version of %s", path)
+		}
+
+		if version == "" {
+			log.Debugf("got tainted match for %s : %s", item.Name, path)
+			skippedTainted += 1
+			item.UpToDate = uptodate
+			item.LocalVersion = "?"
+			item.Tainted = true
+			item.LocalHash = sha
+		} else {
+			item.UpToDate = uptodate
+		}
+		//if it was not present, update the index (it's the first time we're seeing this item. Might be downloaded and not installed)
+		if _, ok := hubIdx[ftype][itemName]; !ok {
+			hubIdx[ftype][itemName] = item
+		}
+		return nil
+	}
+	log.Infof("File %s found in hub directory wasn't found in the hub index, ignoring it", path)
+	return nil
+}
+
+func configdirVisit(path string, f os.FileInfo, err error) error {
+
+	if err != nil {
+		log.Warningf("error visiting %s", err)
+	}
+	allowed_extensions := map[string]bool{".yaml": true, ".yml": true}
+	/*only interested by yaml files */
+	path, discard, err := visitDiscard(path, f)
+	if err != nil {
+		return err
+	}
+	if discard {
+		return nil
+	}
+	if !allowed_extensions[filepath.Ext(path)] {
+		log.Debugf("discarding %s : not a yaml file", path)
+		return nil
+	}
+	path_components := strings.Split(path, string(filepath.Separator))
+
+	if len(path_components) < 3 {
+		log.Fatalf("path is too short : %s (%d)", path, len(path_components))
 	}
 	///.../config/parser/stage/file.yaml
 	///.../config/postoverflow/stage/file.yaml
 	///.../config/scenarios/scenar.yaml
 	///.../config/collections/linux.yaml //file is empty
-		fname = subs[len(subs)-1]
+	fname := path_components[len(path_components)-1]
-		stage = subs[len(subs)-2]
+	fstage := path_components[len(path_components)-2]
-		ftype = subs[len(subs)-3]
+	ftype := path_components[len(path_components)-3]
+
+	if ftype == DATA_FILES {
+		return fmt.Errorf("unexpected data file in install directory : %s", path)
 	}
 
-	setterByPath := map[string]func(){
+	log.Tracef("stage:%s ftype:%s", fstage, ftype)
-		installdir: installDirSetter,
-		hubdir:     hubDirSetter,
-		datadir:    dataDirSetter,
-	}
-
-	paths := []string{installdir, hubdir, datadir}
-	sort.Slice(paths, func(i, j int) bool {
-		return len(paths[i]) > len(paths[j])
-	})
-	foundMatch := false
-	for _, p := range paths {
-		if strings.HasPrefix(path, p) {
-			setterByPath[p]()
-			foundMatch = true
-			break
-		}
-	}
-	if !foundMatch {
-		return fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubdir, installdir)
-	}
-	log.Tracef("stage:%s ftype:%s", stage, ftype)
-	//log.Printf("%s -> name:%s stage:%s", path, fname, stage)
 
 	// correct the stage and type for non-stage stuff.
-	if stage == SCENARIOS {
+	if fstage == SCENARIOS {
 		ftype = SCENARIOS
-		stage = ""
+		fstage = ""
-	} else if stage == COLLECTIONS {
+	} else if fstage == COLLECTIONS {
 		ftype = COLLECTIONS
-		stage = ""
+		fstage = ""
-	} else if ftype != PARSERS && ftype != PARSERS_OVFLW && ftype != DATA_FILES { /*its a PARSER / PARSER_OVFLW with a stage */
+	} else if ftype != PARSERS && ftype != PARSERS_OVFLW { /*its a PARSER / PARSER_OVFLW with a stage */
 		return fmt.Errorf("unknown configuration type for file '%s'", path)
 	}
-	log.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", fname, fauthor, stage, ftype)
+	log.Tracef("CORRECTED [%s] in stage [%s] of type [%s]", fname, fstage, ftype)
 
-	/*
-		we can encounter 'collections' in the form of a symlink :
-		/etc/crowdsec/.../collections/linux.yaml -> ~/.hub/hub/collections/.../linux.yaml
-		when the collection is installed, both files are created
-	*/
 	//non symlinks are local user files or hub files or data files
 	if f.Mode()&os.ModeSymlink == 0 {
-		local = true
 		log.Tracef("%s isn't a symlink", path)
-	} else {
+		local_item := Item{}
-		hubpath, err = os.Readlink(path)
+		log.Tracef("%s is a local file, skip", path)
+		skippedLocal++
+		local_item.Name = fname
+		local_item.Stage = fstage
+		local_item.Installed = true
+		local_item.Type = ftype
+		local_item.Local = true
+		local_item.LocalPath = path
+		local_item.UpToDate = true
+		x := strings.Split(path, string(filepath.Separator))
+		local_item.FileName = x[len(x)-1]
+		hubIdx[ftype][fname] = local_item
+		return nil
+
+	}
+	hubpath, err := os.Readlink(path)
 	if err != nil {
 		return fmt.Errorf("unable to read symlink of %s", path)
 	}
-		//the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
+	//the symlink target doesn't exist, user might have removed hub directory without deleting /etc/crowdsec/....yaml
-		_, err := os.Lstat(hubpath)
+	_, err = os.Lstat(hubpath)
 	if os.IsNotExist(err) {
 		log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, hubpath)
 		//remove the symlink
@@ -150,123 +211,118 @@ func parser_visit(path string, f os.FileInfo, err error) error {
 		}
 		return nil
 	}
-		log.Tracef("%s points to %s", path, hubpath)
+
+	//try to get the matching item version
+	for itemName, item := range hubIdx[ftype] { // eg ftype = "collections", k = crowdsecurity/nginx, v is an Item struct
+		if !item.compareFile(fname, fstage, "", path) {
+			continue
 		}
-
+		log.Tracef("check [%s] vs [%s] : %s", fname, item.RemotePath, ftype+"/"+fstage+"/"+fname+".yaml")
-	//if it's not a symlink and not in hub nor it is a data file. Don't bother checking this with index
+		version, sha, uptodate, err := item.getVersion(path)
-	if local && !inhub && ftype != DATA_FILES {
+		if err != nil {
-		log.Tracef("%s is a local file, skip", path)
+			return errors.Wrapf(err, "while getting version of %s", path)
-		skippedLocal++
+		}
-		//	log.Printf("local scenario, skip.")
+		item.LocalPath = path
-		target.Name = fname
+		item.Installed = true
-		target.Stage = stage
+		item.LocalHash = sha
-		target.Installed = true
+		log.Debugf("found exact match for %s : version is %s (up-to-date:%t)", path, version, uptodate)
-		target.Type = ftype
+		/*we found the matching item, update it*/
-		target.Local = true
+		if version != "" {
-		target.LocalPath = path
+			item.LocalVersion = version
-		target.UpToDate = true
+			item.Tainted = false
-		x := strings.Split(path, "/")
+			item.Downloaded = true
-		target.FileName = x[len(x)-1]
+			item.UpToDate = uptodate
-
+		} else {
-		hubIdx[ftype][fname] = target
+			skippedTainted += 1
+			//the file and the stage is right, but the hash is wrong, it has been tainted by user
+			item.UpToDate = false
+			item.LocalVersion = "?"
+			item.Tainted = true
+		}
+		hubIdx[ftype][itemName] = item
 		return nil
 	}
+	log.Warningf("File %s found in install directory wasn't accounted for : not a symlink to hub, not a local file", path)
+	return nil
+}
+
+func datadirVisit(path string, f os.FileInfo, err error) error {
+
+	if err != nil {
+		log.Warningf("error visiting %s", err)
+	}
+
+	/*only interested by yaml files */
+	path, discard, err := visitDiscard(path, f)
+	if err != nil {
+		return err
+	}
+	if discard {
+		return nil
+	}
+
+	path_components := strings.Split(path, string(filepath.Separator))
+
+	if len(path_components) < 2 {
+		log.Fatalf("path is too short : %s (%d)", path, len(path_components))
+	}
+	fname := path_components[len(path_components)-1]
+	fauthor := path_components[len(path_components)-2]
+	ftype := DATA_FILES
+
+	log.Tracef("CORRECTED [%s] by [%s] of type [%s]", fname, fauthor, ftype)
+
+	//non symlinks are local user files or hub files or data files
+	if f.Mode()&os.ModeSymlink != 0 {
+		log.Warningf("%s is a symlink, that's unexpected (but can be ok)", path)
+		final_path, err := os.Readlink(path)
+		if err != nil {
+			return fmt.Errorf("unable to read symlink of %s", path)
+		}
+		//the symlink target doesn't exist, user might have removed ~/.hub/hub/...yaml without deleting /etc/crowdsec/....yaml
+		_, err = os.Lstat(path)
+		if os.IsNotExist(err) {
+			log.Infof("%s is a symlink to %s that doesn't exist, deleting symlink", path, final_path)
+			if err = os.Remove(path); err != nil {
+				return fmt.Errorf("failed to unlink %s: %+v", path, err)
+			}
+			return nil
+		}
+	}
+
 	//try to find which configuration item it is
 	log.Tracef("check [%s] of %s", fname, ftype)
 
-	match := false
+	for itemName, item := range hubIdx[ftype] { // eg ftype = "collections", k = crowdsecurity/nginx, v is an Item struct
-	for k, v := range hubIdx[ftype] { // eg ftype = "collections", k = crowdsecurity/nginx, v is an Item struct
+		log.Tracef("check [%s] vs [%s]", fname, item.RemotePath)
-		log.Tracef("check [%s] vs [%s] : %s", fname, v.RemotePath, ftype+"/"+stage+"/"+fname+".yaml")
+		if !item.compareFile(fname, "", fauthor, path) {
-		if fname != v.FileName {
-			log.Tracef("%s != %s (filename)", fname, v.FileName)
 			continue
 		}
-		//wrong stage
+
-		if v.Stage != stage {
+		version, sha, uptodate, err := item.getVersion(path)
-			log.Tracef("%s != %s (stage)", v.Stage, stage)
-			continue
-		}
-		/*if we are walking hub dir, just mark present files as downloaded*/
-		if inhub {
-			//wrong author
-			if fauthor != v.Author {
-				continue
-			}
-			//wrong file
-			if v.Name+".yaml" != fauthor+"/"+fname {
-				continue
-			}
-			if path == hubdir+"/"+v.RemotePath {
-				log.Tracef("marking %s as downloaded", v.Name)
-				v.Downloaded = true
-			}
-		}
-		sha, err := getSHA256(path)
 		if err != nil {
-			log.Fatalf("Failed to get sha of %s : %v", path, err)
+			return errors.Wrapf(err, "while getting version of %s", path)
 		}
-		//let's reverse sort the versions to deal with hash collisions (#154)
+		item.LocalPath = path
-		versions := make([]string, 0, len(v.Versions))
+		item.Installed = true
-		for k := range v.Versions {
+		item.LocalHash = sha
-			versions = append(versions, k)
+		log.Debugf("DATA [%s] found exact match for %s : version is %s (up-to-date:%t)", itemName, path, version, uptodate)
-		}
+		if version == "" {
-		sort.Sort(sort.Reverse(sort.StringSlice(versions)))
-
-		for _, version := range versions {
-			val := v.Versions[version]
-			if sha != val.Digest {
-				//log.Printf("matching filenames, wrong hash %s != %s -- %s", sha, val.Digest, spew.Sdump(v))
-				continue
-			}
-			/*we got an exact match, update struct*/
-			if !inhub {
-				log.Tracef("found exact match for %s, version is %s, latest is %s", v.Name, version, v.Version)
-				v.LocalPath = path
-				v.LocalVersion = version
-				v.Tainted = false
-				v.Downloaded = true
-				/*if we're walking the hub, present file doesn't means installed file*/
-				v.Installed = true
-				v.LocalHash = sha
-				x := strings.Split(path, "/")
-				target.FileName = x[len(x)-1]
-			}
-			if version == v.Version {
-				log.Tracef("%s is up-to-date", v.Name)
-				v.UpToDate = true
-			}
-			match = true
-			break
-		}
-		if !match {
-			log.Tracef("got tainted match for %s : %s", v.Name, path)
 			skippedTainted += 1
-			//the file and the stage is right, but the hash is wrong, it has been tainted by user
+			item.UpToDate = uptodate
-			if !inhub {
+			item.LocalVersion = "?"
-				v.LocalPath = path
+			item.Tainted = true
-				v.Installed = true
+			item.LocalHash = sha
-			}
+		} else {
-			v.UpToDate = false
+			item.UpToDate = uptodate
-			v.LocalVersion = "?"
+			item.LocalVersion = version
-			v.Tainted = true
+			item.LocalHash = sha
-			v.LocalHash = sha
-			x := strings.Split(path, "/")
-			target.FileName = x[len(x)-1]
-
-		}
-		//update the entry if appropriate
-		if _, ok := hubIdx[ftype][k]; !ok {
-			hubIdx[ftype][k] = v
-		} else if !inhub {
-			hubIdx[ftype][k] = v
 		}
+		hubIdx[ftype][itemName] = item
 		return nil
 	}
-	if ftype != DATA_FILES {
-		log.Infof("Ignoring file %s of type %s", path, ftype)
-	} else {
 	log.Debugf("Ignoring file %s of type %s", path, ftype)
-	}
+
 	return nil
 }
 
@@ -295,7 +351,7 @@ func CollecDepsCheck(v *Item) error {
 				if val.Type == COLLECTIONS {
 					log.Tracef("collec, recurse.")
 					if err := CollecDepsCheck(&val); err != nil {
-						return fmt.Errorf("sub collection %s is broken : %s", val.Name, err)
+						return fmt.Errorf("sub collection %s warning : %s", val.Name, err)
 					}
 					hubIdx[ptrtype][p] = val
 				}
@@ -336,19 +392,36 @@ func SyncDir(hub *csconfig.Hub, dir string) (error, []string) {
 	datadir = hub.DataDir
 	warnings := []string{}
 
+	//data_dir is quite simple : there is no collections and such
+	if dir == hub.DataDir {
+		var cpath string
+		var err error
+		cpath, err = filepath.Abs(hub.DataDir)
+		if err != nil {
+			log.Errorf("failed %s : %s", cpath, err)
+		}
+		err = filepath.Walk(cpath, datadirVisit)
+		return err, warnings
+	}
+
 	/*For each, scan PARSERS, PARSERS_OVFLW, DATA_FILES, SCENARIOS and COLLECTIONS last*/
 	for _, scan := range ItemTypes {
 		var cpath string
 		var err error
-		if scan == DATA_FILES {
+
-			cpath, err = filepath.Abs(hub.DataDir)
-		} else {
 		cpath, err = filepath.Abs(fmt.Sprintf("%s/%s", dir, scan))
-		}
 		if err != nil {
 			log.Errorf("failed %s : %s", cpath, err)
 		}
-		err = filepath.Walk(cpath, parser_visit)
+
+		switch dir {
+		case hub.HubDir:
+			err = filepath.Walk(cpath, hubdirVisit)
+		case hub.ConfigDir:
+			err = filepath.Walk(cpath, configdirVisit)
+		default:
+			log.Fatalf("unexpected dir %s", dir)
+		}
 		if err != nil {
 			return err, warnings
 		}
@@ -387,6 +460,10 @@ func LocalSync(hub *csconfig.Hub) (error, []string) {
 	if err != nil {
 		return fmt.Errorf("failed to scan %s : %s", hub.HubDir, err), warnings
 	}
+	err, _ = SyncDir(hub, hub.DataDir)
+	if err != nil {
+		return fmt.Errorf("failed to scan %s : %s", hub.DataDir, err), warnings
+	}
 	return nil, warnings
 }