package cwhub import ( "crypto/sha256" "encoding/hex" "fmt" "io" "os" "path/filepath" "slices" "sort" "strings" "github.com/Masterminds/semver/v3" "github.com/sirupsen/logrus" "gopkg.in/yaml.v3" ) func isYAMLFileName(path string) bool { return strings.HasSuffix(path, ".yaml") || strings.HasSuffix(path, ".yml") } // linkTarget returns the target of a symlink, or empty string if it's dangling. func linkTarget(path string, logger *logrus.Logger) (string, error) { hubpath, err := os.Readlink(path) if err != nil { return "", fmt.Errorf("unable to read symlink: %s", path) } logger.Tracef("symlink %s -> %s", path, hubpath) _, err = os.Lstat(hubpath) if os.IsNotExist(err) { logger.Warningf("link target does not exist: %s -> %s", path, hubpath) return "", nil } return hubpath, nil } func getSHA256(filepath string) (string, error) { f, err := os.Open(filepath) if err != nil { return "", fmt.Errorf("unable to open '%s': %w", filepath, err) } defer f.Close() h := sha256.New() if _, err := io.Copy(h, f); err != nil { return "", fmt.Errorf("unable to calculate sha256 of '%s': %w", filepath, err) } return hex.EncodeToString(h.Sum(nil)), nil } // information used to create a new Item, from a file path. type itemFileInfo struct { inhub bool fname string stage string ftype string fauthor string } func (h *Hub) getItemFileInfo(path string, logger *logrus.Logger) (*itemFileInfo, error) { var ret *itemFileInfo hubDir := h.local.HubDir installDir := h.local.InstallDir subs := strings.Split(path, string(os.PathSeparator)) logger.Tracef("path:%s, hubdir:%s, installdir:%s", path, hubDir, installDir) logger.Tracef("subs:%v", subs) // we're in hub (~/.hub/hub/) if strings.HasPrefix(path, hubDir) { logger.Tracef("in hub dir") //.../hub/parsers/s00-raw/crowdsec/skip-pretag.yaml //.../hub/scenarios/crowdsec/ssh_bf.yaml //.../hub/profiles/crowdsec/linux.yaml if len(subs) < 4 { return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs)) } ret = &itemFileInfo{ inhub: true, fname: subs[len(subs)-1], fauthor: subs[len(subs)-2], stage: subs[len(subs)-3], ftype: subs[len(subs)-4], } } else if strings.HasPrefix(path, installDir) { // we're in install /etc/crowdsec//... logger.Tracef("in install dir") if len(subs) < 3 { return nil, fmt.Errorf("path is too short: %s (%d)", path, len(subs)) } ///.../config/parser/stage/file.yaml ///.../config/postoverflow/stage/file.yaml ///.../config/scenarios/scenar.yaml ///.../config/collections/linux.yaml //file is empty ret = &itemFileInfo{ inhub: false, fname: subs[len(subs)-1], stage: subs[len(subs)-2], ftype: subs[len(subs)-3], fauthor: "", } } else { return nil, fmt.Errorf("file '%s' is not from hub '%s' nor from the configuration directory '%s'", path, hubDir, installDir) } logger.Tracef("stage:%s ftype:%s", ret.stage, ret.ftype) if ret.ftype != PARSERS && ret.ftype != POSTOVERFLOWS { if !slices.Contains(ItemTypes, ret.stage) { return nil, fmt.Errorf("unknown configuration type for file '%s'", path) } ret.ftype = ret.stage ret.stage = "" } logger.Tracef("CORRECTED [%s] by [%s] in stage [%s] of type [%s]", ret.fname, ret.fauthor, ret.stage, ret.ftype) return ret, nil } // sortedVersions returns the input data, sorted in reverse order (new, old) by semver. func sortedVersions(raw []string) ([]string, error) { vs := make([]*semver.Version, len(raw)) for idx, r := range raw { v, err := semver.NewVersion(r) if err != nil { return nil, fmt.Errorf("%s: %w", r, err) } vs[idx] = v } sort.Sort(sort.Reverse(semver.Collection(vs))) ret := make([]string, len(vs)) for idx, v := range vs { ret[idx] = v.Original() } return ret, nil } func newLocalItem(h *Hub, path string, info *itemFileInfo) (*Item, error) { type localItemName struct { Name string `yaml:"name"` } _, fileName := filepath.Split(path) item := &Item{ hub: h, Name: info.fname, Stage: info.stage, Type: info.ftype, FileName: fileName, State: ItemState{ LocalPath: path, Installed: true, UpToDate: true, }, } // try to read the name from the file itemName := localItemName{} itemContent, err := os.ReadFile(path) if err != nil { return nil, fmt.Errorf("failed to read %s: %w", path, err) } err = yaml.Unmarshal(itemContent, &itemName) if err != nil { return nil, fmt.Errorf("failed to unmarshal %s: %w", path, err) } if itemName.Name != "" { item.Name = itemName.Name } return item, nil } func (h *Hub) itemVisit(path string, f os.DirEntry, err error) error { hubpath := "" if err != nil { h.logger.Debugf("while syncing hub dir: %s", err) // there is a path error, we ignore the file return nil } // only happens if the current working directory was removed (!) path, err = filepath.Abs(path) if err != nil { return err } // we only care about YAML files if f == nil || f.IsDir() || !isYAMLFileName(f.Name()) { return nil } info, err := h.getItemFileInfo(path, h.logger) if err != nil { return err } // non symlinks are local user files or hub files if f.Type()&os.ModeSymlink == 0 { h.logger.Tracef("%s is not a symlink", path) if !info.inhub { h.logger.Tracef("%s is a local file, skip", path) item, err := newLocalItem(h, path, info) if err != nil { return err } h.addItem(item) return nil } } else { hubpath, err = linkTarget(path, h.logger) if err != nil { return err } if hubpath == "" { // target does not exist, the user might have removed the file // or switched to a hub branch without it return nil } } // try to find which configuration item it is h.logger.Tracef("check [%s] of %s", info.fname, info.ftype) for _, item := range h.GetItemMap(info.ftype) { if info.fname != item.FileName { continue } if item.Stage != info.stage { continue } // if we are walking hub dir, just mark present files as downloaded if info.inhub { // wrong author if info.fauthor != item.Author { continue } // not the item we're looking for if !item.validPath(info.fauthor, info.fname) { continue } src, err := item.downloadPath() if err != nil { return err } if path == src { h.logger.Tracef("marking %s as downloaded", item.Name) item.State.Downloaded = true } } else if !hasPathSuffix(hubpath, item.RemotePath) { // wrong file // ///.yaml continue } err := item.setVersionState(path, info.inhub) if err != nil { return err } return nil } h.logger.Infof("Ignoring file %s of type %s", path, info.ftype) return nil } // checkSubItemVersions checks for the presence, taint and version state of sub-items. func (i *Item) checkSubItemVersions() []string { warn := make([]string, 0) if !i.HasSubItems() { return warn } if i.versionStatus() != versionUpToDate { i.hub.logger.Debugf("%s dependencies not checked: not up-to-date", i.Name) return warn } // ensure all the sub-items are installed, or tag the parent as tainted i.hub.logger.Tracef("checking submembers of %s installed:%t", i.Name, i.State.Installed) for _, sub := range i.SubItems() { i.hub.logger.Tracef("check %s installed:%t", sub.Name, sub.State.Installed) if !i.State.Installed { continue } if w := sub.checkSubItemVersions(); len(w) > 0 { if sub.State.Tainted { i.addTaint(sub) warn = append(warn, fmt.Sprintf("%s is tainted by %s", i.Name, sub.FQName())) } warn = append(warn, w...) continue } if sub.State.Tainted { i.addTaint(sub) warn = append(warn, fmt.Sprintf("%s is tainted by %s", i.Name, sub.FQName())) continue } if !sub.State.Installed && i.State.Installed { i.addTaint(sub) warn = append(warn, fmt.Sprintf("%s is tainted by missing %s", i.Name, sub.FQName())) continue } if !sub.State.UpToDate { i.State.UpToDate = false warn = append(warn, fmt.Sprintf("%s is tainted by outdated %s", i.Name, sub.FQName())) continue } i.hub.logger.Tracef("checking for %s - tainted:%t uptodate:%t", sub.Name, i.State.Tainted, i.State.UpToDate) } return warn } // syncDir scans a directory for items, and updates the Hub state accordingly. func (h *Hub) syncDir(dir string) error { // For each, scan PARSERS, POSTOVERFLOWS... and COLLECTIONS last for _, scan := range ItemTypes { // cpath: top-level item directory, either downloaded or installed items. // i.e. /etc/crowdsec/parsers, /etc/crowdsec/hub/parsers, ... cpath, err := filepath.Abs(fmt.Sprintf("%s/%s", dir, scan)) if err != nil { h.logger.Errorf("failed %s: %s", cpath, err) continue } // explicit check for non existing directory, avoid spamming log.Debug if _, err = os.Stat(cpath); os.IsNotExist(err) { h.logger.Tracef("directory %s doesn't exist, skipping", cpath) continue } if err = filepath.WalkDir(cpath, h.itemVisit); err != nil { return err } } return nil } // insert a string in a sorted slice, case insensitive, and return the new slice. func insertInOrderNoCase(sl []string, value string) []string { i := sort.Search(len(sl), func(i int) bool { return strings.ToLower(sl[i]) >= strings.ToLower(value) }) return append(sl[:i], append([]string{value}, sl[i:]...)...) } func removeDuplicates(sl []string) []string { seen := make(map[string]struct{}, len(sl)) j := 0 for _, v := range sl { if _, ok := seen[v]; ok { continue } seen[v] = struct{}{} sl[j] = v j++ } return sl[:j] } // localSync updates the hub state with downloaded, installed and local items. func (h *Hub) localSync() error { err := h.syncDir(h.local.InstallDir) if err != nil { return fmt.Errorf("failed to scan %s: %w", h.local.InstallDir, err) } if err = h.syncDir(h.local.HubDir); err != nil { return fmt.Errorf("failed to scan %s: %w", h.local.HubDir, err) } warnings := make([]string, 0) for _, item := range h.GetItemMap(COLLECTIONS) { // check for cyclic dependencies subs, err := item.descendants() if err != nil { return err } // populate the sub- and sub-sub-items with the collections they belong to for _, sub := range subs { sub.State.BelongsToCollections = insertInOrderNoCase(sub.State.BelongsToCollections, item.Name) } if !item.State.Installed { continue } vs := item.versionStatus() switch vs { case versionUpToDate: // latest if w := item.checkSubItemVersions(); len(w) > 0 { warnings = append(warnings, w...) } case versionUpdateAvailable: // not up-to-date warnings = append(warnings, fmt.Sprintf("update for collection %s available (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version)) case versionFuture: warnings = append(warnings, fmt.Sprintf("collection %s is in the future (currently:%s, latest:%s)", item.Name, item.State.LocalVersion, item.Version)) case versionUnknown: if !item.State.IsLocal() { warnings = append(warnings, fmt.Sprintf("collection %s is tainted by local changes (latest:%s)", item.Name, item.Version)) } } h.logger.Debugf("installed (%s) - status: %d | installed: %s | latest: %s | full: %+v", item.Name, vs, item.State.LocalVersion, item.Version, item.Versions) } h.Warnings = removeDuplicates(warnings) return nil } func (i *Item) setVersionState(path string, inhub bool) error { var err error i.State.LocalHash, err = getSHA256(path) if err != nil { return fmt.Errorf("failed to get sha256 of %s: %w", path, err) } // let's reverse sort the versions to deal with hash collisions (#154) versions := make([]string, 0, len(i.Versions)) for k := range i.Versions { versions = append(versions, k) } versions, err = sortedVersions(versions) if err != nil { return fmt.Errorf("while syncing %s %s: %w", i.Type, i.FileName, err) } i.State.LocalVersion = "?" for _, version := range versions { if i.Versions[version].Digest == i.State.LocalHash { i.State.LocalVersion = version break } } if i.State.LocalVersion == "?" { i.hub.logger.Tracef("got tainted match for %s: %s", i.Name, path) if !inhub { i.State.LocalPath = path i.State.Installed = true } i.State.UpToDate = false i.addTaint(i) return nil } // we got an exact match, update struct i.State.Downloaded = true if !inhub { i.hub.logger.Tracef("found exact match for %s, version is %s, latest is %s", i.Name, i.State.LocalVersion, i.Version) i.State.LocalPath = path i.State.Tainted = false // if we're walking the hub, present file doesn't means installed file i.State.Installed = true } if i.State.LocalVersion == i.Version { i.hub.logger.Tracef("%s is up-to-date", i.Name) i.State.UpToDate = true } return nil }