crowdsec/pkg/setup/detect.go

package setup

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"sort"

	"github.com/Masterminds/semver"
	"github.com/antonmedv/expr"
	"github.com/blackfireio/osinfo"
	"github.com/shirou/gopsutil/v3/process"
	log "github.com/sirupsen/logrus"
	"gopkg.in/yaml.v3"
	// goccyyaml "github.com/goccy/go-yaml"

	// "github.com/k0kubun/pp"

	"github.com/crowdsecurity/crowdsec/pkg/acquisition"
	"github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration"
)

// ExecCommand can be replaced with a mock during tests.
var ExecCommand = exec.Command

// HubItems contains the objects that are recommended to support a service.
type HubItems struct {
	Collections   []string `yaml:"collections,omitempty"`
	Parsers       []string `yaml:"parsers,omitempty"`
	Scenarios     []string `yaml:"scenarios,omitempty"`
	PostOverflows []string `yaml:"postoverflows,omitempty"`
}

type DataSourceItem map[string]interface{}

// ServiceSetup describes the recommendations (hub objects and datasources) for a detected service.
type ServiceSetup struct {
	DetectedService string         `yaml:"detected_service"`
	Install         *HubItems      `yaml:"install,omitempty"`
	DataSource      DataSourceItem `yaml:"datasource,omitempty"`
}

// Setup is a container for a list of ServiceSetup objects, allowing for future extensions.
type Setup struct {
	Setup []ServiceSetup `yaml:"setup"`
}

func validateDataSource(opaqueDS DataSourceItem) error {
	if len(opaqueDS) == 0 {
		// empty datasource is valid
		return nil
	}


	// formally validate YAML

	commonDS := configuration.DataSourceCommonCfg{}
	body, err := yaml.Marshal(opaqueDS)
	if err != nil {
		return err
	}

	err = yaml.Unmarshal(body, &commonDS)
	if err != nil {
		return err
	}

	// source is mandatory // XXX unless it's not?

	if commonDS.Source == "" {
		return fmt.Errorf("source is empty")
	}


	// source must be known

	ds := acquisition.GetDataSourceIface(commonDS.Source)
	if ds == nil {
		return fmt.Errorf("unknown source '%s'", commonDS.Source)
	}

	// unmarshal and validate the rest with the specific implementation

	err = ds.UnmarshalConfig(body)
	if err != nil {
		return err
	}

	// pp.Println(ds)
	return nil
}

func readDetectConfig(file string) (DetectConfig, error) {
	var dc DetectConfig

	yamlBytes, err := os.ReadFile(file)
	if err != nil {
		return DetectConfig{}, fmt.Errorf("while reading file: %w", err)
	}

	dec := yaml.NewDecoder(bytes.NewBuffer(yamlBytes))
	dec.KnownFields(true)

	if err = dec.Decode(&dc); err != nil {
		return DetectConfig{}, fmt.Errorf("while parsing %s: %w", file, err)
	}

	switch dc.Version {
	case "":
		return DetectConfig{}, fmt.Errorf("missing version tag (must be 1.0)")
	case "1.0":
		// all is well
	default:
		return DetectConfig{}, fmt.Errorf("unsupported version tag '%s' (must be 1.0)", dc.Version)
	}

	for name, svc := range dc.Detect {
		err = validateDataSource(svc.DataSource)
		if err != nil {
			return DetectConfig{}, fmt.Errorf("invalid datasource for %s: %w", name, err)
		}
	}

	return dc, nil
}

// Service describes the rules for detecting a service and its recommended items.
type Service struct {
	When       []string       `yaml:"when"`
	Install    *HubItems      `yaml:"install,omitempty"`
	DataSource DataSourceItem `yaml:"datasource,omitempty"`
	// AcquisYAML []byte
}

// DetectConfig is the container of all detection rules (detect.yaml).
type DetectConfig struct {
	Version string             `yaml:"version"`
	Detect  map[string]Service `yaml:"detect"`
}

// ExprState keeps a global state for the duration of the service detection (cache etc.)
type ExprState struct {
	unitsSearched map[string]bool
	detectOptions DetectOptions

	// cache
	installedUnits map[string]bool
	// true if the list of running processes has already been retrieved, we can
	// avoid getting it a second time.
	processesSearched map[string]bool
	// cache
	runningProcesses map[string]bool
}

// ExprServiceState keep a local state during the detection of a single service. It is reset before each service rules' evaluation.
type ExprServiceState struct {
	detectedUnits []string
}

// ExprOS contains the detected (or forced) OS fields available to the rule engine.
type ExprOS struct {
	Family     string
	ID         string
	RawVersion string
}

// This is not required with Masterminds/semver
/*
// normalizeVersion strips leading zeroes from each part, to allow comparison of ubuntu-like versions.
func normalizeVersion(version string) string {
	// if it doesn't match a version string, return unchanged
	if ok := regexp.MustCompile(`^(\d+)(\.\d+)?(\.\d+)?$`).MatchString(version); !ok {
		// definitely not an ubuntu-like version, return unchanged
		return version
	}

	ret := []rune{}

	var cur rune

	trim := true
	for _, next := range version + "." {
		if trim && cur == '0' && next != '.' {
			cur = next

			continue
		}

		if cur != 0 {
			ret = append(ret, cur)
		}

		trim = (cur == '.' || cur == 0)
		cur = next
	}

	return string(ret)
}
*/

// VersionCheck returns true if the version of the OS matches the given constraint
func (os ExprOS) VersionCheck(constraint string) (bool, error) {
	v, err := semver.NewVersion(os.RawVersion)
	if err != nil {
		return false, err
	}

	c, err := semver.NewConstraint(constraint)
	if err != nil {
		return false, err
	}

	return c.Check(v), nil
}

// VersionAtLeast returns true if the version of the OS is at least the given version.
func (os ExprOS) VersionAtLeast(constraint string) (bool, error) {
	return os.VersionCheck(">=" + constraint)
}

// VersionIsLower returns true if the version of the OS is lower than the given version.
func (os ExprOS) VersionIsLower(version string) (bool, error) {
	result, err := os.VersionAtLeast(version)
	if err != nil {
		return false, err
	}

	return !result, nil
}

// ExprEnvironment is used to expose functions and values to the rule engine.
// It can cache the results of service detection commands, like systemctl etc.
type ExprEnvironment struct {
	OS ExprOS

	_serviceState *ExprServiceState
	_state        *ExprState
}

// NewExprEnvironment creates an environment object for the rule engine.
func NewExprEnvironment(opts DetectOptions, os ExprOS) ExprEnvironment {
	return ExprEnvironment{
		_state: &ExprState{
			detectOptions: opts,

			unitsSearched:  make(map[string]bool),
			installedUnits: make(map[string]bool),

			processesSearched: make(map[string]bool),
			runningProcesses:  make(map[string]bool),
		},
		_serviceState: &ExprServiceState{},
		OS:            os,
	}
}

// PathExists returns true if the given path exists.
func (e ExprEnvironment) PathExists(path string) bool {
	_, err := os.Stat(path)

	return err == nil
}

// UnitFound returns true if the unit is listed in the systemctl output.
// Whether a disabled or failed unit is considered found or not, depends on the
// systemctl parameters used.
func (e ExprEnvironment) UnitFound(unitName string) (bool, error) {
	// fill initial caches
	if len(e._state.unitsSearched) == 0 {
		if !e._state.detectOptions.SnubSystemd {
			units, err := systemdUnitList()
			if err != nil {
				return false, err
			}

			for _, name := range units {
				e._state.installedUnits[name] = true
			}
		}

		for _, name := range e._state.detectOptions.ForcedUnits {
			e._state.installedUnits[name] = true
		}
	}

	e._state.unitsSearched[unitName] = true
	if e._state.installedUnits[unitName] {
		e._serviceState.detectedUnits = append(e._serviceState.detectedUnits, unitName)

		return true, nil
	}

	return false, nil
}

// ProcessRunning returns true if there is a running process with the given name.
func (e ExprEnvironment) ProcessRunning(processName string) (bool, error) {
	if len(e._state.processesSearched) == 0 {
		procs, err := process.Processes()
		if err != nil {
			return false, fmt.Errorf("while looking up running processes: %w", err)
		}

		for _, p := range procs {
			name, err := p.Name()
			if err != nil {
				return false, fmt.Errorf("while looking up running processes: %w", err)
			}

			e._state.runningProcesses[name] = true
		}

		for _, name := range e._state.detectOptions.ForcedProcesses {
			e._state.runningProcesses[name] = true
		}
	}

	e._state.processesSearched[processName] = true

	return e._state.runningProcesses[processName], nil
}

// applyRules checks if the 'when' expressions are true and returns a Service struct,
// augmented with default values and anything that might be useful later on
//
// All expressions are evaluated (no short-circuit) because we want to know if there are errors.
func applyRules(svc Service, env ExprEnvironment) (Service, bool, error) {
	newsvc := svc
	svcok := true
	env._serviceState = &ExprServiceState{}

	for _, rule := range svc.When {
		out, err := expr.Eval(rule, env)
		log.Tracef("  Rule '%s' -> %t, %v", rule, out, err)

		if err != nil {
			return Service{}, false, fmt.Errorf("rule '%s': %w", rule, err)
		}

		outbool, ok := out.(bool)
		if !ok {
			return Service{}, false, fmt.Errorf("rule '%s': type must be a boolean", rule)
		}

		svcok = svcok && outbool
	}

	//	if newsvc.Acquis == nil || (newsvc.Acquis.LogFiles == nil && newsvc.Acquis.JournalCTLFilter == nil) {
	//		for _, unitName := range env._serviceState.detectedUnits {
	//			if newsvc.Acquis == nil {
	//				newsvc.Acquis = &AcquisItem{}
	//			}
	//			// if there is reference to more than one unit in the rules, we use the first one
	//			newsvc.Acquis.JournalCTLFilter = []string{fmt.Sprintf(`_SYSTEMD_UNIT=%s`, unitName)}
	//			break //nolint  // we want to exit after one iteration
	//		}
	//	}

	return newsvc, svcok, nil
}

// filterWithRules decorates a DetectConfig map by filtering according to the when: clauses,
// and applying default values or whatever useful to the Service items.
func filterWithRules(dc DetectConfig, env ExprEnvironment) (map[string]Service, error) {
	ret := make(map[string]Service)

	for name := range dc.Detect {
		//
		// an empty list of when: clauses defaults to true, if we want
		// to change this behavior, the place is here.
		// if len(svc.When) == 0 {
		// 	log.Warningf("empty 'when' clause: %+v", svc)
		// }
		//
		log.Trace("Evaluating rules for: ", name)

		svc, ok, err := applyRules(dc.Detect[name], env)
		if err != nil {
			return nil, fmt.Errorf("while looking for service %s: %w", name, err)
		}

		if !ok {
			log.Tracef("  Skipping %s", name)

			continue
		}

		log.Tracef("  Detected %s", name)

		ret[name] = svc
	}

	return ret, nil
}

// return units that have been forced but not searched yet.
func (e ExprEnvironment) unsearchedUnits() []string {
	ret := []string{}

	for _, unit := range e._state.detectOptions.ForcedUnits {
		if !e._state.unitsSearched[unit] {
			ret = append(ret, unit)
		}
	}

	return ret
}

// return processes that have been forced but not searched yet.
func (e ExprEnvironment) unsearchedProcesses() []string {
	ret := []string{}

	for _, proc := range e._state.detectOptions.ForcedProcesses {
		if !e._state.processesSearched[proc] {
			ret = append(ret, proc)
		}
	}

	return ret
}

// checkConsumedForcedItems checks if all the "forced" options (units or processes) have been evaluated during the service detection.
func checkConsumedForcedItems(e ExprEnvironment) error {
	unconsumed := e.unsearchedUnits()

	unitMsg := ""
	if len(unconsumed) > 0 {
		unitMsg = fmt.Sprintf("unit(s) forced but not supported: %v", unconsumed)
	}

	unconsumed = e.unsearchedProcesses()

	procsMsg := ""
	if len(unconsumed) > 0 {
		procsMsg = fmt.Sprintf("process(es) forced but not supported: %v", unconsumed)
	}

	join := ""
	if unitMsg != "" && procsMsg != "" {
		join = "; "
	}

	if unitMsg != "" || procsMsg != "" {
		return fmt.Errorf("%s%s%s", unitMsg, join, procsMsg)
	}

	return nil
}

// DetectOptions contains parameters for the Detect function.
type DetectOptions struct {
	// slice of unit names that we want to force-detect
	ForcedUnits []string
	// slice of process names that we want to force-detect
	ForcedProcesses []string
	ForcedOS        ExprOS
	SkipServices    []string
	SnubSystemd     bool
}

// Detect performs the service detection from a given configuration.
// It outputs a setup file that can be used as input to "cscli setup install-hub"
// or "cscli setup datasources".
func Detect(serviceDetectionFile string, opts DetectOptions) (Setup, error) {
	ret := Setup{}

	// explicitly initialize to avoid json mashaling an empty slice as "null"
	ret.Setup = make([]ServiceSetup, 0)

	log.Tracef("Reading detection rules: %s", serviceDetectionFile)

	sc, err := readDetectConfig(serviceDetectionFile)
	if err != nil {
		return ret, err
	}

	//	// generate acquis.yaml snippet for this service
	//	for key := range sc.Detect {
	//		svc := sc.Detect[key]
	//		if svc.Acquis != nil {
	//			svc.AcquisYAML, err = yaml.Marshal(svc.Acquis)
	//			if err != nil {
	//				return ret, err
	//			}
	//			sc.Detect[key] = svc
	//		}
	//	}

	var osfull *osinfo.OSInfo

	os := opts.ForcedOS
	if os == (ExprOS{}) {
		osfull, err = osinfo.GetOSInfo()
		if err != nil {
			return ret, fmt.Errorf("detecting OS: %w", err)
		}

		log.Tracef("Detected OS - %+v", *osfull)

		os = ExprOS{
			Family:     osfull.Family,
			ID:         osfull.ID,
			RawVersion: osfull.Version,
		}
	} else {
		log.Tracef("Forced OS - %+v", os)
	}

	if len(opts.ForcedUnits) > 0 {
		log.Tracef("Forced units - %v", opts.ForcedUnits)
	}

	if len(opts.ForcedProcesses) > 0 {
		log.Tracef("Forced processes - %v", opts.ForcedProcesses)
	}

	env := NewExprEnvironment(opts, os)

	detected, err := filterWithRules(sc, env)
	if err != nil {
		return ret, err
	}

	if err = checkConsumedForcedItems(env); err != nil {
		return ret, err
	}

	// remove services the user asked to ignore
	for _, name := range opts.SkipServices {
		delete(detected, name)
	}

	// sort the keys (service names) to have them in a predictable
	// order in the final output

	keys := make([]string, 0)
	for k := range detected {
		keys = append(keys, k)
	}

	sort.Strings(keys)

	for _, name := range keys {
		svc := detected[name]
		//		if svc.DataSource != nil {
		//			if svc.DataSource.Labels["type"] == "" {
		//				return Setup{}, fmt.Errorf("missing type label for service %s", name)
		//			}
		//			err = yaml.Unmarshal(svc.AcquisYAML, svc.DataSource)
		//			if err != nil {
		//				return Setup{}, fmt.Errorf("while unmarshaling datasource for service %s: %w", name, err)
		//			}
		//		}

		ret.Setup = append(ret.Setup, ServiceSetup{
			DetectedService: name,
			Install:         svc.Install,
			DataSource:      svc.DataSource,
		})
	}

	return ret, nil
}

// ListSupported parses the configuration file and outputs a list of the supported services.
func ListSupported(serviceDetectionFile string) ([]string, error) {
	dc, err := readDetectConfig(serviceDetectionFile)
	if err != nil {
		return nil, err
	}

	keys := make([]string, 0)
	for k := range dc.Detect {
		keys = append(keys, k)
	}

	sort.Strings(keys)

	return keys, nil
}