crowdsec/pkg/database/alerts.go

package database

import (
	"encoding/json"
	"fmt"
	"strconv"
	"strings"
	"time"

	"github.com/crowdsecurity/crowdsec/pkg/database/ent"
	"github.com/crowdsecurity/crowdsec/pkg/database/ent/alert"
	"github.com/crowdsecurity/crowdsec/pkg/database/ent/decision"
	"github.com/crowdsecurity/crowdsec/pkg/database/ent/event"
	"github.com/crowdsecurity/crowdsec/pkg/database/ent/meta"
	"github.com/crowdsecurity/crowdsec/pkg/models"
	"github.com/crowdsecurity/crowdsec/pkg/types"
	"github.com/davecgh/go-spew/spew"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
)

const (
	paginationSize = 100 // used to queryAlert to avoid 'too many SQL variable'
	defaultLimit   = 100 // default limit of element to returns when query alerts
	bulkSize       = 50  // bulk size when create alerts
)

func formatAlertAsString(machineId string, alert *models.Alert) []string {
	var retStr []string

	/**/
	src := ""
	if alert.Source != nil {
		if *alert.Source.Scope == types.Ip {
			src = fmt.Sprintf("ip %s", *alert.Source.Value)
			if alert.Source.Cn != "" {
				src += " (" + alert.Source.Cn
				if alert.Source.AsNumber != "" {
					src += "/" + alert.Source.AsNumber
				}
				src += ")"
			}
		} else if *alert.Source.Scope == types.Range {
			src = fmt.Sprintf("range %s", *alert.Source.Value)
			if alert.Source.Cn != "" {
				src += " (" + alert.Source.Cn
				if alert.Source.AsNumber != "" {
					src += "/" + alert.Source.AsNumber
				}
				src += ")"
			}
		} else {
			src = fmt.Sprintf("%s %s", *alert.Source.Scope, *alert.Source.Value)
		}
	} else {
		src = "empty source"
	}

	/**/
	reason := ""
	if *alert.Scenario != "" {
		reason = fmt.Sprintf("%s by %s", *alert.Scenario, src)
	} else if *alert.Message != "" {
		reason = fmt.Sprintf("%s by %s", *alert.Scenario, src)
	} else {
		reason = fmt.Sprintf("empty scenario by %s", src)
	}

	if len(alert.Decisions) > 0 {
		for _, decisionItem := range alert.Decisions {
			decision := ""
			if alert.Simulated != nil && *alert.Simulated {
				decision = "(simulated alert)"
			} else if decisionItem.Simulated != nil && *decisionItem.Simulated {
				decision = "(simulated decision)"
			}
			if log.GetLevel() >= log.DebugLevel {
				/*spew is expensive*/
				log.Debugf("%s", spew.Sdump(decisionItem))
			}
			decision += fmt.Sprintf("%s %s on %s %s", *decisionItem.Duration,
				*decisionItem.Type, *decisionItem.Scope, *decisionItem.Value)
			retStr = append(retStr,
				fmt.Sprintf("(%s/%s) %s : %s", machineId,
					*decisionItem.Origin, reason, decision))
		}
	} else {
		retStr = append(retStr, fmt.Sprintf("(%s) alert : %s", machineId, reason))
	}
	return retStr
}

func (c *Client) CreateAlert(machineID string, alertList []*models.Alert) ([]string, error) {
	pageStart := 0
	pageEnd := bulkSize
	ret := []string{}
	for {
		if pageEnd >= len(alertList) {
			results, err := c.CreateAlertBulk(machineID, alertList[pageStart:])
			if err != nil {
				return []string{}, fmt.Errorf("unable to create alerts: %s", err)
			}
			ret = append(ret, results...)
			break
		}
		results, err := c.CreateAlertBulk(machineID, alertList[pageStart:pageEnd])
		if err != nil {
			return []string{}, fmt.Errorf("unable to create alerts: %s", err)
		}
		ret = append(ret, results...)
		pageStart += bulkSize
		pageEnd += bulkSize
	}
	return ret, nil
}

func (c *Client) CreateAlertBulk(machineId string, alertList []*models.Alert) ([]string, error) {

	ret := []string{}
	bulkSize := 20

	c.Log.Debugf("writting %d items", len(alertList))
	bulk := make([]*ent.AlertCreate, 0, bulkSize)
	for i, alertItem := range alertList {
		var decisions []*ent.Decision
		var metas []*ent.Meta
		var events []*ent.Event

		owner, err := c.QueryMachineByID(machineId)
		if err != nil {
			if errors.Cause(err) != UserNotExists {
				return []string{}, errors.Wrapf(QueryFail, "machine '%s': %s", alertItem.MachineID, err)
			}
			c.Log.Debugf("CreateAlertBulk: Machine Id %s doesn't exist", machineId)
			owner = nil
		}
		startAtTime, err := time.Parse(time.RFC3339, *alertItem.StartAt)
		if err != nil {
			return []string{}, errors.Wrapf(ParseTimeFail, "start_at field time '%s': %s", *alertItem.StartAt, err)
		}

		stopAtTime, err := time.Parse(time.RFC3339, *alertItem.StopAt)
		if err != nil {
			return []string{}, errors.Wrapf(ParseTimeFail, "stop_at field time '%s': %s", *alertItem.StopAt, err)
		}
		/*display proper alert in logs*/
		for _, disp := range formatAlertAsString(machineId, alertItem) {
			c.Log.Info(disp)
		}

		if len(alertItem.Events) > 0 {
			eventBulk := make([]*ent.EventCreate, len(alertItem.Events))
			for i, eventItem := range alertItem.Events {
				ts, err := time.Parse(time.RFC3339, *eventItem.Timestamp)
				if err != nil {
					return []string{}, errors.Wrapf(ParseTimeFail, "event timestamp '%s' : %s", *eventItem.Timestamp, err)
				}
				marshallMetas, err := json.Marshal(eventItem.Meta)
				if err != nil {
					return []string{}, errors.Wrapf(MarshalFail, "event meta '%v' : %s", eventItem.Meta, err)
				}

				eventBulk[i] = c.Ent.Event.Create().
					SetTime(ts).
					SetSerialized(string(marshallMetas))
			}
			events, err = c.Ent.Event.CreateBulk(eventBulk...).Save(c.CTX)
			if err != nil {
				return []string{}, errors.Wrapf(BulkError, "creating alert events: %s", err)
			}
		}

		if len(alertItem.Meta) > 0 {
			metaBulk := make([]*ent.MetaCreate, len(alertItem.Meta))
			for i, metaItem := range alertItem.Meta {
				metaBulk[i] = c.Ent.Meta.Create().
					SetKey(metaItem.Key).
					SetValue(metaItem.Value)
			}
			metas, err = c.Ent.Meta.CreateBulk(metaBulk...).Save(c.CTX)
			if err != nil {
				return []string{}, errors.Wrapf(BulkError, "creating alert meta: %s", err)
			}
		}

		ts, err := time.Parse(time.RFC3339, *alertItem.StopAt)
		if err != nil {
			c.Log.Errorf("While parsing StartAt of item %s : %s", *alertItem.StopAt, err)
			ts = time.Now()
		}
		if len(alertItem.Decisions) > 0 {
			decisionBulk := make([]*ent.DecisionCreate, len(alertItem.Decisions))
			for i, decisionItem := range alertItem.Decisions {
				var start_ip, start_sfx, end_ip, end_sfx int64
				var sz int

				duration, err := time.ParseDuration(*decisionItem.Duration)
				if err != nil {
					return []string{}, errors.Wrapf(ParseDurationFail, "decision duration '%v' : %s", decisionItem.Duration, err)
				}

				/*if the scope is IP or Range, convert the value to integers */
				if strings.ToLower(*decisionItem.Scope) == "ip" || strings.ToLower(*decisionItem.Scope) == "range" {
					sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(*decisionItem.Value)
					if err != nil {
						return []string{}, errors.Wrapf(ParseDurationFail, "invalid addr/range %s : %s", *decisionItem.Value, err)
					}
				}
				decisionBulk[i] = c.Ent.Decision.Create().
					SetUntil(ts.Add(duration)).
					SetScenario(*decisionItem.Scenario).
					SetType(*decisionItem.Type).
					SetStartIP(start_ip).
					SetStartSuffix(start_sfx).
					SetEndIP(end_ip).
					SetEndSuffix(end_sfx).
					SetIPSize(int64(sz)).
					SetValue(*decisionItem.Value).
					SetScope(*decisionItem.Scope).
					SetOrigin(*decisionItem.Origin).
					SetSimulated(*alertItem.Simulated)
			}
			decisions, err = c.Ent.Decision.CreateBulk(decisionBulk...).Save(c.CTX)
			if err != nil {
				return []string{}, errors.Wrapf(BulkError, "creating alert decisions: %s", err)

			}
		}

		alertB := c.Ent.Alert.
			Create().
			SetScenario(*alertItem.Scenario).
			SetMessage(*alertItem.Message).
			SetEventsCount(*alertItem.EventsCount).
			SetStartedAt(startAtTime).
			SetStoppedAt(stopAtTime).
			SetSourceScope(*alertItem.Source.Scope).
			SetSourceValue(*alertItem.Source.Value).
			SetSourceIp(alertItem.Source.IP).
			SetSourceRange(alertItem.Source.Range).
			SetSourceAsNumber(alertItem.Source.AsNumber).
			SetSourceAsName(alertItem.Source.AsName).
			SetSourceCountry(alertItem.Source.Cn).
			SetSourceLatitude(alertItem.Source.Latitude).
			SetSourceLongitude(alertItem.Source.Longitude).
			SetCapacity(*alertItem.Capacity).
			SetLeakSpeed(*alertItem.Leakspeed).
			SetSimulated(*alertItem.Simulated).
			SetScenarioVersion(*alertItem.ScenarioVersion).
			SetScenarioHash(*alertItem.ScenarioHash).
			AddDecisions(decisions...).
			AddEvents(events...).
			AddMetas(metas...)

		if owner != nil {
			alertB.SetOwner(owner)
		}
		bulk = append(bulk, alertB)

		if len(bulk) == bulkSize {
			alerts, err := c.Ent.Alert.CreateBulk(bulk...).Save(c.CTX)
			if err != nil {
				return []string{}, errors.Wrapf(BulkError, "bulk creating alert : %s", err)
			}
			for _, alert := range alerts {
				ret = append(ret, strconv.Itoa(alert.ID))
			}

			if len(alertList)-i <= bulkSize {
				bulk = make([]*ent.AlertCreate, 0, (len(alertList) - i))
			} else {
				bulk = make([]*ent.AlertCreate, 0, bulkSize)
			}
		}
	}

	alerts, err := c.Ent.Alert.CreateBulk(bulk...).Save(c.CTX)
	if err != nil {
		return []string{}, errors.Wrapf(BulkError, "leftovers creating alert : %s", err)
	}

	for _, alert := range alerts {
		ret = append(ret, strconv.Itoa(alert.ID))
	}

	return ret, nil
}

func BuildAlertRequestFromFilter(alerts *ent.AlertQuery, filter map[string][]string) (*ent.AlertQuery, error) {
	var err error
	var start_ip, start_sfx, end_ip, end_sfx int64
	var hasActiveDecision bool
	var ip_sz int
	var contains bool = true
	/*if contains is true, return bans that *contains* the given value (value is the inner)
	  else, return bans that are *contained* by the given value (value is the outer)*/

	/*the simulated filter is a bit different : if it's not present *or* set to false, specifically exclude records with simulated to true */
	if v, ok := filter["simulated"]; ok {
		if v[0] == "false" {
			alerts = alerts.Where(alert.SimulatedEQ(false))
		}
		delete(filter, "simulated")
	}

	for param, value := range filter {
		switch param {
		case "contains":
			contains, err = strconv.ParseBool(value[0])
			if err != nil {
				return nil, errors.Wrapf(InvalidFilter, "invalid contains value : %s", err)
			}
		case "scope":
			var scope string = value[0]
			if strings.ToLower(scope) == "ip" {
				scope = types.Ip
			} else if strings.ToLower(scope) == "range" {
				scope = types.Range
			}
			alerts = alerts.Where(alert.SourceScopeEQ(scope))
		case "value":
			alerts = alerts.Where(alert.SourceValueEQ(value[0]))
		case "scenario":
			alerts = alerts.Where(alert.ScenarioEQ(value[0]))
		case "ip", "range":
			ip_sz, start_ip, start_sfx, end_ip, end_sfx, err = types.Addr2Ints(value[0])
			if err != nil {
				return nil, errors.Wrapf(InvalidIPOrRange, "unable to convert '%s' to int: %s", value[0], err)
			}
		case "since":
			duration, err := types.ParseDuration(value[0])
			if err != nil {
				return nil, errors.Wrap(err, "while parsing duration")
			}
			since := time.Now().Add(-duration)
			if since.IsZero() {
				return nil, fmt.Errorf("Empty time now() - %s", since.String())
			}
			alerts = alerts.Where(alert.StartedAtGTE(since))
		case "created_before":
			duration, err := types.ParseDuration(value[0])
			if err != nil {
				return nil, errors.Wrap(err, "while parsing duration")
			}
			since := time.Now().Add(-duration)
			if since.IsZero() {
				return nil, fmt.Errorf("Empty time now() - %s", since.String())
			}
			alerts = alerts.Where(alert.CreatedAtLTE(since))
		case "until":
			duration, err := types.ParseDuration(value[0])
			if err != nil {
				return nil, errors.Wrap(err, "while parsing duration")
			}
			until := time.Now().Add(-duration)
			if until.IsZero() {
				return nil, fmt.Errorf("Empty time now() - %s", until.String())
			}
			alerts = alerts.Where(alert.StartedAtLTE(until))
		case "decision_type":
			alerts = alerts.Where(alert.HasDecisionsWith(decision.TypeEQ(value[0])))
		case "include_capi": //allows to exclude one or more specific origins
			if value[0] == "false" {
				alerts = alerts.Where(alert.HasDecisionsWith(decision.OriginNEQ("CAPI")))
			} else if value[0] != "true" {
				log.Errorf("Invalid bool '%s' for include_capi", value[0])
			}
		case "has_active_decision":
			if hasActiveDecision, err = strconv.ParseBool(value[0]); err != nil {
				return nil, errors.Wrapf(ParseType, "'%s' is not a boolean: %s", value[0], err)
			}
			if hasActiveDecision {
				alerts = alerts.Where(alert.HasDecisionsWith(decision.UntilGTE(time.Now())))
			} else {
				alerts = alerts.Where(alert.Not(alert.HasDecisions()))
			}
		case "limit":
			continue
		case "sort":
			continue
		default:
			return nil, errors.Wrapf(InvalidFilter, "Filter parameter '%s' is unknown (=%s)", param, value[0])
		}
	}

	if ip_sz == 4 {
		if contains { /*decision contains {start_ip,end_ip}*/
			alerts = alerts.Where(alert.And(
				alert.HasDecisionsWith(decision.StartIPLTE(start_ip)),
				alert.HasDecisionsWith(decision.EndIPGTE(end_ip)),
				alert.HasDecisionsWith(decision.IPSizeEQ(int64(ip_sz))),
			))
		} else { /*decision is contained within {start_ip,end_ip}*/
			alerts = alerts.Where(alert.And(
				alert.HasDecisionsWith(decision.StartIPGTE(start_ip)),
				alert.HasDecisionsWith(decision.EndIPLTE(end_ip)),
				alert.HasDecisionsWith(decision.IPSizeEQ(int64(ip_sz))),
			))
		}
	} else if ip_sz == 16 {

		if contains { /*decision contains {start_ip,end_ip}*/
			alerts = alerts.Where(alert.And(
				//matching addr size
				alert.HasDecisionsWith(decision.IPSizeEQ(int64(ip_sz))),
				alert.Or(
					//decision.start_ip < query.start_ip
					alert.HasDecisionsWith(decision.StartIPLT(start_ip)),
					alert.And(
						//decision.start_ip == query.start_ip
						alert.HasDecisionsWith(decision.StartIPEQ(start_ip)),
						//decision.start_suffix <= query.start_suffix
						alert.HasDecisionsWith(decision.StartSuffixLTE(start_sfx)),
					)),
				alert.Or(
					//decision.end_ip > query.end_ip
					alert.HasDecisionsWith(decision.EndIPGT(end_ip)),
					alert.And(
						//decision.end_ip == query.end_ip
						alert.HasDecisionsWith(decision.EndIPEQ(end_ip)),
						//decision.end_suffix >= query.end_suffix
						alert.HasDecisionsWith(decision.EndSuffixGTE(end_sfx)),
					),
				),
			))
		} else { /*decision is contained within {start_ip,end_ip}*/
			alerts = alerts.Where(alert.And(
				//matching addr size
				alert.HasDecisionsWith(decision.IPSizeEQ(int64(ip_sz))),
				alert.Or(
					//decision.start_ip > query.start_ip
					alert.HasDecisionsWith(decision.StartIPGT(start_ip)),
					alert.And(
						//decision.start_ip == query.start_ip
						alert.HasDecisionsWith(decision.StartIPEQ(start_ip)),
						//decision.start_suffix >= query.start_suffix
						alert.HasDecisionsWith(decision.StartSuffixGTE(start_sfx)),
					)),
				alert.Or(
					//decision.end_ip < query.end_ip
					alert.HasDecisionsWith(decision.EndIPLT(end_ip)),
					alert.And(
						//decision.end_ip == query.end_ip
						alert.HasDecisionsWith(decision.EndIPEQ(end_ip)),
						//decision.end_suffix <= query.end_suffix
						alert.HasDecisionsWith(decision.EndSuffixLTE(end_sfx)),
					),
				),
			))
		}
	} else if ip_sz != 0 {
		return nil, errors.Wrapf(InvalidFilter, "Unknown ip size %d", ip_sz)
	}
	return alerts, nil
}

func (c *Client) TotalAlerts() (int, error) {
	return c.Ent.Alert.Query().Count(c.CTX)
}

func (c *Client) QueryAlertWithFilter(filter map[string][]string) ([]*ent.Alert, error) {
	sort := "DESC" // we sort by desc by default
	if val, ok := filter["sort"]; ok {
		if val[0] != "ASC" && val[0] != "DESC" {
			c.Log.Errorf("invalid 'sort' parameter: %s", val)
		} else {
			sort = val[0]
		}
	}
	limit := defaultLimit
	if val, ok := filter["limit"]; ok {
		limitConv, err := strconv.Atoi(val[0])
		if err != nil {
			return []*ent.Alert{}, errors.Wrapf(QueryFail, "bad limit in parameters: %s", val)
		}
		limit = limitConv

	}
	offset := 0
	ret := make([]*ent.Alert, 0)
	for {
		alerts := c.Ent.Alert.Query()
		alerts, err := BuildAlertRequestFromFilter(alerts, filter)
		if err != nil {
			return []*ent.Alert{}, err
		}
		alerts = alerts.
			WithDecisions().
			WithEvents().
			WithMetas().
			WithOwner()
		if sort == "ASC" {
			alerts = alerts.Order(ent.Asc(alert.FieldCreatedAt))
		} else {
			alerts = alerts.Order(ent.Desc(alert.FieldCreatedAt))
		}
		if limit == 0 {
			limit, err = alerts.Count(c.CTX)
			if err != nil {
				return []*ent.Alert{}, fmt.Errorf("unable to count nb alerts: %s", err)
			}
		}
		result, err := alerts.Limit(paginationSize).Offset(offset).All(c.CTX)
		if err != nil {
			return []*ent.Alert{}, errors.Wrapf(QueryFail, "pagination size: %d, offset: %d: %s", paginationSize, offset, err)
		}
		if diff := limit - len(ret); diff < paginationSize {
			if len(result) < diff {
				ret = append(ret, result...)
				c.Log.Debugf("Pagination done, %d < %d", len(result), diff)
				break
			}
			ret = append(ret, result[0:diff]...)
		} else {
			ret = append(ret, result...)
		}
		if len(ret) == limit || len(ret) == 0 {
			c.Log.Debugf("Pagination done len(ret) = %d", len(ret))
			break
		}
		offset += paginationSize
	}

	return ret, nil
}

func (c *Client) DeleteAlertGraph(alertItem *ent.Alert) error {
	// delete the associated events
	_, err := c.Ent.Event.Delete().
		Where(event.HasOwnerWith(alert.IDEQ(alertItem.ID))).Exec(c.CTX)
	if err != nil {
		c.Log.Warningf("DeleteAlertGraph : %s", err)
		return errors.Wrapf(DeleteFail, "event with alert ID '%d'", alertItem.ID)
	}

	// delete the associated meta
	_, err = c.Ent.Meta.Delete().
		Where(meta.HasOwnerWith(alert.IDEQ(alertItem.ID))).Exec(c.CTX)
	if err != nil {
		c.Log.Warningf("DeleteAlertGraph : %s", err)
		return errors.Wrapf(DeleteFail, "meta with alert ID '%d'", alertItem.ID)
	}

	// delete the associated decisions
	_, err = c.Ent.Decision.Delete().
		Where(decision.HasOwnerWith(alert.IDEQ(alertItem.ID))).Exec(c.CTX)
	if err != nil {
		c.Log.Warningf("DeleteAlertGraph : %s", err)
		return errors.Wrapf(DeleteFail, "decision with alert ID '%d'", alertItem.ID)
	}

	// delete the alert
	err = c.Ent.Alert.DeleteOne(alertItem).Exec(c.CTX)
	if err != nil {
		c.Log.Warningf("DeleteAlertGraph : %s", err)
		return errors.Wrapf(DeleteFail, "alert with ID '%d'", alertItem.ID)
	}

	return nil
}

func (c *Client) DeleteAlertWithFilter(filter map[string][]string) (int, error) {
	var err error

	// Get all the alerts that match the filter
	alertsToDelete, err := c.QueryAlertWithFilter(filter)

	for _, alertItem := range alertsToDelete {
		err = c.DeleteAlertGraph(alertItem)
		if err != nil {
			c.Log.Warningf("DeleteAlertWithFilter : %s", err)
			return 0, errors.Wrapf(DeleteFail, "event with alert ID '%d'", alertItem.ID)
		}
	}
	return len(alertsToDelete), nil
}

func (c *Client) FlushAlerts(MaxAge string, MaxItems int) error {
	var deletedByAge int
	var deletedByNbItem int
	var totalAlerts int
	var err error
	totalAlerts, err = c.TotalAlerts()
	if err != nil {
		c.Log.Warningf("FlushAlerts (max items count) : %s", err)
		return errors.Wrap(err, "unable to get alerts count")
	}
	if MaxAge != "" {
		filter := map[string][]string{
			"created_before": {MaxAge},
		}
		nbDeleted, err := c.DeleteAlertWithFilter(filter)
		if err != nil {
			c.Log.Warningf("FlushAlerts (max age) : %s", err)
			return errors.Wrapf(err, "unable to flush alerts with filter until: %s", MaxAge)
		}
		deletedByAge = nbDeleted
	}
	if MaxItems > 0 {
		if totalAlerts > MaxItems {
			nbToDelete := totalAlerts - MaxItems
			alerts, err := c.QueryAlertWithFilter(map[string][]string{
				"sort":  {"ASC"},
				"limit": {strconv.Itoa(nbToDelete)},
			}) // we want to delete older alerts if we reach the max number of items
			if err != nil {
				c.Log.Warningf("FlushAlerts (max items query) : %s", err)
				return errors.Wrap(err, "unable to get all alerts")
			}
			for itemNb, alert := range alerts {
				if itemNb < nbToDelete {
					err := c.DeleteAlertGraph(alert)
					if err != nil {
						c.Log.Warningf("FlushAlerts : %s", err)
						return errors.Wrap(err, "unable to flush alert")
					}
					deletedByNbItem++
				}
			}
		}
	}
	if deletedByNbItem > 0 {
		c.Log.Infof("flushed %d/%d alerts because max number of alerts has been reached (%d max)", deletedByNbItem, totalAlerts, MaxItems)
	}
	if deletedByAge > 0 {
		c.Log.Infof("flushed %d/%d alerts because they were created %s ago or more", deletedByAge, totalAlerts, MaxAge)
	}
	return nil
}

func (c *Client) GetAlertByID(alertID int) (*ent.Alert, error) {
	alert, err := c.Ent.Alert.Query().Where(alert.IDEQ(alertID)).WithDecisions().WithEvents().WithMetas().WithOwner().First(c.CTX)
	if err != nil {
		/*record not found, 404*/
		if ent.IsNotFound(err) {
			log.Warningf("GetAlertByID (not found): %s", err)
			return &ent.Alert{}, ItemNotFound
		}
		c.Log.Warningf("GetAlertByID : %s", err)
		return &ent.Alert{}, QueryFail
	}
	return alert, nil
}