exprlib.go 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486
  1. package exprhelpers
  2. import (
  3. "bufio"
  4. "fmt"
  5. "net"
  6. "net/url"
  7. "os"
  8. "path"
  9. "regexp"
  10. "strconv"
  11. "strings"
  12. "time"
  13. "github.com/bluele/gcache"
  14. "github.com/c-robinson/iplib"
  15. "github.com/cespare/xxhash/v2"
  16. "github.com/davecgh/go-spew/spew"
  17. "github.com/prometheus/client_golang/prometheus"
  18. log "github.com/sirupsen/logrus"
  19. "github.com/umahmood/haversine"
  20. "github.com/crowdsecurity/crowdsec/pkg/cache"
  21. "github.com/crowdsecurity/crowdsec/pkg/database"
  22. "github.com/crowdsecurity/crowdsec/pkg/types"
  23. )
  24. var dataFile map[string][]string
  25. var dataFileRegex map[string][]*regexp.Regexp
  26. // This is used to (optionally) cache regexp results for RegexpInFile operations
  27. var dataFileRegexCache map[string]gcache.Cache = make(map[string]gcache.Cache)
  28. /*prometheus*/
  29. var RegexpCacheMetrics = prometheus.NewGaugeVec(
  30. prometheus.GaugeOpts{
  31. Name: "cs_regexp_cache_size",
  32. Help: "Entries per regexp cache.",
  33. },
  34. []string{"name"},
  35. )
  36. var dbClient *database.Client
  37. func Get(arr []string, index int) string {
  38. if index >= len(arr) {
  39. return ""
  40. }
  41. return arr[index]
  42. }
  43. func Atof(x string) float64 {
  44. log.Debugf("debug atof %s", x)
  45. ret, err := strconv.ParseFloat(x, 64)
  46. if err != nil {
  47. log.Warningf("Atof : can't convert float '%s' : %v", x, err)
  48. }
  49. return ret
  50. }
  51. func Upper(s string) string {
  52. return strings.ToUpper(s)
  53. }
  54. func Lower(s string) string {
  55. return strings.ToLower(s)
  56. }
  57. func GetExprEnv(ctx map[string]interface{}) map[string]interface{} {
  58. var ExprLib = map[string]interface{}{
  59. "Atof": Atof,
  60. "JsonExtract": JsonExtract,
  61. "JsonExtractUnescape": JsonExtractUnescape,
  62. "JsonExtractLib": JsonExtractLib,
  63. "JsonExtractSlice": JsonExtractSlice,
  64. "JsonExtractObject": JsonExtractObject,
  65. "ToJsonString": ToJson,
  66. "File": File,
  67. "RegexpInFile": RegexpInFile,
  68. "Upper": Upper,
  69. "Lower": Lower,
  70. "IpInRange": IpInRange,
  71. "TimeNow": TimeNow,
  72. "ParseUri": ParseUri,
  73. "PathUnescape": PathUnescape,
  74. "QueryUnescape": QueryUnescape,
  75. "PathEscape": PathEscape,
  76. "QueryEscape": QueryEscape,
  77. "XMLGetAttributeValue": XMLGetAttributeValue,
  78. "XMLGetNodeValue": XMLGetNodeValue,
  79. "IpToRange": IpToRange,
  80. "IsIPV6": IsIPV6,
  81. "IsIPV4": IsIPV4,
  82. "IsIP": IsIP,
  83. "LookupHost": LookupHost,
  84. "GetDecisionsCount": GetDecisionsCount,
  85. "GetDecisionsSinceCount": GetDecisionsSinceCount,
  86. "Sprintf": fmt.Sprintf,
  87. "CrowdsecCTI": CrowdsecCTI,
  88. "ParseUnix": ParseUnix,
  89. "GetFromStash": cache.GetKey,
  90. "SetInStash": cache.SetKey,
  91. //go 1.20 "CutPrefix": strings.CutPrefix,
  92. //go 1.20 "CutSuffix": strings.CutSuffix,
  93. //"Cut": strings.Cut, -> returns more than 2 values, not supported by expr
  94. "Fields": strings.Fields,
  95. "Index": strings.Index,
  96. "IndexAny": strings.IndexAny,
  97. "Join": strings.Join,
  98. "Split": strings.Split,
  99. "SplitAfter": strings.SplitAfter,
  100. "SplitAfterN": strings.SplitAfterN,
  101. "SplitN": strings.SplitN,
  102. "Replace": strings.Replace,
  103. "ReplaceAll": strings.ReplaceAll,
  104. "Trim": strings.Trim,
  105. "TrimLeft": strings.TrimLeft,
  106. "TrimRight": strings.TrimRight,
  107. "TrimSpace": strings.TrimSpace,
  108. "TrimPrefix": strings.TrimPrefix,
  109. "TrimSuffix": strings.TrimSuffix,
  110. "Get": Get,
  111. "Distance": Distance,
  112. }
  113. for k, v := range ctx {
  114. ExprLib[k] = v
  115. }
  116. return ExprLib
  117. }
  118. func Distance(lat1 string, long1 string, lat2 string, long2 string) (float64, error) {
  119. lat1f, err := strconv.ParseFloat(lat1, 64)
  120. if err != nil {
  121. log.Warningf("lat1 is not a float : %v", err)
  122. return 0, fmt.Errorf("lat1 is not a float : %v", err)
  123. }
  124. long1f, err := strconv.ParseFloat(long1, 64)
  125. if err != nil {
  126. log.Warningf("long1 is not a float : %v", err)
  127. return 0, fmt.Errorf("long1 is not a float : %v", err)
  128. }
  129. lat2f, err := strconv.ParseFloat(lat2, 64)
  130. if err != nil {
  131. log.Warningf("lat2 is not a float : %v", err)
  132. return 0, fmt.Errorf("lat2 is not a float : %v", err)
  133. }
  134. long2f, err := strconv.ParseFloat(long2, 64)
  135. if err != nil {
  136. log.Warningf("long2 is not a float : %v", err)
  137. return 0, fmt.Errorf("long2 is not a float : %v", err)
  138. }
  139. //either set of coordinates is 0,0, return 0 to avoid FPs
  140. if (lat1f == 0.0 && long1f == 0.0) || (lat2f == 0.0 && long2f == 0.0) {
  141. log.Warningf("one of the coordinates is 0,0, returning 0")
  142. return 0, nil
  143. }
  144. first := haversine.Coord{Lat: lat1f, Lon: long1f}
  145. second := haversine.Coord{Lat: lat2f, Lon: long2f}
  146. _, km := haversine.Distance(first, second)
  147. return km, nil
  148. }
  149. func Init(databaseClient *database.Client) error {
  150. dataFile = make(map[string][]string)
  151. dataFileRegex = make(map[string][]*regexp.Regexp)
  152. dbClient = databaseClient
  153. return nil
  154. }
  155. func RegexpCacheInit(filename string, CacheCfg types.DataSource) error {
  156. //cache is explicitly disabled
  157. if CacheCfg.Cache != nil && !*CacheCfg.Cache {
  158. return nil
  159. }
  160. //cache is implicitly disabled if no cache config is provided
  161. if CacheCfg.Strategy == nil && CacheCfg.TTL == nil && CacheCfg.Size == nil {
  162. return nil
  163. }
  164. //cache is enabled
  165. if CacheCfg.Size == nil {
  166. CacheCfg.Size = types.IntPtr(50)
  167. }
  168. gc := gcache.New(*CacheCfg.Size)
  169. if CacheCfg.Strategy == nil {
  170. CacheCfg.Strategy = types.StrPtr("LRU")
  171. }
  172. switch *CacheCfg.Strategy {
  173. case "LRU":
  174. gc = gc.LRU()
  175. case "LFU":
  176. gc = gc.LFU()
  177. case "ARC":
  178. gc = gc.ARC()
  179. default:
  180. return fmt.Errorf("unknown cache strategy '%s'", *CacheCfg.Strategy)
  181. }
  182. if CacheCfg.TTL != nil {
  183. gc.Expiration(*CacheCfg.TTL)
  184. }
  185. cache := gc.Build()
  186. dataFileRegexCache[filename] = cache
  187. return nil
  188. }
  189. // UpdateCacheMetrics is called directly by the prom handler
  190. func UpdateRegexpCacheMetrics() {
  191. RegexpCacheMetrics.Reset()
  192. for name := range dataFileRegexCache {
  193. RegexpCacheMetrics.With(prometheus.Labels{"name": name}).Set(float64(dataFileRegexCache[name].Len(true)))
  194. }
  195. }
  196. func FileInit(fileFolder string, filename string, fileType string) error {
  197. log.Debugf("init (folder:%s) (file:%s) (type:%s)", fileFolder, filename, fileType)
  198. filepath := path.Join(fileFolder, filename)
  199. file, err := os.Open(filepath)
  200. if err != nil {
  201. return err
  202. }
  203. defer file.Close()
  204. if fileType == "" {
  205. log.Debugf("ignored file %s%s because no type specified", fileFolder, filename)
  206. return nil
  207. }
  208. if _, ok := dataFile[filename]; !ok {
  209. dataFile[filename] = []string{}
  210. }
  211. scanner := bufio.NewScanner(file)
  212. for scanner.Scan() {
  213. if strings.HasPrefix(scanner.Text(), "#") { // allow comments
  214. continue
  215. }
  216. if len(scanner.Text()) == 0 { //skip empty lines
  217. continue
  218. }
  219. switch fileType {
  220. case "regex", "regexp":
  221. dataFileRegex[filename] = append(dataFileRegex[filename], regexp.MustCompile(scanner.Text()))
  222. case "string":
  223. dataFile[filename] = append(dataFile[filename], scanner.Text())
  224. default:
  225. return fmt.Errorf("unknown data type '%s' for : '%s'", fileType, filename)
  226. }
  227. }
  228. if err := scanner.Err(); err != nil {
  229. return err
  230. }
  231. return nil
  232. }
  233. func QueryEscape(s string) string {
  234. return url.QueryEscape(s)
  235. }
  236. func PathEscape(s string) string {
  237. return url.PathEscape(s)
  238. }
  239. func PathUnescape(s string) string {
  240. ret, err := url.PathUnescape(s)
  241. if err != nil {
  242. log.Debugf("unable to PathUnescape '%s': %+v", s, err)
  243. return s
  244. }
  245. return ret
  246. }
  247. func QueryUnescape(s string) string {
  248. ret, err := url.QueryUnescape(s)
  249. if err != nil {
  250. log.Debugf("unable to QueryUnescape '%s': %+v", s, err)
  251. return s
  252. }
  253. return ret
  254. }
  255. func File(filename string) []string {
  256. if _, ok := dataFile[filename]; ok {
  257. return dataFile[filename]
  258. }
  259. log.Errorf("file '%s' (type:string) not found in expr library", filename)
  260. log.Errorf("expr library : %s", spew.Sdump(dataFile))
  261. return []string{}
  262. }
  263. func RegexpInFile(data string, filename string) bool {
  264. var hash uint64
  265. hasCache := false
  266. if _, ok := dataFileRegexCache[filename]; ok {
  267. hasCache = true
  268. hash = xxhash.Sum64String(data)
  269. if val, err := dataFileRegexCache[filename].Get(hash); err == nil {
  270. return val.(bool)
  271. }
  272. }
  273. if _, ok := dataFileRegex[filename]; ok {
  274. for _, re := range dataFileRegex[filename] {
  275. if re.Match([]byte(data)) {
  276. if hasCache {
  277. dataFileRegexCache[filename].Set(hash, true)
  278. }
  279. return true
  280. }
  281. }
  282. } else {
  283. log.Errorf("file '%s' (type:regexp) not found in expr library", filename)
  284. log.Errorf("expr library : %s", spew.Sdump(dataFileRegex))
  285. }
  286. if hasCache {
  287. dataFileRegexCache[filename].Set(hash, false)
  288. }
  289. return false
  290. }
  291. func IpInRange(ip string, ipRange string) bool {
  292. var err error
  293. var ipParsed net.IP
  294. var ipRangeParsed *net.IPNet
  295. ipParsed = net.ParseIP(ip)
  296. if ipParsed == nil {
  297. log.Debugf("'%s' is not a valid IP", ip)
  298. return false
  299. }
  300. if _, ipRangeParsed, err = net.ParseCIDR(ipRange); err != nil {
  301. log.Debugf("'%s' is not a valid IP Range", ipRange)
  302. return false
  303. }
  304. if ipRangeParsed.Contains(ipParsed) {
  305. return true
  306. }
  307. return false
  308. }
  309. func IsIPV6(ip string) bool {
  310. ipParsed := net.ParseIP(ip)
  311. if ipParsed == nil {
  312. log.Debugf("'%s' is not a valid IP", ip)
  313. return false
  314. }
  315. // If it's a valid IP and can't be converted to IPv4 then it is an IPv6
  316. return ipParsed.To4() == nil
  317. }
  318. func IsIPV4(ip string) bool {
  319. ipParsed := net.ParseIP(ip)
  320. if ipParsed == nil {
  321. log.Debugf("'%s' is not a valid IP", ip)
  322. return false
  323. }
  324. return ipParsed.To4() != nil
  325. }
  326. func IsIP(ip string) bool {
  327. ipParsed := net.ParseIP(ip)
  328. if ipParsed == nil {
  329. log.Debugf("'%s' is not a valid IP", ip)
  330. return false
  331. }
  332. return true
  333. }
  334. func IpToRange(ip string, cidr string) string {
  335. cidr = strings.TrimPrefix(cidr, "/")
  336. mask, err := strconv.Atoi(cidr)
  337. if err != nil {
  338. log.Errorf("bad cidr '%s': %s", cidr, err)
  339. return ""
  340. }
  341. ipAddr := net.ParseIP(ip)
  342. if ipAddr == nil {
  343. log.Errorf("can't parse IP address '%s'", ip)
  344. return ""
  345. }
  346. ipRange := iplib.NewNet(ipAddr, mask)
  347. if ipRange.IP() == nil {
  348. log.Errorf("can't get cidr '%s' of '%s'", cidr, ip)
  349. return ""
  350. }
  351. return ipRange.String()
  352. }
  353. func TimeNow() string {
  354. return time.Now().UTC().Format(time.RFC3339)
  355. }
  356. func ParseUri(uri string) map[string][]string {
  357. ret := make(map[string][]string)
  358. u, err := url.Parse(uri)
  359. if err != nil {
  360. log.Errorf("Could not parse URI: %s", err)
  361. return ret
  362. }
  363. parsed, err := url.ParseQuery(u.RawQuery)
  364. if err != nil {
  365. log.Errorf("Could not parse query uri : %s", err)
  366. return ret
  367. }
  368. for k, v := range parsed {
  369. ret[k] = v
  370. }
  371. return ret
  372. }
  373. func KeyExists(key string, dict map[string]interface{}) bool {
  374. _, ok := dict[key]
  375. return ok
  376. }
  377. func GetDecisionsCount(value string) int {
  378. if dbClient == nil {
  379. log.Error("No database config to call GetDecisionsCount()")
  380. return 0
  381. }
  382. count, err := dbClient.CountDecisionsByValue(value)
  383. if err != nil {
  384. log.Errorf("Failed to get decisions count from value '%s'", value)
  385. return 0
  386. }
  387. return count
  388. }
  389. func GetDecisionsSinceCount(value string, since string) int {
  390. if dbClient == nil {
  391. log.Error("No database config to call GetDecisionsCount()")
  392. return 0
  393. }
  394. sinceDuration, err := time.ParseDuration(since)
  395. if err != nil {
  396. log.Errorf("Failed to parse since parameter '%s' : %s", since, err)
  397. return 0
  398. }
  399. sinceTime := time.Now().UTC().Add(-sinceDuration)
  400. count, err := dbClient.CountDecisionsSinceByValue(value, sinceTime)
  401. if err != nil {
  402. log.Errorf("Failed to get decisions count from value '%s'", value)
  403. return 0
  404. }
  405. return count
  406. }
  407. func LookupHost(value string) []string {
  408. addresses, err := net.LookupHost(value)
  409. if err != nil {
  410. log.Errorf("Failed to lookup host '%s' : %s", value, err)
  411. return []string{}
  412. }
  413. return addresses
  414. }
  415. func ParseUnixTime(value string) (time.Time, error) {
  416. //Splitting string here as some unix timestamp may have milliseconds and break ParseInt
  417. i, err := strconv.ParseInt(strings.Split(value, ".")[0], 10, 64)
  418. if err != nil || i <= 0 {
  419. return time.Time{}, fmt.Errorf("unable to parse %s as unix timestamp", value)
  420. }
  421. return time.Unix(i, 0), nil
  422. }
  423. func ParseUnix(value string) string {
  424. t, err := ParseUnixTime(value)
  425. if err != nil {
  426. log.Error(err)
  427. return ""
  428. }
  429. return t.Format(time.RFC3339)
  430. }