utils.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779
  1. package main
  2. import (
  3. "bytes"
  4. "encoding/csv"
  5. "encoding/json"
  6. "fmt"
  7. "io/ioutil"
  8. "math"
  9. "net"
  10. "net/http"
  11. "os"
  12. "strconv"
  13. "strings"
  14. "time"
  15. "github.com/crowdsecurity/crowdsec/pkg/cwhub"
  16. "github.com/crowdsecurity/crowdsec/pkg/types"
  17. "github.com/enescakir/emoji"
  18. "github.com/olekukonko/tablewriter"
  19. dto "github.com/prometheus/client_model/go"
  20. "github.com/prometheus/prom2json"
  21. log "github.com/sirupsen/logrus"
  22. "github.com/spf13/cobra"
  23. "github.com/texttheater/golang-levenshtein/levenshtein"
  24. "gopkg.in/yaml.v2"
  25. )
  26. const MaxDistance = 7
  27. func printHelp(cmd *cobra.Command) {
  28. err := cmd.Help()
  29. if err != nil {
  30. log.Fatalf("unable to print help(): %s", err)
  31. }
  32. }
  33. func inSlice(s string, slice []string) bool {
  34. for _, str := range slice {
  35. if s == str {
  36. return true
  37. }
  38. }
  39. return false
  40. }
  41. func indexOf(s string, slice []string) int {
  42. for i, elem := range slice {
  43. if s == elem {
  44. return i
  45. }
  46. }
  47. return -1
  48. }
  49. func LoadHub() error {
  50. if err := csConfig.LoadHub(); err != nil {
  51. log.Fatal(err)
  52. }
  53. if csConfig.Hub == nil {
  54. return fmt.Errorf("unable to load hub")
  55. }
  56. if err := cwhub.SetHubBranch(); err != nil {
  57. log.Warningf("unable to set hub branch (%s), default to master", err)
  58. }
  59. if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
  60. return fmt.Errorf("Failed to get Hub index : '%w'. Run 'sudo cscli hub update' to get the hub index", err)
  61. }
  62. return nil
  63. }
  64. func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
  65. errMsg := ""
  66. if score < MaxDistance {
  67. errMsg = fmt.Sprintf("unable to find %s '%s', did you mean %s ?", itemType, baseItem, suggestItem)
  68. } else {
  69. errMsg = fmt.Sprintf("unable to find %s '%s'", itemType, baseItem)
  70. }
  71. if ignoreErr {
  72. log.Error(errMsg)
  73. } else {
  74. log.Fatalf(errMsg)
  75. }
  76. }
  77. func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
  78. allItems := make([]string, 0)
  79. nearestScore := 100
  80. nearestItem := &cwhub.Item{}
  81. hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
  82. for _, item := range hubItems {
  83. allItems = append(allItems, item.Name)
  84. }
  85. for _, s := range allItems {
  86. d := levenshtein.DistanceForStrings([]rune(itemName), []rune(s), levenshtein.DefaultOptions)
  87. if d < nearestScore {
  88. nearestScore = d
  89. nearestItem = cwhub.GetItem(itemType, s)
  90. }
  91. }
  92. return nearestItem, nearestScore
  93. }
  94. func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
  95. if err := LoadHub(); err != nil {
  96. return nil, cobra.ShellCompDirectiveDefault
  97. }
  98. comp := make([]string, 0)
  99. hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
  100. for _, item := range hubItems {
  101. if toComplete == "" {
  102. comp = append(comp, item.Name)
  103. } else {
  104. if strings.Contains(item.Name, toComplete) {
  105. comp = append(comp, item.Name)
  106. }
  107. }
  108. }
  109. cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
  110. return comp, cobra.ShellCompDirectiveNoFileComp
  111. }
  112. func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
  113. if err := LoadHub(); err != nil {
  114. return nil, cobra.ShellCompDirectiveDefault
  115. }
  116. var items []string
  117. var err error
  118. switch itemType {
  119. case cwhub.PARSERS:
  120. items, err = cwhub.GetInstalledParsersAsString()
  121. case cwhub.SCENARIOS:
  122. items, err = cwhub.GetInstalledScenariosAsString()
  123. case cwhub.PARSERS_OVFLW:
  124. items, err = cwhub.GetInstalledPostOverflowsAsString()
  125. case cwhub.COLLECTIONS:
  126. items, err = cwhub.GetInstalledCollectionsAsString()
  127. default:
  128. return nil, cobra.ShellCompDirectiveDefault
  129. }
  130. if err != nil {
  131. cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
  132. return nil, cobra.ShellCompDirectiveDefault
  133. }
  134. comp := make([]string, 0)
  135. if toComplete != "" {
  136. for _, item := range items {
  137. if strings.Contains(item, toComplete) {
  138. comp = append(comp, item)
  139. }
  140. }
  141. } else {
  142. comp = items
  143. }
  144. cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
  145. return comp, cobra.ShellCompDirectiveNoFileComp
  146. }
  147. func ListItems(itemTypes []string, args []string, showType bool, showHeader bool, all bool) []byte {
  148. var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus)
  149. for _, itemType := range itemTypes {
  150. itemName := ""
  151. if len(args) == 1 {
  152. itemName = args[0]
  153. }
  154. hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all)
  155. }
  156. w := bytes.NewBuffer(nil)
  157. if csConfig.Cscli.Output == "human" {
  158. for _, itemType := range itemTypes {
  159. var statuses []cwhub.ItemHubStatus
  160. var ok bool
  161. if statuses, ok = hubStatusByItemType[itemType]; !ok {
  162. log.Errorf("unknown item type: %s", itemType)
  163. continue
  164. }
  165. fmt.Fprintf(w, "%s\n", strings.ToUpper(itemType))
  166. table := tablewriter.NewWriter(w)
  167. table.SetCenterSeparator("")
  168. table.SetColumnSeparator("")
  169. table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
  170. table.SetAlignment(tablewriter.ALIGN_LEFT)
  171. table.SetHeader([]string{"Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path"})
  172. for _, status := range statuses {
  173. table.Append([]string{status.Name, status.UTF8_Status, status.LocalVersion, status.LocalPath})
  174. }
  175. table.Render()
  176. }
  177. } else if csConfig.Cscli.Output == "json" {
  178. x, err := json.MarshalIndent(hubStatusByItemType, "", " ")
  179. if err != nil {
  180. log.Fatalf("failed to unmarshal")
  181. }
  182. w.Write(x)
  183. } else if csConfig.Cscli.Output == "raw" {
  184. csvwriter := csv.NewWriter(w)
  185. if showHeader {
  186. header := []string{"name", "status", "version", "description"}
  187. if showType {
  188. header = append(header, "type")
  189. }
  190. err := csvwriter.Write(header)
  191. if err != nil {
  192. log.Fatalf("failed to write header: %s", err)
  193. }
  194. }
  195. for _, itemType := range itemTypes {
  196. var statuses []cwhub.ItemHubStatus
  197. var ok bool
  198. if statuses, ok = hubStatusByItemType[itemType]; !ok {
  199. log.Errorf("unknown item type: %s", itemType)
  200. continue
  201. }
  202. for _, status := range statuses {
  203. if status.LocalVersion == "" {
  204. status.LocalVersion = "n/a"
  205. }
  206. row := []string{
  207. status.Name,
  208. status.Status,
  209. status.LocalVersion,
  210. status.Description,
  211. }
  212. if showType {
  213. row = append(row, itemType)
  214. }
  215. err := csvwriter.Write(row)
  216. if err != nil {
  217. log.Fatalf("failed to write raw output : %s", err)
  218. }
  219. }
  220. }
  221. csvwriter.Flush()
  222. }
  223. return w.Bytes()
  224. }
  225. func InspectItem(name string, objecitemType string) {
  226. hubItem := cwhub.GetItem(objecitemType, name)
  227. if hubItem == nil {
  228. log.Fatalf("unable to retrieve item.")
  229. }
  230. var b []byte
  231. var err error
  232. switch csConfig.Cscli.Output {
  233. case "human", "raw":
  234. b, err = yaml.Marshal(*hubItem)
  235. if err != nil {
  236. log.Fatalf("unable to marshal item : %s", err)
  237. }
  238. case "json":
  239. b, err = json.MarshalIndent(*hubItem, "", " ")
  240. if err != nil {
  241. log.Fatalf("unable to marshal item : %s", err)
  242. }
  243. }
  244. fmt.Printf("%s", string(b))
  245. if csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
  246. return
  247. }
  248. if csConfig.Prometheus.Enabled {
  249. if csConfig.Prometheus.ListenAddr == "" || csConfig.Prometheus.ListenPort == 0 {
  250. log.Warningf("No prometheus address or port specified in '%s', can't show metrics", *csConfig.FilePath)
  251. return
  252. }
  253. if prometheusURL == "" {
  254. log.Debugf("No prometheus URL provided using: %s:%d", csConfig.Prometheus.ListenAddr, csConfig.Prometheus.ListenPort)
  255. prometheusURL = fmt.Sprintf("http://%s:%d/metrics", csConfig.Prometheus.ListenAddr, csConfig.Prometheus.ListenPort)
  256. }
  257. fmt.Printf("\nCurrent metrics : \n\n")
  258. ShowMetrics(hubItem)
  259. }
  260. }
  261. func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
  262. /*if a range is provided, change the scope*/
  263. if *ipRange != "" {
  264. _, _, err := net.ParseCIDR(*ipRange)
  265. if err != nil {
  266. return fmt.Errorf("%s isn't a valid range", *ipRange)
  267. }
  268. }
  269. if *ip != "" {
  270. ipRepr := net.ParseIP(*ip)
  271. if ipRepr == nil {
  272. return fmt.Errorf("%s isn't a valid ip", *ip)
  273. }
  274. }
  275. //avoid confusion on scope (ip vs Ip and range vs Range)
  276. switch strings.ToLower(*scope) {
  277. case "ip":
  278. *scope = types.Ip
  279. case "range":
  280. *scope = types.Range
  281. case "country":
  282. *scope = types.Country
  283. case "as":
  284. *scope = types.AS
  285. }
  286. return nil
  287. }
  288. func ShowMetrics(hubItem *cwhub.Item) {
  289. switch hubItem.Type {
  290. case cwhub.PARSERS:
  291. metrics := GetParserMetric(prometheusURL, hubItem.Name)
  292. ShowParserMetric(hubItem.Name, metrics)
  293. case cwhub.SCENARIOS:
  294. metrics := GetScenarioMetric(prometheusURL, hubItem.Name)
  295. ShowScenarioMetric(hubItem.Name, metrics)
  296. case cwhub.COLLECTIONS:
  297. for _, item := range hubItem.Parsers {
  298. metrics := GetParserMetric(prometheusURL, item)
  299. ShowParserMetric(item, metrics)
  300. }
  301. for _, item := range hubItem.Scenarios {
  302. metrics := GetScenarioMetric(prometheusURL, item)
  303. ShowScenarioMetric(item, metrics)
  304. }
  305. for _, item := range hubItem.Collections {
  306. hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
  307. if hubItem == nil {
  308. log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
  309. }
  310. ShowMetrics(hubItem)
  311. }
  312. default:
  313. log.Errorf("item of type '%s' is unknown", hubItem.Type)
  314. }
  315. }
  316. /*This is a complete rip from prom2json*/
  317. func GetParserMetric(url string, itemName string) map[string]map[string]int {
  318. stats := make(map[string]map[string]int)
  319. result := GetPrometheusMetric(url)
  320. for idx, fam := range result {
  321. if !strings.HasPrefix(fam.Name, "cs_") {
  322. continue
  323. }
  324. log.Tracef("round %d", idx)
  325. for _, m := range fam.Metrics {
  326. metric, ok := m.(prom2json.Metric)
  327. if !ok {
  328. log.Debugf("failed to convert metric to prom2json.Metric")
  329. continue
  330. }
  331. name, ok := metric.Labels["name"]
  332. if !ok {
  333. log.Debugf("no name in Metric %v", metric.Labels)
  334. }
  335. if name != itemName {
  336. continue
  337. }
  338. source, ok := metric.Labels["source"]
  339. if !ok {
  340. log.Debugf("no source in Metric %v", metric.Labels)
  341. } else {
  342. if srctype, ok := metric.Labels["type"]; ok {
  343. source = srctype + ":" + source
  344. }
  345. }
  346. value := m.(prom2json.Metric).Value
  347. fval, err := strconv.ParseFloat(value, 32)
  348. if err != nil {
  349. log.Errorf("Unexpected int value %s : %s", value, err)
  350. continue
  351. }
  352. ival := int(fval)
  353. switch fam.Name {
  354. case "cs_reader_hits_total":
  355. if _, ok := stats[source]; !ok {
  356. stats[source] = make(map[string]int)
  357. stats[source]["parsed"] = 0
  358. stats[source]["reads"] = 0
  359. stats[source]["unparsed"] = 0
  360. stats[source]["hits"] = 0
  361. }
  362. stats[source]["reads"] += ival
  363. case "cs_parser_hits_ok_total":
  364. if _, ok := stats[source]; !ok {
  365. stats[source] = make(map[string]int)
  366. }
  367. stats[source]["parsed"] += ival
  368. case "cs_parser_hits_ko_total":
  369. if _, ok := stats[source]; !ok {
  370. stats[source] = make(map[string]int)
  371. }
  372. stats[source]["unparsed"] += ival
  373. case "cs_node_hits_total":
  374. if _, ok := stats[source]; !ok {
  375. stats[source] = make(map[string]int)
  376. }
  377. stats[source]["hits"] += ival
  378. case "cs_node_hits_ok_total":
  379. if _, ok := stats[source]; !ok {
  380. stats[source] = make(map[string]int)
  381. }
  382. stats[source]["parsed"] += ival
  383. case "cs_node_hits_ko_total":
  384. if _, ok := stats[source]; !ok {
  385. stats[source] = make(map[string]int)
  386. }
  387. stats[source]["unparsed"] += ival
  388. default:
  389. continue
  390. }
  391. }
  392. }
  393. return stats
  394. }
  395. func GetScenarioMetric(url string, itemName string) map[string]int {
  396. stats := make(map[string]int)
  397. stats["instanciation"] = 0
  398. stats["curr_count"] = 0
  399. stats["overflow"] = 0
  400. stats["pour"] = 0
  401. stats["underflow"] = 0
  402. result := GetPrometheusMetric(url)
  403. for idx, fam := range result {
  404. if !strings.HasPrefix(fam.Name, "cs_") {
  405. continue
  406. }
  407. log.Tracef("round %d", idx)
  408. for _, m := range fam.Metrics {
  409. metric, ok := m.(prom2json.Metric)
  410. if !ok {
  411. log.Debugf("failed to convert metric to prom2json.Metric")
  412. continue
  413. }
  414. name, ok := metric.Labels["name"]
  415. if !ok {
  416. log.Debugf("no name in Metric %v", metric.Labels)
  417. }
  418. if name != itemName {
  419. continue
  420. }
  421. value := m.(prom2json.Metric).Value
  422. fval, err := strconv.ParseFloat(value, 32)
  423. if err != nil {
  424. log.Errorf("Unexpected int value %s : %s", value, err)
  425. continue
  426. }
  427. ival := int(fval)
  428. switch fam.Name {
  429. case "cs_bucket_created_total":
  430. stats["instanciation"] += ival
  431. case "cs_buckets":
  432. stats["curr_count"] += ival
  433. case "cs_bucket_overflowed_total":
  434. stats["overflow"] += ival
  435. case "cs_bucket_poured_total":
  436. stats["pour"] += ival
  437. case "cs_bucket_underflowed_total":
  438. stats["underflow"] += ival
  439. default:
  440. continue
  441. }
  442. }
  443. }
  444. return stats
  445. }
  446. //it's a rip of the cli version, but in silent-mode
  447. func silenceInstallItem(name string, obtype string) (string, error) {
  448. var item = cwhub.GetItem(obtype, name)
  449. if item == nil {
  450. return "", fmt.Errorf("error retrieving item")
  451. }
  452. it := *item
  453. if downloadOnly && it.Downloaded && it.UpToDate {
  454. return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
  455. }
  456. it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction, false)
  457. if err != nil {
  458. return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
  459. }
  460. if err := cwhub.AddItem(obtype, it); err != nil {
  461. return "", err
  462. }
  463. if downloadOnly {
  464. return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
  465. }
  466. it, err = cwhub.EnableItem(csConfig.Hub, it)
  467. if err != nil {
  468. return "", fmt.Errorf("error while enabling %s : %v", it.Name, err)
  469. }
  470. if err := cwhub.AddItem(obtype, it); err != nil {
  471. return "", err
  472. }
  473. return fmt.Sprintf("Enabled %s", it.Name), nil
  474. }
  475. func GetPrometheusMetric(url string) []*prom2json.Family {
  476. mfChan := make(chan *dto.MetricFamily, 1024)
  477. // Start with the DefaultTransport for sane defaults.
  478. transport := http.DefaultTransport.(*http.Transport).Clone()
  479. // Conservatively disable HTTP keep-alives as this program will only
  480. // ever need a single HTTP request.
  481. transport.DisableKeepAlives = true
  482. // Timeout early if the server doesn't even return the headers.
  483. transport.ResponseHeaderTimeout = time.Minute
  484. go func() {
  485. defer types.CatchPanic("crowdsec/GetPrometheusMetric")
  486. err := prom2json.FetchMetricFamilies(url, mfChan, transport)
  487. if err != nil {
  488. log.Fatalf("failed to fetch prometheus metrics : %v", err)
  489. }
  490. }()
  491. result := []*prom2json.Family{}
  492. for mf := range mfChan {
  493. result = append(result, prom2json.NewFamily(mf))
  494. }
  495. log.Debugf("Finished reading prometheus output, %d entries", len(result))
  496. return result
  497. }
  498. func ShowScenarioMetric(itemName string, metrics map[string]int) {
  499. if metrics["instanciation"] == 0 {
  500. return
  501. }
  502. table := tablewriter.NewWriter(os.Stdout)
  503. table.SetHeader([]string{"Current Count", "Overflows", "Instanciated", "Poured", "Expired"})
  504. table.Append([]string{fmt.Sprintf("%d", metrics["curr_count"]), fmt.Sprintf("%d", metrics["overflow"]), fmt.Sprintf("%d", metrics["instanciation"]), fmt.Sprintf("%d", metrics["pour"]), fmt.Sprintf("%d", metrics["underflow"])})
  505. fmt.Printf(" - (Scenario) %s: \n", itemName)
  506. table.Render()
  507. fmt.Println()
  508. }
  509. func ShowParserMetric(itemName string, metrics map[string]map[string]int) {
  510. skip := true
  511. table := tablewriter.NewWriter(os.Stdout)
  512. table.SetHeader([]string{"Parsers", "Hits", "Parsed", "Unparsed"})
  513. for source, stats := range metrics {
  514. if stats["hits"] > 0 {
  515. table.Append([]string{source, fmt.Sprintf("%d", stats["hits"]), fmt.Sprintf("%d", stats["parsed"]), fmt.Sprintf("%d", stats["unparsed"])})
  516. skip = false
  517. }
  518. }
  519. if !skip {
  520. fmt.Printf(" - (Parser) %s: \n", itemName)
  521. table.Render()
  522. fmt.Println()
  523. }
  524. }
  525. func RestoreHub(dirPath string) error {
  526. var err error
  527. if err := csConfig.LoadHub(); err != nil {
  528. return err
  529. }
  530. if err := cwhub.SetHubBranch(); err != nil {
  531. return fmt.Errorf("error while setting hub branch: %s", err)
  532. }
  533. for _, itype := range cwhub.ItemTypes {
  534. itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
  535. if _, err = os.Stat(itemDirectory); err != nil {
  536. log.Infof("no %s in backup", itype)
  537. continue
  538. }
  539. /*restore the upstream items*/
  540. upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
  541. file, err := ioutil.ReadFile(upstreamListFN)
  542. if err != nil {
  543. return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
  544. }
  545. var upstreamList []string
  546. err = json.Unmarshal(file, &upstreamList)
  547. if err != nil {
  548. return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
  549. }
  550. for _, toinstall := range upstreamList {
  551. label, err := silenceInstallItem(toinstall, itype)
  552. if err != nil {
  553. log.Errorf("Error while installing %s : %s", toinstall, err)
  554. } else if label != "" {
  555. log.Infof("Installed %s : %s", toinstall, label)
  556. } else {
  557. log.Printf("Installed %s : ok", toinstall)
  558. }
  559. }
  560. /*restore the local and tainted items*/
  561. files, err := ioutil.ReadDir(itemDirectory)
  562. if err != nil {
  563. return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
  564. }
  565. for _, file := range files {
  566. //this was the upstream data
  567. if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
  568. continue
  569. }
  570. if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
  571. //we expect a stage here
  572. if !file.IsDir() {
  573. continue
  574. }
  575. stage := file.Name()
  576. stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
  577. log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
  578. if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
  579. return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
  580. }
  581. /*find items*/
  582. ifiles, err := ioutil.ReadDir(itemDirectory + "/" + stage + "/")
  583. if err != nil {
  584. return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
  585. }
  586. //finally copy item
  587. for _, tfile := range ifiles {
  588. log.Infof("Going to restore local/tainted [%s]", tfile.Name())
  589. sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
  590. destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
  591. if err = types.CopyFile(sourceFile, destinationFile); err != nil {
  592. return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
  593. }
  594. log.Infof("restored %s to %s", sourceFile, destinationFile)
  595. }
  596. } else {
  597. log.Infof("Going to restore local/tainted [%s]", file.Name())
  598. sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
  599. destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
  600. if err = types.CopyFile(sourceFile, destinationFile); err != nil {
  601. return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
  602. }
  603. log.Infof("restored %s to %s", sourceFile, destinationFile)
  604. }
  605. }
  606. }
  607. return nil
  608. }
  609. func BackupHub(dirPath string) error {
  610. var err error
  611. var itemDirectory string
  612. var upstreamParsers []string
  613. for _, itemType := range cwhub.ItemTypes {
  614. clog := log.WithFields(log.Fields{
  615. "type": itemType,
  616. })
  617. itemMap := cwhub.GetItemMap(itemType)
  618. if itemMap == nil {
  619. clog.Infof("No %s to backup.", itemType)
  620. continue
  621. }
  622. itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
  623. if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
  624. return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
  625. }
  626. upstreamParsers = []string{}
  627. for k, v := range itemMap {
  628. clog = clog.WithFields(log.Fields{
  629. "file": v.Name,
  630. })
  631. if !v.Installed { //only backup installed ones
  632. clog.Debugf("[%s] : not installed", k)
  633. continue
  634. }
  635. //for the local/tainted ones, we backup the full file
  636. if v.Tainted || v.Local || !v.UpToDate {
  637. //we need to backup stages for parsers
  638. if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
  639. fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
  640. if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
  641. return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
  642. }
  643. }
  644. clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
  645. tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
  646. if err = types.CopyFile(v.LocalPath, tfile); err != nil {
  647. return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
  648. }
  649. clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
  650. continue
  651. }
  652. clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
  653. clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
  654. upstreamParsers = append(upstreamParsers, v.Name)
  655. }
  656. //write the upstream items
  657. upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
  658. upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
  659. if err != nil {
  660. return fmt.Errorf("failed marshaling upstream parsers : %s", err)
  661. }
  662. err = ioutil.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
  663. if err != nil {
  664. return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
  665. }
  666. clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
  667. }
  668. return nil
  669. }
  670. type unit struct {
  671. value int64
  672. symbol string
  673. }
  674. var ranges = []unit{
  675. {
  676. value: 1e18,
  677. symbol: "E",
  678. },
  679. {
  680. value: 1e15,
  681. symbol: "P",
  682. },
  683. {
  684. value: 1e12,
  685. symbol: "T",
  686. },
  687. {
  688. value: 1e6,
  689. symbol: "M",
  690. },
  691. {
  692. value: 1e3,
  693. symbol: "k",
  694. },
  695. {
  696. value: 1,
  697. symbol: "",
  698. },
  699. }
  700. func formatNumber(num int) string {
  701. goodUnit := unit{}
  702. for _, u := range ranges {
  703. if int64(num) >= u.value {
  704. goodUnit = u
  705. break
  706. }
  707. }
  708. if goodUnit.value == 1 {
  709. return fmt.Sprintf("%d%s", num, goodUnit.symbol)
  710. }
  711. res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
  712. return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
  713. }