utils.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775
  1. package main
  2. import (
  3. "encoding/csv"
  4. "encoding/json"
  5. "fmt"
  6. "io/ioutil"
  7. "math"
  8. "net"
  9. "net/http"
  10. "os"
  11. "strconv"
  12. "strings"
  13. "time"
  14. "github.com/crowdsecurity/crowdsec/pkg/cwhub"
  15. "github.com/crowdsecurity/crowdsec/pkg/types"
  16. "github.com/enescakir/emoji"
  17. "github.com/olekukonko/tablewriter"
  18. dto "github.com/prometheus/client_model/go"
  19. "github.com/prometheus/prom2json"
  20. log "github.com/sirupsen/logrus"
  21. "github.com/spf13/cobra"
  22. "github.com/texttheater/golang-levenshtein/levenshtein"
  23. "gopkg.in/yaml.v2"
  24. )
  25. const MaxDistance = 7
  26. func printHelp(cmd *cobra.Command) {
  27. err := cmd.Help()
  28. if err != nil {
  29. log.Fatalf("unable to print help(): %s", err)
  30. }
  31. }
  32. func inSlice(s string, slice []string) bool {
  33. for _, str := range slice {
  34. if s == str {
  35. return true
  36. }
  37. }
  38. return false
  39. }
  40. func indexOf(s string, slice []string) int {
  41. for i, elem := range slice {
  42. if s == elem {
  43. return i
  44. }
  45. }
  46. return -1
  47. }
  48. func LoadHub() error {
  49. if err := csConfig.LoadHub(); err != nil {
  50. log.Fatal(err)
  51. }
  52. if csConfig.Hub == nil {
  53. return fmt.Errorf("unable to load hub")
  54. }
  55. if err := cwhub.SetHubBranch(); err != nil {
  56. log.Warningf("unable to set hub branch (%s), default to master", err)
  57. }
  58. if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
  59. return fmt.Errorf("Failed to get Hub index : '%w'. Run 'sudo cscli hub update' to get the hub index", err)
  60. }
  61. return nil
  62. }
  63. func Suggest(itemType string, baseItem string, suggestItem string, score int, ignoreErr bool) {
  64. errMsg := ""
  65. if score < MaxDistance {
  66. errMsg = fmt.Sprintf("unable to find %s '%s', did you mean %s ?", itemType, baseItem, suggestItem)
  67. } else {
  68. errMsg = fmt.Sprintf("unable to find %s '%s'", itemType, baseItem)
  69. }
  70. if ignoreErr {
  71. log.Error(errMsg)
  72. } else {
  73. log.Fatalf(errMsg)
  74. }
  75. }
  76. func GetDistance(itemType string, itemName string) (*cwhub.Item, int) {
  77. allItems := make([]string, 0)
  78. nearestScore := 100
  79. nearestItem := &cwhub.Item{}
  80. hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
  81. for _, item := range hubItems {
  82. allItems = append(allItems, item.Name)
  83. }
  84. for _, s := range allItems {
  85. d := levenshtein.DistanceForStrings([]rune(itemName), []rune(s), levenshtein.DefaultOptions)
  86. if d < nearestScore {
  87. nearestScore = d
  88. nearestItem = cwhub.GetItem(itemType, s)
  89. }
  90. }
  91. return nearestItem, nearestScore
  92. }
  93. func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
  94. if err := LoadHub(); err != nil {
  95. return nil, cobra.ShellCompDirectiveDefault
  96. }
  97. comp := make([]string, 0)
  98. hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
  99. for _, item := range hubItems {
  100. if toComplete == "" {
  101. comp = append(comp, item.Name)
  102. } else {
  103. if strings.Contains(item.Name, toComplete) {
  104. comp = append(comp, item.Name)
  105. }
  106. }
  107. }
  108. cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
  109. return comp, cobra.ShellCompDirectiveNoFileComp
  110. }
  111. func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
  112. if err := LoadHub(); err != nil {
  113. return nil, cobra.ShellCompDirectiveDefault
  114. }
  115. var items []string
  116. var err error
  117. switch itemType {
  118. case cwhub.PARSERS:
  119. items, err = cwhub.GetInstalledParsersAsString()
  120. case cwhub.SCENARIOS:
  121. items, err = cwhub.GetInstalledScenariosAsString()
  122. case cwhub.PARSERS_OVFLW:
  123. items, err = cwhub.GetInstalledPostOverflowsAsString()
  124. case cwhub.COLLECTIONS:
  125. items, err = cwhub.GetInstalledCollectionsAsString()
  126. default:
  127. return nil, cobra.ShellCompDirectiveDefault
  128. }
  129. if err != nil {
  130. cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
  131. return nil, cobra.ShellCompDirectiveDefault
  132. }
  133. comp := make([]string, 0)
  134. if toComplete != "" {
  135. for _, item := range items {
  136. if strings.Contains(item, toComplete) {
  137. comp = append(comp, item)
  138. }
  139. }
  140. } else {
  141. comp = items
  142. }
  143. cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
  144. return comp, cobra.ShellCompDirectiveNoFileComp
  145. }
  146. func ListItems(itemTypes []string, args []string, showType bool, showHeader bool, all bool) {
  147. var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus)
  148. for _, itemType := range itemTypes {
  149. itemName := ""
  150. if len(args) == 1 {
  151. itemName = args[0]
  152. }
  153. hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all)
  154. }
  155. if csConfig.Cscli.Output == "human" {
  156. for _, itemType := range itemTypes {
  157. var statuses []cwhub.ItemHubStatus
  158. var ok bool
  159. if statuses, ok = hubStatusByItemType[itemType]; !ok {
  160. log.Errorf("unknown item type: %s", itemType)
  161. continue
  162. }
  163. fmt.Println(strings.ToUpper(itemType))
  164. table := tablewriter.NewWriter(os.Stdout)
  165. table.SetCenterSeparator("")
  166. table.SetColumnSeparator("")
  167. table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
  168. table.SetAlignment(tablewriter.ALIGN_LEFT)
  169. table.SetHeader([]string{"Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path"})
  170. for _, status := range statuses {
  171. table.Append([]string{status.Name, status.UTF8_Status, status.LocalVersion, status.LocalPath})
  172. }
  173. table.Render()
  174. }
  175. } else if csConfig.Cscli.Output == "json" {
  176. x, err := json.MarshalIndent(hubStatusByItemType, "", " ")
  177. if err != nil {
  178. log.Fatalf("failed to unmarshal")
  179. }
  180. fmt.Printf("%s", string(x))
  181. } else if csConfig.Cscli.Output == "raw" {
  182. csvwriter := csv.NewWriter(os.Stdout)
  183. if showHeader {
  184. header := []string{"name", "status", "version", "description"}
  185. if showType {
  186. header = append(header, "type")
  187. }
  188. err := csvwriter.Write(header)
  189. if err != nil {
  190. log.Fatalf("failed to write header: %s", err)
  191. }
  192. }
  193. for _, itemType := range itemTypes {
  194. var statuses []cwhub.ItemHubStatus
  195. var ok bool
  196. if statuses, ok = hubStatusByItemType[itemType]; !ok {
  197. log.Errorf("unknown item type: %s", itemType)
  198. continue
  199. }
  200. for _, status := range statuses {
  201. if status.LocalVersion == "" {
  202. status.LocalVersion = "n/a"
  203. }
  204. row := []string{
  205. status.Name,
  206. status.Status,
  207. status.LocalVersion,
  208. status.Description,
  209. }
  210. if showType {
  211. row = append(row, itemType)
  212. }
  213. err := csvwriter.Write(row)
  214. if err != nil {
  215. log.Fatalf("failed to write raw output : %s", err)
  216. }
  217. }
  218. }
  219. csvwriter.Flush()
  220. }
  221. }
  222. func InspectItem(name string, objecitemType string) {
  223. hubItem := cwhub.GetItem(objecitemType, name)
  224. if hubItem == nil {
  225. log.Fatalf("unable to retrieve item.")
  226. }
  227. var b []byte
  228. var err error
  229. switch csConfig.Cscli.Output {
  230. case "human", "raw":
  231. b, err = yaml.Marshal(*hubItem)
  232. if err != nil {
  233. log.Fatalf("unable to marshal item : %s", err)
  234. }
  235. case "json":
  236. b, err = json.MarshalIndent(*hubItem, "", " ")
  237. if err != nil {
  238. log.Fatalf("unable to marshal item : %s", err)
  239. }
  240. }
  241. fmt.Printf("%s", string(b))
  242. if csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
  243. return
  244. }
  245. if csConfig.Prometheus.Enabled {
  246. if csConfig.Prometheus.ListenAddr == "" || csConfig.Prometheus.ListenPort == 0 {
  247. log.Warningf("No prometheus address or port specified in '%s', can't show metrics", *csConfig.FilePath)
  248. return
  249. }
  250. if prometheusURL == "" {
  251. log.Debugf("No prometheus URL provided using: %s:%d", csConfig.Prometheus.ListenAddr, csConfig.Prometheus.ListenPort)
  252. prometheusURL = fmt.Sprintf("http://%s:%d/metrics", csConfig.Prometheus.ListenAddr, csConfig.Prometheus.ListenPort)
  253. }
  254. fmt.Printf("\nCurrent metrics : \n\n")
  255. ShowMetrics(hubItem)
  256. }
  257. }
  258. func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
  259. /*if a range is provided, change the scope*/
  260. if *ipRange != "" {
  261. _, _, err := net.ParseCIDR(*ipRange)
  262. if err != nil {
  263. return fmt.Errorf("%s isn't a valid range", *ipRange)
  264. }
  265. }
  266. if *ip != "" {
  267. ipRepr := net.ParseIP(*ip)
  268. if ipRepr == nil {
  269. return fmt.Errorf("%s isn't a valid ip", *ip)
  270. }
  271. }
  272. //avoid confusion on scope (ip vs Ip and range vs Range)
  273. switch strings.ToLower(*scope) {
  274. case "ip":
  275. *scope = types.Ip
  276. case "range":
  277. *scope = types.Range
  278. case "country":
  279. *scope = types.Country
  280. case "as":
  281. *scope = types.AS
  282. }
  283. return nil
  284. }
  285. func ShowMetrics(hubItem *cwhub.Item) {
  286. switch hubItem.Type {
  287. case cwhub.PARSERS:
  288. metrics := GetParserMetric(prometheusURL, hubItem.Name)
  289. ShowParserMetric(hubItem.Name, metrics)
  290. case cwhub.SCENARIOS:
  291. metrics := GetScenarioMetric(prometheusURL, hubItem.Name)
  292. ShowScenarioMetric(hubItem.Name, metrics)
  293. case cwhub.COLLECTIONS:
  294. for _, item := range hubItem.Parsers {
  295. metrics := GetParserMetric(prometheusURL, item)
  296. ShowParserMetric(item, metrics)
  297. }
  298. for _, item := range hubItem.Scenarios {
  299. metrics := GetScenarioMetric(prometheusURL, item)
  300. ShowScenarioMetric(item, metrics)
  301. }
  302. for _, item := range hubItem.Collections {
  303. hubItem = cwhub.GetItem(cwhub.COLLECTIONS, item)
  304. if hubItem == nil {
  305. log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
  306. }
  307. ShowMetrics(hubItem)
  308. }
  309. default:
  310. log.Errorf("item of type '%s' is unknown", hubItem.Type)
  311. }
  312. }
  313. /*This is a complete rip from prom2json*/
  314. func GetParserMetric(url string, itemName string) map[string]map[string]int {
  315. stats := make(map[string]map[string]int)
  316. result := GetPrometheusMetric(url)
  317. for idx, fam := range result {
  318. if !strings.HasPrefix(fam.Name, "cs_") {
  319. continue
  320. }
  321. log.Tracef("round %d", idx)
  322. for _, m := range fam.Metrics {
  323. metric, ok := m.(prom2json.Metric)
  324. if !ok {
  325. log.Debugf("failed to convert metric to prom2json.Metric")
  326. continue
  327. }
  328. name, ok := metric.Labels["name"]
  329. if !ok {
  330. log.Debugf("no name in Metric %v", metric.Labels)
  331. }
  332. if name != itemName {
  333. continue
  334. }
  335. source, ok := metric.Labels["source"]
  336. if !ok {
  337. log.Debugf("no source in Metric %v", metric.Labels)
  338. } else {
  339. if srctype, ok := metric.Labels["type"]; ok {
  340. source = srctype + ":" + source
  341. }
  342. }
  343. value := m.(prom2json.Metric).Value
  344. fval, err := strconv.ParseFloat(value, 32)
  345. if err != nil {
  346. log.Errorf("Unexpected int value %s : %s", value, err)
  347. continue
  348. }
  349. ival := int(fval)
  350. switch fam.Name {
  351. case "cs_reader_hits_total":
  352. if _, ok := stats[source]; !ok {
  353. stats[source] = make(map[string]int)
  354. stats[source]["parsed"] = 0
  355. stats[source]["reads"] = 0
  356. stats[source]["unparsed"] = 0
  357. stats[source]["hits"] = 0
  358. }
  359. stats[source]["reads"] += ival
  360. case "cs_parser_hits_ok_total":
  361. if _, ok := stats[source]; !ok {
  362. stats[source] = make(map[string]int)
  363. }
  364. stats[source]["parsed"] += ival
  365. case "cs_parser_hits_ko_total":
  366. if _, ok := stats[source]; !ok {
  367. stats[source] = make(map[string]int)
  368. }
  369. stats[source]["unparsed"] += ival
  370. case "cs_node_hits_total":
  371. if _, ok := stats[source]; !ok {
  372. stats[source] = make(map[string]int)
  373. }
  374. stats[source]["hits"] += ival
  375. case "cs_node_hits_ok_total":
  376. if _, ok := stats[source]; !ok {
  377. stats[source] = make(map[string]int)
  378. }
  379. stats[source]["parsed"] += ival
  380. case "cs_node_hits_ko_total":
  381. if _, ok := stats[source]; !ok {
  382. stats[source] = make(map[string]int)
  383. }
  384. stats[source]["unparsed"] += ival
  385. default:
  386. continue
  387. }
  388. }
  389. }
  390. return stats
  391. }
  392. func GetScenarioMetric(url string, itemName string) map[string]int {
  393. stats := make(map[string]int)
  394. stats["instanciation"] = 0
  395. stats["curr_count"] = 0
  396. stats["overflow"] = 0
  397. stats["pour"] = 0
  398. stats["underflow"] = 0
  399. result := GetPrometheusMetric(url)
  400. for idx, fam := range result {
  401. if !strings.HasPrefix(fam.Name, "cs_") {
  402. continue
  403. }
  404. log.Tracef("round %d", idx)
  405. for _, m := range fam.Metrics {
  406. metric, ok := m.(prom2json.Metric)
  407. if !ok {
  408. log.Debugf("failed to convert metric to prom2json.Metric")
  409. continue
  410. }
  411. name, ok := metric.Labels["name"]
  412. if !ok {
  413. log.Debugf("no name in Metric %v", metric.Labels)
  414. }
  415. if name != itemName {
  416. continue
  417. }
  418. value := m.(prom2json.Metric).Value
  419. fval, err := strconv.ParseFloat(value, 32)
  420. if err != nil {
  421. log.Errorf("Unexpected int value %s : %s", value, err)
  422. continue
  423. }
  424. ival := int(fval)
  425. switch fam.Name {
  426. case "cs_bucket_created_total":
  427. stats["instanciation"] += ival
  428. case "cs_buckets":
  429. stats["curr_count"] += ival
  430. case "cs_bucket_overflowed_total":
  431. stats["overflow"] += ival
  432. case "cs_bucket_poured_total":
  433. stats["pour"] += ival
  434. case "cs_bucket_underflowed_total":
  435. stats["underflow"] += ival
  436. default:
  437. continue
  438. }
  439. }
  440. }
  441. return stats
  442. }
  443. //it's a rip of the cli version, but in silent-mode
  444. func silenceInstallItem(name string, obtype string) (string, error) {
  445. var item = cwhub.GetItem(obtype, name)
  446. if item == nil {
  447. return "", fmt.Errorf("error retrieving item")
  448. }
  449. it := *item
  450. if downloadOnly && it.Downloaded && it.UpToDate {
  451. return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
  452. }
  453. it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction, false)
  454. if err != nil {
  455. return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
  456. }
  457. if err := cwhub.AddItem(obtype, it); err != nil {
  458. return "", err
  459. }
  460. if downloadOnly {
  461. return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
  462. }
  463. it, err = cwhub.EnableItem(csConfig.Hub, it)
  464. if err != nil {
  465. return "", fmt.Errorf("error while enabling %s : %v", it.Name, err)
  466. }
  467. if err := cwhub.AddItem(obtype, it); err != nil {
  468. return "", err
  469. }
  470. return fmt.Sprintf("Enabled %s", it.Name), nil
  471. }
  472. func GetPrometheusMetric(url string) []*prom2json.Family {
  473. mfChan := make(chan *dto.MetricFamily, 1024)
  474. // Start with the DefaultTransport for sane defaults.
  475. transport := http.DefaultTransport.(*http.Transport).Clone()
  476. // Conservatively disable HTTP keep-alives as this program will only
  477. // ever need a single HTTP request.
  478. transport.DisableKeepAlives = true
  479. // Timeout early if the server doesn't even return the headers.
  480. transport.ResponseHeaderTimeout = time.Minute
  481. go func() {
  482. defer types.CatchPanic("crowdsec/GetPrometheusMetric")
  483. err := prom2json.FetchMetricFamilies(url, mfChan, transport)
  484. if err != nil {
  485. log.Fatalf("failed to fetch prometheus metrics : %v", err)
  486. }
  487. }()
  488. result := []*prom2json.Family{}
  489. for mf := range mfChan {
  490. result = append(result, prom2json.NewFamily(mf))
  491. }
  492. log.Debugf("Finished reading prometheus output, %d entries", len(result))
  493. return result
  494. }
  495. func ShowScenarioMetric(itemName string, metrics map[string]int) {
  496. if metrics["instanciation"] == 0 {
  497. return
  498. }
  499. table := tablewriter.NewWriter(os.Stdout)
  500. table.SetHeader([]string{"Current Count", "Overflows", "Instanciated", "Poured", "Expired"})
  501. table.Append([]string{fmt.Sprintf("%d", metrics["curr_count"]), fmt.Sprintf("%d", metrics["overflow"]), fmt.Sprintf("%d", metrics["instanciation"]), fmt.Sprintf("%d", metrics["pour"]), fmt.Sprintf("%d", metrics["underflow"])})
  502. fmt.Printf(" - (Scenario) %s: \n", itemName)
  503. table.Render()
  504. fmt.Println()
  505. }
  506. func ShowParserMetric(itemName string, metrics map[string]map[string]int) {
  507. skip := true
  508. table := tablewriter.NewWriter(os.Stdout)
  509. table.SetHeader([]string{"Parsers", "Hits", "Parsed", "Unparsed"})
  510. for source, stats := range metrics {
  511. if stats["hits"] > 0 {
  512. table.Append([]string{source, fmt.Sprintf("%d", stats["hits"]), fmt.Sprintf("%d", stats["parsed"]), fmt.Sprintf("%d", stats["unparsed"])})
  513. skip = false
  514. }
  515. }
  516. if !skip {
  517. fmt.Printf(" - (Parser) %s: \n", itemName)
  518. table.Render()
  519. fmt.Println()
  520. }
  521. }
  522. func RestoreHub(dirPath string) error {
  523. var err error
  524. if err := csConfig.LoadHub(); err != nil {
  525. return err
  526. }
  527. if err := cwhub.SetHubBranch(); err != nil {
  528. return fmt.Errorf("error while setting hub branch: %s", err)
  529. }
  530. for _, itype := range cwhub.ItemTypes {
  531. itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
  532. if _, err = os.Stat(itemDirectory); err != nil {
  533. log.Infof("no %s in backup", itype)
  534. continue
  535. }
  536. /*restore the upstream items*/
  537. upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
  538. file, err := ioutil.ReadFile(upstreamListFN)
  539. if err != nil {
  540. return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
  541. }
  542. var upstreamList []string
  543. err = json.Unmarshal(file, &upstreamList)
  544. if err != nil {
  545. return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
  546. }
  547. for _, toinstall := range upstreamList {
  548. label, err := silenceInstallItem(toinstall, itype)
  549. if err != nil {
  550. log.Errorf("Error while installing %s : %s", toinstall, err)
  551. } else if label != "" {
  552. log.Infof("Installed %s : %s", toinstall, label)
  553. } else {
  554. log.Printf("Installed %s : ok", toinstall)
  555. }
  556. }
  557. /*restore the local and tainted items*/
  558. files, err := ioutil.ReadDir(itemDirectory)
  559. if err != nil {
  560. return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
  561. }
  562. for _, file := range files {
  563. //this was the upstream data
  564. if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
  565. continue
  566. }
  567. if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
  568. //we expect a stage here
  569. if !file.IsDir() {
  570. continue
  571. }
  572. stage := file.Name()
  573. stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
  574. log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
  575. if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
  576. return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
  577. }
  578. /*find items*/
  579. ifiles, err := ioutil.ReadDir(itemDirectory + "/" + stage + "/")
  580. if err != nil {
  581. return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
  582. }
  583. //finally copy item
  584. for _, tfile := range ifiles {
  585. log.Infof("Going to restore local/tainted [%s]", tfile.Name())
  586. sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
  587. destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
  588. if err = types.CopyFile(sourceFile, destinationFile); err != nil {
  589. return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
  590. }
  591. log.Infof("restored %s to %s", sourceFile, destinationFile)
  592. }
  593. } else {
  594. log.Infof("Going to restore local/tainted [%s]", file.Name())
  595. sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
  596. destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
  597. if err = types.CopyFile(sourceFile, destinationFile); err != nil {
  598. return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
  599. }
  600. log.Infof("restored %s to %s", sourceFile, destinationFile)
  601. }
  602. }
  603. }
  604. return nil
  605. }
  606. func BackupHub(dirPath string) error {
  607. var err error
  608. var itemDirectory string
  609. var upstreamParsers []string
  610. for _, itemType := range cwhub.ItemTypes {
  611. clog := log.WithFields(log.Fields{
  612. "type": itemType,
  613. })
  614. itemMap := cwhub.GetItemMap(itemType)
  615. if itemMap == nil {
  616. clog.Infof("No %s to backup.", itemType)
  617. continue
  618. }
  619. itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
  620. if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
  621. return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
  622. }
  623. upstreamParsers = []string{}
  624. for k, v := range itemMap {
  625. clog = clog.WithFields(log.Fields{
  626. "file": v.Name,
  627. })
  628. if !v.Installed { //only backup installed ones
  629. clog.Debugf("[%s] : not installed", k)
  630. continue
  631. }
  632. //for the local/tainted ones, we backup the full file
  633. if v.Tainted || v.Local || !v.UpToDate {
  634. //we need to backup stages for parsers
  635. if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
  636. fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
  637. if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
  638. return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
  639. }
  640. }
  641. clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
  642. tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
  643. if err = types.CopyFile(v.LocalPath, tfile); err != nil {
  644. return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
  645. }
  646. clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
  647. continue
  648. }
  649. clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
  650. clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
  651. upstreamParsers = append(upstreamParsers, v.Name)
  652. }
  653. //write the upstream items
  654. upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
  655. upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
  656. if err != nil {
  657. return fmt.Errorf("failed marshaling upstream parsers : %s", err)
  658. }
  659. err = ioutil.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
  660. if err != nil {
  661. return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
  662. }
  663. clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
  664. }
  665. return nil
  666. }
  667. type unit struct {
  668. value int64
  669. symbol string
  670. }
  671. var ranges = []unit{
  672. {
  673. value: 1e18,
  674. symbol: "E",
  675. },
  676. {
  677. value: 1e15,
  678. symbol: "P",
  679. },
  680. {
  681. value: 1e12,
  682. symbol: "T",
  683. },
  684. {
  685. value: 1e6,
  686. symbol: "M",
  687. },
  688. {
  689. value: 1e3,
  690. symbol: "k",
  691. },
  692. {
  693. value: 1,
  694. symbol: "",
  695. },
  696. }
  697. func formatNumber(num int) string {
  698. goodUnit := unit{}
  699. for _, u := range ranges {
  700. if int64(num) >= u.value {
  701. goodUnit = u
  702. break
  703. }
  704. }
  705. if goodUnit.value == 1 {
  706. return fmt.Sprintf("%d%s", num, goodUnit.symbol)
  707. }
  708. res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
  709. return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
  710. }