utils.go 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762
  1. package main
  2. import (
  3. "encoding/csv"
  4. "encoding/json"
  5. "fmt"
  6. "io/ioutil"
  7. "math"
  8. "net"
  9. "net/http"
  10. "os"
  11. "strconv"
  12. "strings"
  13. "time"
  14. "github.com/crowdsecurity/crowdsec/pkg/cwhub"
  15. "github.com/crowdsecurity/crowdsec/pkg/cwversion"
  16. "github.com/crowdsecurity/crowdsec/pkg/types"
  17. "github.com/enescakir/emoji"
  18. "github.com/olekukonko/tablewriter"
  19. dto "github.com/prometheus/client_model/go"
  20. "github.com/prometheus/prom2json"
  21. log "github.com/sirupsen/logrus"
  22. "github.com/spf13/cobra"
  23. "golang.org/x/mod/semver"
  24. "gopkg.in/yaml.v2"
  25. )
  26. func printHelp(cmd *cobra.Command) {
  27. err := cmd.Help()
  28. if err != nil {
  29. log.Fatalf("unable to print help(): %s", err)
  30. }
  31. }
  32. func inSlice(s string, slice []string) bool {
  33. for _, str := range slice {
  34. if s == str {
  35. return true
  36. }
  37. }
  38. return false
  39. }
  40. func indexOf(s string, slice []string) int {
  41. for i, elem := range slice {
  42. if s == elem {
  43. return i
  44. }
  45. }
  46. return -1
  47. }
  48. func LoadHub() error {
  49. if err := csConfig.LoadHub(); err != nil {
  50. log.Fatalf(err.Error())
  51. }
  52. if csConfig.Hub == nil {
  53. return fmt.Errorf("unable to load hub")
  54. }
  55. if err := setHubBranch(); err != nil {
  56. log.Warningf("unable to set hub branch (%s), default to master", err)
  57. }
  58. if err := cwhub.GetHubIdx(csConfig.Hub); err != nil {
  59. return fmt.Errorf("Failed to get Hub index : '%v'. Run 'sudo cscli hub update' to get the hub index", err)
  60. }
  61. return nil
  62. }
  63. func compAllItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
  64. if err := LoadHub(); err != nil {
  65. return nil, cobra.ShellCompDirectiveDefault
  66. }
  67. comp := make([]string, 0)
  68. hubItems := cwhub.GetHubStatusForItemType(itemType, "", true)
  69. for _, item := range hubItems {
  70. if toComplete == "" {
  71. comp = append(comp, item.Name)
  72. } else {
  73. if strings.Contains(item.Name, toComplete) {
  74. comp = append(comp, item.Name)
  75. }
  76. }
  77. }
  78. cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
  79. return comp, cobra.ShellCompDirectiveNoFileComp
  80. }
  81. func compInstalledItems(itemType string, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
  82. if err := LoadHub(); err != nil {
  83. return nil, cobra.ShellCompDirectiveDefault
  84. }
  85. var items []string
  86. var err error
  87. switch itemType {
  88. case cwhub.PARSERS:
  89. items, err = cwhub.GetInstalledParsersAsString()
  90. case cwhub.SCENARIOS:
  91. items, err = cwhub.GetInstalledScenariosAsString()
  92. case cwhub.PARSERS_OVFLW:
  93. items, err = cwhub.GetInstalledPostOverflowsAsString()
  94. case cwhub.COLLECTIONS:
  95. items, err = cwhub.GetInstalledCollectionsAsString()
  96. default:
  97. return nil, cobra.ShellCompDirectiveDefault
  98. }
  99. if err != nil {
  100. cobra.CompDebugln(fmt.Sprintf("list installed %s err: %s", itemType, err), true)
  101. return nil, cobra.ShellCompDirectiveDefault
  102. }
  103. comp := make([]string, 0)
  104. if toComplete != "" {
  105. for _, item := range items {
  106. if strings.Contains(item, toComplete) {
  107. comp = append(comp, item)
  108. }
  109. }
  110. } else {
  111. comp = items
  112. }
  113. cobra.CompDebugln(fmt.Sprintf("%s: %+v", itemType, comp), true)
  114. return comp, cobra.ShellCompDirectiveNoFileComp
  115. }
  116. func ListItems(itemTypes []string, args []string, showType bool, showHeader bool, all bool) {
  117. var hubStatusByItemType = make(map[string][]cwhub.ItemHubStatus)
  118. for _, itemType := range itemTypes {
  119. itemName := ""
  120. if len(args) == 1 {
  121. itemName = args[0]
  122. }
  123. hubStatusByItemType[itemType] = cwhub.GetHubStatusForItemType(itemType, itemName, all)
  124. }
  125. if csConfig.Cscli.Output == "human" {
  126. for _, itemType := range itemTypes {
  127. var statuses []cwhub.ItemHubStatus
  128. var ok bool
  129. if statuses, ok = hubStatusByItemType[itemType]; !ok {
  130. log.Errorf("unknown item type: %s", itemType)
  131. continue
  132. }
  133. fmt.Println(strings.ToUpper(itemType))
  134. table := tablewriter.NewWriter(os.Stdout)
  135. table.SetCenterSeparator("")
  136. table.SetColumnSeparator("")
  137. table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
  138. table.SetAlignment(tablewriter.ALIGN_LEFT)
  139. table.SetHeader([]string{"Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path"})
  140. for _, status := range statuses {
  141. table.Append([]string{status.Name, status.UTF8_Status, status.LocalVersion, status.LocalPath})
  142. }
  143. table.Render()
  144. }
  145. } else if csConfig.Cscli.Output == "json" {
  146. x, err := json.MarshalIndent(hubStatusByItemType, "", " ")
  147. if err != nil {
  148. log.Fatalf("failed to unmarshal")
  149. }
  150. fmt.Printf("%s", string(x))
  151. } else if csConfig.Cscli.Output == "raw" {
  152. csvwriter := csv.NewWriter(os.Stdout)
  153. if showHeader {
  154. header := []string{"name", "status", "version", "description"}
  155. if showType {
  156. header = append(header, "type")
  157. }
  158. err := csvwriter.Write(header)
  159. if err != nil {
  160. log.Fatalf("failed to write header: %s", err)
  161. }
  162. }
  163. for _, itemType := range itemTypes {
  164. var statuses []cwhub.ItemHubStatus
  165. var ok bool
  166. if statuses, ok = hubStatusByItemType[itemType]; !ok {
  167. log.Errorf("unknown item type: %s", itemType)
  168. continue
  169. }
  170. for _, status := range statuses {
  171. if status.LocalVersion == "" {
  172. status.LocalVersion = "n/a"
  173. }
  174. row := []string{
  175. status.Name,
  176. status.Status,
  177. status.LocalVersion,
  178. status.Description,
  179. }
  180. if showType {
  181. row = append(row, itemType)
  182. }
  183. err := csvwriter.Write(row)
  184. if err != nil {
  185. log.Fatalf("failed to write raw output : %s", err)
  186. }
  187. }
  188. }
  189. csvwriter.Flush()
  190. }
  191. }
  192. func InspectItem(name string, objecitemType string) {
  193. hubItem := cwhub.GetItem(objecitemType, name)
  194. if hubItem == nil {
  195. log.Fatalf("unable to retrieve item.")
  196. }
  197. var b []byte
  198. var err error
  199. switch csConfig.Cscli.Output {
  200. case "human", "raw":
  201. b, err = yaml.Marshal(*hubItem)
  202. if err != nil {
  203. log.Fatalf("unable to marshal item : %s", err)
  204. }
  205. case "json":
  206. b, err = json.MarshalIndent(*hubItem, "", " ")
  207. if err != nil {
  208. log.Fatalf("unable to marshal item : %s", err)
  209. }
  210. }
  211. fmt.Printf("%s", string(b))
  212. if csConfig.Cscli.Output == "json" || csConfig.Cscli.Output == "raw" {
  213. return
  214. }
  215. if csConfig.Prometheus.Enabled {
  216. if csConfig.Prometheus.ListenAddr == "" || csConfig.Prometheus.ListenPort == 0 {
  217. log.Warningf("No prometheus address or port specified in '%s', can't show metrics", *csConfig.FilePath)
  218. return
  219. }
  220. if prometheusURL == "" {
  221. log.Debugf("No prometheus URL provided using: %s:%d", csConfig.Prometheus.ListenAddr, csConfig.Prometheus.ListenPort)
  222. prometheusURL = fmt.Sprintf("http://%s:%d/metrics", csConfig.Prometheus.ListenAddr, csConfig.Prometheus.ListenPort)
  223. }
  224. fmt.Printf("\nCurrent metrics : \n\n")
  225. ShowMetrics(hubItem)
  226. }
  227. }
  228. func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
  229. /*if a range is provided, change the scope*/
  230. if *ipRange != "" {
  231. _, _, err := net.ParseCIDR(*ipRange)
  232. if err != nil {
  233. return fmt.Errorf("%s isn't a valid range", *ipRange)
  234. }
  235. }
  236. if *ip != "" {
  237. ipRepr := net.ParseIP(*ip)
  238. if ipRepr == nil {
  239. return fmt.Errorf("%s isn't a valid ip", *ip)
  240. }
  241. }
  242. //avoid confusion on scope (ip vs Ip and range vs Range)
  243. switch strings.ToLower(*scope) {
  244. case "ip":
  245. *scope = types.Ip
  246. case "range":
  247. *scope = types.Range
  248. case "country":
  249. *scope = types.Country
  250. case "as":
  251. *scope = types.AS
  252. }
  253. return nil
  254. }
  255. func setHubBranch() error {
  256. /*
  257. if no branch has been specified in flags for the hub, then use the one corresponding to crowdsec version
  258. */
  259. if cwhub.HubBranch == "" {
  260. latest, err := cwversion.Latest()
  261. if err != nil {
  262. cwhub.HubBranch = "master"
  263. return err
  264. }
  265. csVersion := cwversion.VersionStrip()
  266. if csVersion == latest {
  267. cwhub.HubBranch = "master"
  268. } else if semver.Compare(csVersion, latest) == 1 { // if current version is greater than the latest we are in pre-release
  269. log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion)
  270. cwhub.HubBranch = "master"
  271. } else if csVersion == "" {
  272. log.Warningf("Crowdsec version is '', using master branch for the hub")
  273. cwhub.HubBranch = "master"
  274. } else {
  275. log.Warnf("Crowdsec is not the latest version. Current version is '%s' and the latest stable version is '%s'. Please update it!", csVersion, latest)
  276. log.Warnf("As a result, you will not be able to use parsers/scenarios/collections added to Crowdsec Hub after CrowdSec %s", latest)
  277. cwhub.HubBranch = csVersion
  278. }
  279. log.Debugf("Using branch '%s' for the hub", cwhub.HubBranch)
  280. }
  281. return nil
  282. }
  283. func ShowMetrics(hubItem *cwhub.Item) {
  284. switch hubItem.Type {
  285. case cwhub.PARSERS:
  286. metrics := GetParserMetric(prometheusURL, hubItem.Name)
  287. ShowParserMetric(hubItem.Name, metrics)
  288. case cwhub.SCENARIOS:
  289. metrics := GetScenarioMetric(prometheusURL, hubItem.Name)
  290. ShowScenarioMetric(hubItem.Name, metrics)
  291. case cwhub.COLLECTIONS:
  292. for _, item := range hubItem.Parsers {
  293. metrics := GetParserMetric(prometheusURL, item)
  294. ShowParserMetric(item, metrics)
  295. }
  296. for _, item := range hubItem.Scenarios {
  297. metrics := GetScenarioMetric(prometheusURL, item)
  298. ShowScenarioMetric(item, metrics)
  299. }
  300. for _, item := range hubItem.Collections {
  301. hubItem := cwhub.GetItem(cwhub.COLLECTIONS, item)
  302. if hubItem == nil {
  303. log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
  304. }
  305. ShowMetrics(hubItem)
  306. }
  307. default:
  308. log.Errorf("item of type '%s' is unknown", hubItem.Type)
  309. }
  310. }
  311. /*This is a complete rip from prom2json*/
  312. func GetParserMetric(url string, itemName string) map[string]map[string]int {
  313. stats := make(map[string]map[string]int)
  314. result := GetPrometheusMetric(url)
  315. for idx, fam := range result {
  316. if !strings.HasPrefix(fam.Name, "cs_") {
  317. continue
  318. }
  319. log.Tracef("round %d", idx)
  320. for _, m := range fam.Metrics {
  321. metric := m.(prom2json.Metric)
  322. name, ok := metric.Labels["name"]
  323. if !ok {
  324. log.Debugf("no name in Metric %v", metric.Labels)
  325. }
  326. if name != itemName {
  327. continue
  328. }
  329. source, ok := metric.Labels["source"]
  330. if !ok {
  331. log.Debugf("no source in Metric %v", metric.Labels)
  332. } else {
  333. if srctype, ok := metric.Labels["type"]; ok {
  334. source = srctype + ":" + source
  335. }
  336. }
  337. value := m.(prom2json.Metric).Value
  338. fval, err := strconv.ParseFloat(value, 32)
  339. if err != nil {
  340. log.Errorf("Unexpected int value %s : %s", value, err)
  341. continue
  342. }
  343. ival := int(fval)
  344. switch fam.Name {
  345. case "cs_reader_hits_total":
  346. if _, ok := stats[source]; !ok {
  347. stats[source] = make(map[string]int)
  348. stats[source]["parsed"] = 0
  349. stats[source]["reads"] = 0
  350. stats[source]["unparsed"] = 0
  351. stats[source]["hits"] = 0
  352. }
  353. stats[source]["reads"] += ival
  354. case "cs_parser_hits_ok_total":
  355. if _, ok := stats[source]; !ok {
  356. stats[source] = make(map[string]int)
  357. }
  358. stats[source]["parsed"] += ival
  359. case "cs_parser_hits_ko_total":
  360. if _, ok := stats[source]; !ok {
  361. stats[source] = make(map[string]int)
  362. }
  363. stats[source]["unparsed"] += ival
  364. case "cs_node_hits_total":
  365. if _, ok := stats[source]; !ok {
  366. stats[source] = make(map[string]int)
  367. }
  368. stats[source]["hits"] += ival
  369. case "cs_node_hits_ok_total":
  370. if _, ok := stats[source]; !ok {
  371. stats[source] = make(map[string]int)
  372. }
  373. stats[source]["parsed"] += ival
  374. case "cs_node_hits_ko_total":
  375. if _, ok := stats[source]; !ok {
  376. stats[source] = make(map[string]int)
  377. }
  378. stats[source]["unparsed"] += ival
  379. default:
  380. continue
  381. }
  382. }
  383. }
  384. return stats
  385. }
  386. func GetScenarioMetric(url string, itemName string) map[string]int {
  387. stats := make(map[string]int)
  388. stats["instanciation"] = 0
  389. stats["curr_count"] = 0
  390. stats["overflow"] = 0
  391. stats["pour"] = 0
  392. stats["underflow"] = 0
  393. result := GetPrometheusMetric(url)
  394. for idx, fam := range result {
  395. if !strings.HasPrefix(fam.Name, "cs_") {
  396. continue
  397. }
  398. log.Tracef("round %d", idx)
  399. for _, m := range fam.Metrics {
  400. metric := m.(prom2json.Metric)
  401. name, ok := metric.Labels["name"]
  402. if !ok {
  403. log.Debugf("no name in Metric %v", metric.Labels)
  404. }
  405. if name != itemName {
  406. continue
  407. }
  408. value := m.(prom2json.Metric).Value
  409. fval, err := strconv.ParseFloat(value, 32)
  410. if err != nil {
  411. log.Errorf("Unexpected int value %s : %s", value, err)
  412. continue
  413. }
  414. ival := int(fval)
  415. switch fam.Name {
  416. case "cs_bucket_created_total":
  417. stats["instanciation"] += ival
  418. case "cs_buckets":
  419. stats["curr_count"] += ival
  420. case "cs_bucket_overflowed_total":
  421. stats["overflow"] += ival
  422. case "cs_bucket_poured_total":
  423. stats["pour"] += ival
  424. case "cs_bucket_underflowed_total":
  425. stats["underflow"] += ival
  426. default:
  427. continue
  428. }
  429. }
  430. }
  431. return stats
  432. }
  433. //it's a rip of the cli version, but in silent-mode
  434. func silenceInstallItem(name string, obtype string) (string, error) {
  435. var item = cwhub.GetItem(obtype, name)
  436. if item == nil {
  437. return "", fmt.Errorf("error retrieving item")
  438. }
  439. it := *item
  440. if downloadOnly && it.Downloaded && it.UpToDate {
  441. return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
  442. }
  443. it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction, false)
  444. if err != nil {
  445. return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
  446. }
  447. if err := cwhub.AddItem(obtype, it); err != nil {
  448. return "", err
  449. }
  450. if downloadOnly {
  451. return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
  452. }
  453. it, err = cwhub.EnableItem(csConfig.Hub, it)
  454. if err != nil {
  455. return "", fmt.Errorf("error while enabling %s : %v", it.Name, err)
  456. }
  457. if err := cwhub.AddItem(obtype, it); err != nil {
  458. return "", err
  459. }
  460. return fmt.Sprintf("Enabled %s", it.Name), nil
  461. }
  462. func GetPrometheusMetric(url string) []*prom2json.Family {
  463. mfChan := make(chan *dto.MetricFamily, 1024)
  464. // Start with the DefaultTransport for sane defaults.
  465. transport := http.DefaultTransport.(*http.Transport).Clone()
  466. // Conservatively disable HTTP keep-alives as this program will only
  467. // ever need a single HTTP request.
  468. transport.DisableKeepAlives = true
  469. // Timeout early if the server doesn't even return the headers.
  470. transport.ResponseHeaderTimeout = time.Minute
  471. go func() {
  472. defer types.CatchPanic("crowdsec/GetPrometheusMetric")
  473. err := prom2json.FetchMetricFamilies(url, mfChan, transport)
  474. if err != nil {
  475. log.Fatalf("failed to fetch prometheus metrics : %v", err)
  476. }
  477. }()
  478. result := []*prom2json.Family{}
  479. for mf := range mfChan {
  480. result = append(result, prom2json.NewFamily(mf))
  481. }
  482. log.Debugf("Finished reading prometheus output, %d entries", len(result))
  483. return result
  484. }
  485. func ShowScenarioMetric(itemName string, metrics map[string]int) {
  486. if metrics["instanciation"] == 0 {
  487. return
  488. }
  489. table := tablewriter.NewWriter(os.Stdout)
  490. table.SetHeader([]string{"Current Count", "Overflows", "Instanciated", "Poured", "Expired"})
  491. table.Append([]string{fmt.Sprintf("%d", metrics["curr_count"]), fmt.Sprintf("%d", metrics["overflow"]), fmt.Sprintf("%d", metrics["instanciation"]), fmt.Sprintf("%d", metrics["pour"]), fmt.Sprintf("%d", metrics["underflow"])})
  492. fmt.Printf(" - (Scenario) %s: \n", itemName)
  493. table.Render()
  494. fmt.Println()
  495. }
  496. func ShowParserMetric(itemName string, metrics map[string]map[string]int) {
  497. skip := true
  498. table := tablewriter.NewWriter(os.Stdout)
  499. table.SetHeader([]string{"Parsers", "Hits", "Parsed", "Unparsed"})
  500. for source, stats := range metrics {
  501. if stats["hits"] > 0 {
  502. table.Append([]string{source, fmt.Sprintf("%d", stats["hits"]), fmt.Sprintf("%d", stats["parsed"]), fmt.Sprintf("%d", stats["unparsed"])})
  503. skip = false
  504. }
  505. }
  506. if !skip {
  507. fmt.Printf(" - (Parser) %s: \n", itemName)
  508. table.Render()
  509. fmt.Println()
  510. }
  511. }
  512. func RestoreHub(dirPath string) error {
  513. var err error
  514. if err := csConfig.LoadHub(); err != nil {
  515. return err
  516. }
  517. if err := setHubBranch(); err != nil {
  518. return fmt.Errorf("error while setting hub branch: %s", err)
  519. }
  520. for _, itype := range cwhub.ItemTypes {
  521. itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
  522. if _, err = os.Stat(itemDirectory); err != nil {
  523. log.Infof("no %s in backup", itype)
  524. continue
  525. }
  526. /*restore the upstream items*/
  527. upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
  528. file, err := ioutil.ReadFile(upstreamListFN)
  529. if err != nil {
  530. return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
  531. }
  532. var upstreamList []string
  533. err = json.Unmarshal([]byte(file), &upstreamList)
  534. if err != nil {
  535. return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
  536. }
  537. for _, toinstall := range upstreamList {
  538. label, err := silenceInstallItem(toinstall, itype)
  539. if err != nil {
  540. log.Errorf("Error while installing %s : %s", toinstall, err)
  541. } else if label != "" {
  542. log.Infof("Installed %s : %s", toinstall, label)
  543. } else {
  544. log.Printf("Installed %s : ok", toinstall)
  545. }
  546. }
  547. /*restore the local and tainted items*/
  548. files, err := ioutil.ReadDir(itemDirectory)
  549. if err != nil {
  550. return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
  551. }
  552. for _, file := range files {
  553. //this was the upstream data
  554. if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
  555. continue
  556. }
  557. if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
  558. //we expect a stage here
  559. if !file.IsDir() {
  560. continue
  561. }
  562. stage := file.Name()
  563. stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
  564. log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
  565. if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
  566. return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
  567. }
  568. /*find items*/
  569. ifiles, err := ioutil.ReadDir(itemDirectory + "/" + stage + "/")
  570. if err != nil {
  571. return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
  572. }
  573. //finally copy item
  574. for _, tfile := range ifiles {
  575. log.Infof("Going to restore local/tainted [%s]", tfile.Name())
  576. sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
  577. destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
  578. if err = types.CopyFile(sourceFile, destinationFile); err != nil {
  579. return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
  580. }
  581. log.Infof("restored %s to %s", sourceFile, destinationFile)
  582. }
  583. } else {
  584. log.Infof("Going to restore local/tainted [%s]", file.Name())
  585. sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
  586. destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
  587. if err = types.CopyFile(sourceFile, destinationFile); err != nil {
  588. return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
  589. }
  590. log.Infof("restored %s to %s", sourceFile, destinationFile)
  591. }
  592. }
  593. }
  594. return nil
  595. }
  596. func BackupHub(dirPath string) error {
  597. var err error
  598. var itemDirectory string
  599. var upstreamParsers []string
  600. for _, itemType := range cwhub.ItemTypes {
  601. clog := log.WithFields(log.Fields{
  602. "type": itemType,
  603. })
  604. itemMap := cwhub.GetItemMap(itemType)
  605. if itemMap == nil {
  606. clog.Infof("No %s to backup.", itemType)
  607. continue
  608. }
  609. itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
  610. if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
  611. return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
  612. }
  613. upstreamParsers = []string{}
  614. for k, v := range itemMap {
  615. clog = clog.WithFields(log.Fields{
  616. "file": v.Name,
  617. })
  618. if !v.Installed { //only backup installed ones
  619. clog.Debugf("[%s] : not installed", k)
  620. continue
  621. }
  622. //for the local/tainted ones, we backup the full file
  623. if v.Tainted || v.Local || !v.UpToDate {
  624. //we need to backup stages for parsers
  625. if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
  626. fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
  627. if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
  628. return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
  629. }
  630. }
  631. clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
  632. tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
  633. if err = types.CopyFile(v.LocalPath, tfile); err != nil {
  634. return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
  635. }
  636. clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
  637. continue
  638. }
  639. clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
  640. clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
  641. upstreamParsers = append(upstreamParsers, v.Name)
  642. }
  643. //write the upstream items
  644. upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
  645. upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
  646. if err != nil {
  647. return fmt.Errorf("failed marshaling upstream parsers : %s", err)
  648. }
  649. err = ioutil.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
  650. if err != nil {
  651. return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
  652. }
  653. clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
  654. }
  655. return nil
  656. }
  657. type unit struct {
  658. value int64
  659. symbol string
  660. }
  661. var ranges = []unit{
  662. {
  663. value: 1e18,
  664. symbol: "E",
  665. },
  666. {
  667. value: 1e15,
  668. symbol: "P",
  669. },
  670. {
  671. value: 1e12,
  672. symbol: "T",
  673. },
  674. {
  675. value: 1e6,
  676. symbol: "M",
  677. },
  678. {
  679. value: 1e3,
  680. symbol: "k",
  681. },
  682. {
  683. value: 1,
  684. symbol: "",
  685. },
  686. }
  687. func formatNumber(num int) string {
  688. goodUnit := unit{}
  689. for _, u := range ranges {
  690. if int64(num) >= u.value {
  691. goodUnit = u
  692. break
  693. }
  694. }
  695. if goodUnit.value == 1 {
  696. return fmt.Sprintf("%d%s", num, goodUnit.symbol)
  697. }
  698. res := math.Round(float64(num)/float64(goodUnit.value)*100) / 100
  699. return fmt.Sprintf("%.2f%s", res, goodUnit.symbol)
  700. }