utils.go 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688
  1. package main
  2. import (
  3. "encoding/json"
  4. "fmt"
  5. "io/ioutil"
  6. "net"
  7. "net/http"
  8. "os"
  9. "strconv"
  10. "strings"
  11. "time"
  12. "github.com/crowdsecurity/crowdsec/pkg/cwhub"
  13. "github.com/crowdsecurity/crowdsec/pkg/cwversion"
  14. "github.com/crowdsecurity/crowdsec/pkg/types"
  15. "github.com/enescakir/emoji"
  16. "github.com/olekukonko/tablewriter"
  17. dto "github.com/prometheus/client_model/go"
  18. "github.com/prometheus/prom2json"
  19. log "github.com/sirupsen/logrus"
  20. "golang.org/x/mod/semver"
  21. "gopkg.in/yaml.v2"
  22. )
  23. func inSlice(s string, slice []string) bool {
  24. for _, str := range slice {
  25. if s == str {
  26. return true
  27. }
  28. }
  29. return false
  30. }
  31. func indexOf(s string, slice []string) int {
  32. for i, elem := range slice {
  33. if s == elem {
  34. return i
  35. }
  36. }
  37. return -1
  38. }
  39. func manageCliDecisionAlerts(ip *string, ipRange *string, scope *string, value *string) error {
  40. /*if a range is provided, change the scope*/
  41. if *ipRange != "" {
  42. _, _, err := net.ParseCIDR(*ipRange)
  43. if err != nil {
  44. return fmt.Errorf("%s isn't a valid range", *ipRange)
  45. }
  46. }
  47. if *ip != "" {
  48. ipRepr := net.ParseIP(*ip)
  49. if ipRepr == nil {
  50. return fmt.Errorf("%s isn't a valid ip", *ip)
  51. }
  52. }
  53. //avoid confusion on scope (ip vs Ip and range vs Range)
  54. switch strings.ToLower(*scope) {
  55. case "ip":
  56. *scope = types.Ip
  57. case "range":
  58. *scope = types.Range
  59. case "country":
  60. *scope = types.Country
  61. case "as":
  62. *scope = types.AS
  63. }
  64. return nil
  65. }
  66. func setHubBranch() error {
  67. /*
  68. if no branch has been specified in flags for the hub, then use the one corresponding to crowdsec version
  69. */
  70. if cwhub.HubBranch == "" {
  71. latest, err := cwversion.Latest()
  72. if err != nil {
  73. cwhub.HubBranch = "master"
  74. return err
  75. }
  76. csVersion := cwversion.VersionStrip()
  77. if csVersion == latest {
  78. cwhub.HubBranch = "master"
  79. } else if semver.Compare(csVersion, latest) == 1 { // if current version is greater than the latest we are in pre-release
  80. log.Debugf("Your current crowdsec version seems to be a pre-release (%s)", csVersion)
  81. cwhub.HubBranch = "master"
  82. } else if csVersion == "" {
  83. log.Warningf("Crowdsec version is '', using master branch for the hub")
  84. } else {
  85. log.Warnf("Crowdsec is not the latest version. Current version is '%s' and the latest stable version is '%s'. Please update it!", csVersion, latest)
  86. log.Warnf("As a result, you will not be able to use parsers/scenarios/collections added to Crowdsec Hub after CrowdSec %s", latest)
  87. cwhub.HubBranch = csVersion
  88. }
  89. log.Debugf("Using branch '%s' for the hub", cwhub.HubBranch)
  90. }
  91. return nil
  92. }
  93. func ListItem(itemType string, args []string) {
  94. var hubStatus []map[string]string
  95. if len(args) == 1 {
  96. hubStatus = cwhub.HubStatus(itemType, args[0], all)
  97. } else {
  98. hubStatus = cwhub.HubStatus(itemType, "", all)
  99. }
  100. if csConfig.Cscli.Output == "human" {
  101. table := tablewriter.NewWriter(os.Stdout)
  102. table.SetCenterSeparator("")
  103. table.SetColumnSeparator("")
  104. table.SetHeaderAlignment(tablewriter.ALIGN_LEFT)
  105. table.SetAlignment(tablewriter.ALIGN_LEFT)
  106. table.SetHeader([]string{"Name", fmt.Sprintf("%v Status", emoji.Package), "Version", "Local Path"})
  107. for _, v := range hubStatus {
  108. table.Append([]string{v["name"], v["utf8_status"], v["local_version"], v["local_path"]})
  109. }
  110. table.Render()
  111. } else if csConfig.Cscli.Output == "json" {
  112. x, err := json.MarshalIndent(hubStatus, "", " ")
  113. if err != nil {
  114. log.Fatalf("failed to unmarshal")
  115. }
  116. fmt.Printf("%s", string(x))
  117. } else if csConfig.Cscli.Output == "raw" {
  118. for _, v := range hubStatus {
  119. fmt.Printf("%s %s\n", v["name"], v["description"])
  120. }
  121. }
  122. }
  123. func InstallItem(name string, obtype string, force bool) {
  124. it := cwhub.GetItem(obtype, name)
  125. if it == nil {
  126. log.Fatalf("unable to retrieve item : %s", name)
  127. }
  128. item := *it
  129. if downloadOnly && item.Downloaded && item.UpToDate {
  130. log.Warningf("%s is already downloaded and up-to-date", item.Name)
  131. if !force {
  132. return
  133. }
  134. }
  135. item, err := cwhub.DownloadLatest(csConfig.Hub, item, force)
  136. if err != nil {
  137. log.Fatalf("error while downloading %s : %v", item.Name, err)
  138. }
  139. cwhub.AddItem(obtype, item)
  140. if downloadOnly {
  141. log.Infof("Downloaded %s to %s", item.Name, csConfig.Hub.HubDir+"/"+item.RemotePath)
  142. return
  143. }
  144. item, err = cwhub.EnableItem(csConfig.Hub, item)
  145. if err != nil {
  146. log.Fatalf("error while enabled %s : %v.", item.Name, err)
  147. }
  148. cwhub.AddItem(obtype, item)
  149. log.Infof("Enabled %s", item.Name)
  150. return
  151. }
  152. func RemoveMany(itemType string, name string) {
  153. var err error
  154. var disabled int
  155. if name != "" {
  156. it := cwhub.GetItem(itemType, name)
  157. if it == nil {
  158. log.Fatalf("unable to retrieve: %s", name)
  159. }
  160. item := *it
  161. item, err = cwhub.DisableItem(csConfig.Hub, item, purge, forceAction)
  162. if err != nil {
  163. log.Fatalf("unable to disable %s : %v", item.Name, err)
  164. }
  165. cwhub.AddItem(itemType, item)
  166. return
  167. } else if name == "" && all {
  168. for _, v := range cwhub.GetItemMap(itemType) {
  169. v, err = cwhub.DisableItem(csConfig.Hub, v, purge, forceAction)
  170. if err != nil {
  171. log.Fatalf("unable to disable %s : %v", v.Name, err)
  172. }
  173. cwhub.AddItem(itemType, v)
  174. disabled++
  175. }
  176. }
  177. if name != "" && !all {
  178. log.Errorf("%s not found", name)
  179. return
  180. }
  181. log.Infof("Disabled %d items", disabled)
  182. }
  183. func UpgradeConfig(itemType string, name string, force bool) {
  184. var err error
  185. var updated int
  186. var found bool
  187. for _, v := range cwhub.GetItemMap(itemType) {
  188. if name != "" && name != v.Name {
  189. continue
  190. }
  191. if !v.Installed {
  192. log.Tracef("skip %s, not installed", v.Name)
  193. if !force {
  194. continue
  195. }
  196. }
  197. if !v.Downloaded {
  198. log.Warningf("%s : not downloaded, please install.", v.Name)
  199. if !force {
  200. continue
  201. }
  202. }
  203. found = true
  204. if v.UpToDate {
  205. log.Infof("%s : up-to-date", v.Name)
  206. if !force {
  207. continue
  208. }
  209. }
  210. v, err = cwhub.DownloadLatest(csConfig.Hub, v, force)
  211. if err != nil {
  212. log.Fatalf("%s : download failed : %v", v.Name, err)
  213. }
  214. if !v.UpToDate {
  215. if v.Tainted {
  216. log.Infof("%v %s is tainted, --force to overwrite", emoji.Warning, v.Name)
  217. } else if v.Local {
  218. log.Infof("%v %s is local", emoji.Prohibited, v.Name)
  219. }
  220. } else {
  221. log.Infof("%v %s : updated", emoji.Package, v.Name)
  222. updated++
  223. }
  224. cwhub.AddItem(itemType, v)
  225. }
  226. if !found && name == "" {
  227. log.Infof("No %s installed, nothing to upgrade", itemType)
  228. } else if !found {
  229. log.Errorf("Item '%s' not found in hub", name)
  230. } else if updated == 0 && found {
  231. if name == "" {
  232. log.Infof("All %s are already up-to-date", itemType)
  233. } else {
  234. log.Infof("Item '%s' is up-to-date", name)
  235. }
  236. } else if updated != 0 {
  237. log.Infof("Upgraded %d items", updated)
  238. }
  239. }
  240. func InspectItem(name string, objecitemType string) {
  241. hubItem := cwhub.GetItem(objecitemType, name)
  242. if hubItem == nil {
  243. log.Fatalf("unable to retrieve item.")
  244. }
  245. buff, err := yaml.Marshal(*hubItem)
  246. if err != nil {
  247. log.Fatalf("unable to marshal item : %s", err)
  248. }
  249. fmt.Printf("%s", string(buff))
  250. if csConfig.Prometheus.Enabled {
  251. if csConfig.Prometheus.ListenAddr == "" || csConfig.Prometheus.ListenPort == 0 {
  252. log.Warningf("No prometheus address or port specified in '%s', can't show metrics", *csConfig.FilePath)
  253. return
  254. }
  255. if prometheusURL == "" {
  256. log.Debugf("No prometheus URL provided using: %s:%d", csConfig.Prometheus.ListenAddr, csConfig.Prometheus.ListenPort)
  257. prometheusURL = fmt.Sprintf("http://%s:%d/metrics", csConfig.Prometheus.ListenAddr, csConfig.Prometheus.ListenPort)
  258. }
  259. fmt.Printf("\nCurrent metrics : \n\n")
  260. ShowMetrics(hubItem)
  261. }
  262. }
  263. func ShowMetrics(hubItem *cwhub.Item) {
  264. switch hubItem.Type {
  265. case cwhub.PARSERS:
  266. metrics := GetParserMetric(prometheusURL, hubItem.Name)
  267. ShowParserMetric(hubItem.Name, metrics)
  268. case cwhub.SCENARIOS:
  269. metrics := GetScenarioMetric(prometheusURL, hubItem.Name)
  270. ShowScenarioMetric(hubItem.Name, metrics)
  271. case cwhub.COLLECTIONS:
  272. for _, item := range hubItem.Parsers {
  273. metrics := GetParserMetric(prometheusURL, item)
  274. ShowParserMetric(item, metrics)
  275. }
  276. for _, item := range hubItem.Scenarios {
  277. metrics := GetScenarioMetric(prometheusURL, item)
  278. ShowScenarioMetric(item, metrics)
  279. }
  280. for _, item := range hubItem.Collections {
  281. hubItem := cwhub.GetItem(cwhub.COLLECTIONS, item)
  282. if hubItem == nil {
  283. log.Fatalf("unable to retrieve item '%s' from collection '%s'", item, hubItem.Name)
  284. }
  285. ShowMetrics(hubItem)
  286. }
  287. default:
  288. log.Errorf("item of type '%s' is unknown", hubItem.Type)
  289. }
  290. }
  291. /*This is a complete rip from prom2json*/
  292. func GetParserMetric(url string, itemName string) map[string]map[string]int {
  293. stats := make(map[string]map[string]int)
  294. result := GetPrometheusMetric(url)
  295. for idx, fam := range result {
  296. if !strings.HasPrefix(fam.Name, "cs_") {
  297. continue
  298. }
  299. log.Tracef("round %d", idx)
  300. for _, m := range fam.Metrics {
  301. metric := m.(prom2json.Metric)
  302. name, ok := metric.Labels["name"]
  303. if !ok {
  304. log.Debugf("no name in Metric %v", metric.Labels)
  305. }
  306. if name != itemName {
  307. continue
  308. }
  309. source, ok := metric.Labels["source"]
  310. if !ok {
  311. log.Debugf("no source in Metric %v", metric.Labels)
  312. } else {
  313. if srctype, ok := metric.Labels["type"]; ok {
  314. source = srctype + ":" + source
  315. }
  316. }
  317. value := m.(prom2json.Metric).Value
  318. fval, err := strconv.ParseFloat(value, 32)
  319. if err != nil {
  320. log.Errorf("Unexpected int value %s : %s", value, err)
  321. continue
  322. }
  323. ival := int(fval)
  324. switch fam.Name {
  325. case "cs_reader_hits_total":
  326. if _, ok := stats[source]; !ok {
  327. stats[source] = make(map[string]int)
  328. stats[source]["parsed"] = 0
  329. stats[source]["reads"] = 0
  330. stats[source]["unparsed"] = 0
  331. stats[source]["hits"] = 0
  332. }
  333. stats[source]["reads"] += ival
  334. case "cs_parser_hits_ok_total":
  335. if _, ok := stats[source]; !ok {
  336. stats[source] = make(map[string]int)
  337. }
  338. stats[source]["parsed"] += ival
  339. case "cs_parser_hits_ko_total":
  340. if _, ok := stats[source]; !ok {
  341. stats[source] = make(map[string]int)
  342. }
  343. stats[source]["unparsed"] += ival
  344. case "cs_node_hits_total":
  345. if _, ok := stats[source]; !ok {
  346. stats[source] = make(map[string]int)
  347. }
  348. stats[source]["hits"] += ival
  349. case "cs_node_hits_ok_total":
  350. if _, ok := stats[source]; !ok {
  351. stats[source] = make(map[string]int)
  352. }
  353. stats[source]["parsed"] += ival
  354. case "cs_node_hits_ko_total":
  355. if _, ok := stats[source]; !ok {
  356. stats[source] = make(map[string]int)
  357. }
  358. stats[source]["unparsed"] += ival
  359. default:
  360. continue
  361. }
  362. }
  363. }
  364. return stats
  365. }
  366. func GetScenarioMetric(url string, itemName string) map[string]int {
  367. stats := make(map[string]int)
  368. stats["instanciation"] = 0
  369. stats["curr_count"] = 0
  370. stats["overflow"] = 0
  371. stats["pour"] = 0
  372. stats["underflow"] = 0
  373. result := GetPrometheusMetric(url)
  374. for idx, fam := range result {
  375. if !strings.HasPrefix(fam.Name, "cs_") {
  376. continue
  377. }
  378. log.Tracef("round %d", idx)
  379. for _, m := range fam.Metrics {
  380. metric := m.(prom2json.Metric)
  381. name, ok := metric.Labels["name"]
  382. if !ok {
  383. log.Debugf("no name in Metric %v", metric.Labels)
  384. }
  385. if name != itemName {
  386. continue
  387. }
  388. value := m.(prom2json.Metric).Value
  389. fval, err := strconv.ParseFloat(value, 32)
  390. if err != nil {
  391. log.Errorf("Unexpected int value %s : %s", value, err)
  392. continue
  393. }
  394. ival := int(fval)
  395. switch fam.Name {
  396. case "cs_bucket_created_total":
  397. stats["instanciation"] += ival
  398. case "cs_buckets":
  399. stats["curr_count"] += ival
  400. case "cs_bucket_overflowed_total":
  401. stats["overflow"] += ival
  402. case "cs_bucket_poured_total":
  403. stats["pour"] += ival
  404. case "cs_bucket_underflowed_total":
  405. stats["underflow"] += ival
  406. default:
  407. continue
  408. }
  409. }
  410. }
  411. return stats
  412. }
  413. func GetPrometheusMetric(url string) []*prom2json.Family {
  414. mfChan := make(chan *dto.MetricFamily, 1024)
  415. // Start with the DefaultTransport for sane defaults.
  416. transport := http.DefaultTransport.(*http.Transport).Clone()
  417. // Conservatively disable HTTP keep-alives as this program will only
  418. // ever need a single HTTP request.
  419. transport.DisableKeepAlives = true
  420. // Timeout early if the server doesn't even return the headers.
  421. transport.ResponseHeaderTimeout = time.Minute
  422. go func() {
  423. defer types.CatchPanic("crowdsec/GetPrometheusMetric")
  424. err := prom2json.FetchMetricFamilies(url, mfChan, transport)
  425. if err != nil {
  426. log.Fatalf("failed to fetch prometheus metrics : %v", err)
  427. }
  428. }()
  429. result := []*prom2json.Family{}
  430. for mf := range mfChan {
  431. result = append(result, prom2json.NewFamily(mf))
  432. }
  433. log.Debugf("Finished reading prometheus output, %d entries", len(result))
  434. return result
  435. }
  436. func ShowScenarioMetric(itemName string, metrics map[string]int) {
  437. if metrics["instanciation"] == 0 {
  438. return
  439. }
  440. table := tablewriter.NewWriter(os.Stdout)
  441. table.SetHeader([]string{"Current Count", "Overflows", "Instanciated", "Poured", "Expired"})
  442. table.Append([]string{fmt.Sprintf("%d", metrics["curr_count"]), fmt.Sprintf("%d", metrics["overflow"]), fmt.Sprintf("%d", metrics["instanciation"]), fmt.Sprintf("%d", metrics["pour"]), fmt.Sprintf("%d", metrics["underflow"])})
  443. fmt.Printf(" - (Scenario) %s: \n", itemName)
  444. table.Render()
  445. fmt.Println()
  446. }
  447. func ShowParserMetric(itemName string, metrics map[string]map[string]int) {
  448. skip := true
  449. table := tablewriter.NewWriter(os.Stdout)
  450. table.SetHeader([]string{"Parsers", "Hits", "Parsed", "Unparsed"})
  451. for source, stats := range metrics {
  452. if stats["hits"] > 0 {
  453. table.Append([]string{source, fmt.Sprintf("%d", stats["hits"]), fmt.Sprintf("%d", stats["parsed"]), fmt.Sprintf("%d", stats["unparsed"])})
  454. skip = false
  455. }
  456. }
  457. if !skip {
  458. fmt.Printf(" - (Parser) %s: \n", itemName)
  459. table.Render()
  460. fmt.Println()
  461. }
  462. }
  463. //it's a rip of the cli version, but in silent-mode
  464. func silenceInstallItem(name string, obtype string) (string, error) {
  465. var item *cwhub.Item
  466. item = cwhub.GetItem(obtype, name)
  467. if item == nil {
  468. return "", fmt.Errorf("error retrieving item")
  469. }
  470. it := *item
  471. if downloadOnly && it.Downloaded && it.UpToDate {
  472. return fmt.Sprintf("%s is already downloaded and up-to-date", it.Name), nil
  473. }
  474. it, err := cwhub.DownloadLatest(csConfig.Hub, it, forceAction)
  475. if err != nil {
  476. return "", fmt.Errorf("error while downloading %s : %v", it.Name, err)
  477. }
  478. if err := cwhub.AddItem(obtype, it); err != nil {
  479. return "", err
  480. }
  481. if downloadOnly {
  482. return fmt.Sprintf("Downloaded %s to %s", it.Name, csConfig.Cscli.HubDir+"/"+it.RemotePath), nil
  483. }
  484. it, err = cwhub.EnableItem(csConfig.Hub, it)
  485. if err != nil {
  486. return "", fmt.Errorf("error while enabled %s : %v", it.Name, err)
  487. }
  488. if err := cwhub.AddItem(obtype, it); err != nil {
  489. return "", err
  490. }
  491. return fmt.Sprintf("Enabled %s", it.Name), nil
  492. }
  493. func RestoreHub(dirPath string) error {
  494. var err error
  495. if err := csConfig.LoadHub(); err != nil {
  496. return err
  497. }
  498. if err := setHubBranch(); err != nil {
  499. return fmt.Errorf("error while setting hub branch: %s", err)
  500. }
  501. for _, itype := range cwhub.ItemTypes {
  502. itemDirectory := fmt.Sprintf("%s/%s/", dirPath, itype)
  503. if _, err = os.Stat(itemDirectory); err != nil {
  504. log.Infof("no %s in backup", itype)
  505. continue
  506. }
  507. /*restore the upstream items*/
  508. upstreamListFN := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itype)
  509. file, err := ioutil.ReadFile(upstreamListFN)
  510. if err != nil {
  511. return fmt.Errorf("error while opening %s : %s", upstreamListFN, err)
  512. }
  513. var upstreamList []string
  514. err = json.Unmarshal([]byte(file), &upstreamList)
  515. if err != nil {
  516. return fmt.Errorf("error unmarshaling %s : %s", upstreamListFN, err)
  517. }
  518. for _, toinstall := range upstreamList {
  519. label, err := silenceInstallItem(toinstall, itype)
  520. if err != nil {
  521. log.Errorf("Error while installing %s : %s", toinstall, err)
  522. } else if label != "" {
  523. log.Infof("Installed %s : %s", toinstall, label)
  524. } else {
  525. log.Printf("Installed %s : ok", toinstall)
  526. }
  527. }
  528. /*restore the local and tainted items*/
  529. files, err := ioutil.ReadDir(itemDirectory)
  530. if err != nil {
  531. return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory, err)
  532. }
  533. for _, file := range files {
  534. //this was the upstream data
  535. if file.Name() == fmt.Sprintf("upstream-%s.json", itype) {
  536. continue
  537. }
  538. if itype == cwhub.PARSERS || itype == cwhub.PARSERS_OVFLW {
  539. //we expect a stage here
  540. if !file.IsDir() {
  541. continue
  542. }
  543. stage := file.Name()
  544. stagedir := fmt.Sprintf("%s/%s/%s/", csConfig.ConfigPaths.ConfigDir, itype, stage)
  545. log.Debugf("Found stage %s in %s, target directory : %s", stage, itype, stagedir)
  546. if err = os.MkdirAll(stagedir, os.ModePerm); err != nil {
  547. return fmt.Errorf("error while creating stage directory %s : %s", stagedir, err)
  548. }
  549. /*find items*/
  550. ifiles, err := ioutil.ReadDir(itemDirectory + "/" + stage + "/")
  551. if err != nil {
  552. return fmt.Errorf("failed enumerating files of %s : %s", itemDirectory+"/"+stage, err)
  553. }
  554. //finaly copy item
  555. for _, tfile := range ifiles {
  556. log.Infof("Going to restore local/tainted [%s]", tfile.Name())
  557. sourceFile := fmt.Sprintf("%s/%s/%s", itemDirectory, stage, tfile.Name())
  558. destinationFile := fmt.Sprintf("%s%s", stagedir, tfile.Name())
  559. if err = types.CopyFile(sourceFile, destinationFile); err != nil {
  560. return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
  561. }
  562. log.Infof("restored %s to %s", sourceFile, destinationFile)
  563. }
  564. } else {
  565. log.Infof("Going to restore local/tainted [%s]", file.Name())
  566. sourceFile := fmt.Sprintf("%s/%s", itemDirectory, file.Name())
  567. destinationFile := fmt.Sprintf("%s/%s/%s", csConfig.ConfigPaths.ConfigDir, itype, file.Name())
  568. if err = types.CopyFile(sourceFile, destinationFile); err != nil {
  569. return fmt.Errorf("failed copy %s %s to %s : %s", itype, sourceFile, destinationFile, err)
  570. }
  571. log.Infof("restored %s to %s", sourceFile, destinationFile)
  572. }
  573. }
  574. }
  575. return nil
  576. }
  577. func BackupHub(dirPath string) error {
  578. var err error
  579. var itemDirectory string
  580. var upstreamParsers []string
  581. for _, itemType := range cwhub.ItemTypes {
  582. clog := log.WithFields(log.Fields{
  583. "type": itemType,
  584. })
  585. itemMap := cwhub.GetItemMap(itemType)
  586. if itemMap != nil {
  587. itemDirectory = fmt.Sprintf("%s/%s/", dirPath, itemType)
  588. if err := os.MkdirAll(itemDirectory, os.ModePerm); err != nil {
  589. return fmt.Errorf("error while creating %s : %s", itemDirectory, err)
  590. }
  591. upstreamParsers = []string{}
  592. for k, v := range itemMap {
  593. clog = clog.WithFields(log.Fields{
  594. "file": v.Name,
  595. })
  596. if !v.Installed { //only backup installed ones
  597. clog.Debugf("[%s] : not installed", k)
  598. continue
  599. }
  600. //for the local/tainted ones, we backup the full file
  601. if v.Tainted || v.Local || !v.UpToDate {
  602. //we need to backup stages for parsers
  603. if itemType == cwhub.PARSERS || itemType == cwhub.PARSERS_OVFLW {
  604. fstagedir := fmt.Sprintf("%s%s", itemDirectory, v.Stage)
  605. if err := os.MkdirAll(fstagedir, os.ModePerm); err != nil {
  606. return fmt.Errorf("error while creating stage dir %s : %s", fstagedir, err)
  607. }
  608. }
  609. clog.Debugf("[%s] : backuping file (tainted:%t local:%t up-to-date:%t)", k, v.Tainted, v.Local, v.UpToDate)
  610. tfile := fmt.Sprintf("%s%s/%s", itemDirectory, v.Stage, v.FileName)
  611. if err = types.CopyFile(v.LocalPath, tfile); err != nil {
  612. return fmt.Errorf("failed copy %s %s to %s : %s", itemType, v.LocalPath, tfile, err)
  613. }
  614. clog.Infof("local/tainted saved %s to %s", v.LocalPath, tfile)
  615. continue
  616. }
  617. clog.Debugf("[%s] : from hub, just backup name (up-to-date:%t)", k, v.UpToDate)
  618. clog.Infof("saving, version:%s, up-to-date:%t", v.Version, v.UpToDate)
  619. upstreamParsers = append(upstreamParsers, v.Name)
  620. }
  621. //write the upstream items
  622. upstreamParsersFname := fmt.Sprintf("%s/upstream-%s.json", itemDirectory, itemType)
  623. upstreamParsersContent, err := json.MarshalIndent(upstreamParsers, "", " ")
  624. if err != nil {
  625. return fmt.Errorf("failed marshaling upstream parsers : %s", err)
  626. }
  627. err = ioutil.WriteFile(upstreamParsersFname, upstreamParsersContent, 0644)
  628. if err != nil {
  629. return fmt.Errorf("unable to write to %s %s : %s", itemType, upstreamParsersFname, err)
  630. }
  631. clog.Infof("Wrote %d entries for %s to %s", len(upstreamParsers), itemType, upstreamParsersFname)
  632. } else {
  633. clog.Infof("No %s to backup.", itemType)
  634. }
  635. }
  636. return nil
  637. }