utils.go 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648
  1. package utils
  2. import (
  3. "bytes"
  4. "crypto/sha256"
  5. "encoding/hex"
  6. "encoding/json"
  7. "errors"
  8. "fmt"
  9. "index/suffixarray"
  10. "io"
  11. "io/ioutil"
  12. "net/http"
  13. "os"
  14. "os/exec"
  15. "path/filepath"
  16. "runtime"
  17. "strconv"
  18. "strings"
  19. "sync"
  20. "time"
  21. )
  22. // Go is a basic promise implementation: it wraps calls a function in a goroutine,
  23. // and returns a channel which will later return the function's return value.
  24. func Go(f func() error) chan error {
  25. ch := make(chan error)
  26. go func() {
  27. ch <- f()
  28. }()
  29. return ch
  30. }
  31. // Request a given URL and return an io.Reader
  32. func Download(url string, stderr io.Writer) (*http.Response, error) {
  33. var resp *http.Response
  34. var err error
  35. if resp, err = http.Get(url); err != nil {
  36. return nil, err
  37. }
  38. if resp.StatusCode >= 400 {
  39. return nil, errors.New("Got HTTP status code >= 400: " + resp.Status)
  40. }
  41. return resp, nil
  42. }
  43. // Debug function, if the debug flag is set, then display. Do nothing otherwise
  44. // If Docker is in damon mode, also send the debug info on the socket
  45. func Debugf(format string, a ...interface{}) {
  46. if os.Getenv("DEBUG") != "" {
  47. // Retrieve the stack infos
  48. _, file, line, ok := runtime.Caller(1)
  49. if !ok {
  50. file = "<unknown>"
  51. line = -1
  52. } else {
  53. file = file[strings.LastIndex(file, "/")+1:]
  54. }
  55. fmt.Fprintf(os.Stderr, fmt.Sprintf("[debug] %s:%d %s\n", file, line, format), a...)
  56. }
  57. }
  58. // Reader with progress bar
  59. type progressReader struct {
  60. reader io.ReadCloser // Stream to read from
  61. output io.Writer // Where to send progress bar to
  62. readTotal int // Expected stream length (bytes)
  63. readProgress int // How much has been read so far (bytes)
  64. lastUpdate int // How many bytes read at least update
  65. template string // Template to print. Default "%v/%v (%v)"
  66. sf *StreamFormatter
  67. }
  68. func (r *progressReader) Read(p []byte) (n int, err error) {
  69. read, err := io.ReadCloser(r.reader).Read(p)
  70. r.readProgress += read
  71. updateEvery := 4096
  72. if r.readTotal > 0 {
  73. // Only update progress for every 1% read
  74. if increment := int(0.01 * float64(r.readTotal)); increment > updateEvery {
  75. updateEvery = increment
  76. }
  77. }
  78. if r.readProgress-r.lastUpdate > updateEvery || err != nil {
  79. if r.readTotal > 0 {
  80. fmt.Fprintf(r.output, r.template, HumanSize(r.readProgress), HumanSize(r.readTotal), fmt.Sprintf("%.0f%%", float64(r.readProgress)/float64(r.readTotal)*100))
  81. } else {
  82. fmt.Fprintf(r.output, r.template, r.readProgress, "?", "n/a")
  83. }
  84. r.lastUpdate = r.readProgress
  85. }
  86. // Send newline when complete
  87. if err != nil {
  88. r.output.Write(r.sf.FormatStatus(""))
  89. }
  90. return read, err
  91. }
  92. func (r *progressReader) Close() error {
  93. return io.ReadCloser(r.reader).Close()
  94. }
  95. func ProgressReader(r io.ReadCloser, size int, output io.Writer, template []byte, sf *StreamFormatter) *progressReader {
  96. tpl := string(template)
  97. if tpl == "" {
  98. tpl = string(sf.FormatProgress("", "%v/%v (%v)"))
  99. }
  100. return &progressReader{r, NewWriteFlusher(output), size, 0, 0, tpl, sf}
  101. }
  102. func HumanSize(origSize int) string {
  103. size := float64(origSize)
  104. for _, unit := range []string{"b", "Kb", "Mb", "Gb", "Tb"} {
  105. if int(size)/1024 == 0 {
  106. return fmt.Sprintf("%.03f%s", size, unit)
  107. } else {
  108. size = size / 1024
  109. }
  110. }
  111. return strconv.Itoa(origSize)
  112. }
  113. // HumanDuration returns a human-readable approximation of a duration
  114. // (eg. "About a minute", "4 hours ago", etc.)
  115. func HumanDuration(d time.Duration) string {
  116. if seconds := int(d.Seconds()); seconds < 1 {
  117. return "Less than a second"
  118. } else if seconds < 60 {
  119. return fmt.Sprintf("%d seconds", seconds)
  120. } else if minutes := int(d.Minutes()); minutes == 1 {
  121. return "About a minute"
  122. } else if minutes < 60 {
  123. return fmt.Sprintf("%d minutes", minutes)
  124. } else if hours := int(d.Hours()); hours == 1 {
  125. return "About an hour"
  126. } else if hours < 48 {
  127. return fmt.Sprintf("%d hours", hours)
  128. } else if hours < 24*7*2 {
  129. return fmt.Sprintf("%d days", hours/24)
  130. } else if hours < 24*30*3 {
  131. return fmt.Sprintf("%d weeks", hours/24/7)
  132. } else if hours < 24*365*2 {
  133. return fmt.Sprintf("%d months", hours/24/30)
  134. }
  135. return fmt.Sprintf("%d years", d.Hours()/24/365)
  136. }
  137. // HumanSize returns a human-readable approximation of a size
  138. // using SI standard (eg. "44kB", "17MB")
  139. func HumanSize(size int64) string {
  140. i := 0
  141. var sizef float64
  142. sizef = float64(size)
  143. units := []string{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}
  144. for sizef >= 1000.0 {
  145. sizef = sizef / 1000.0
  146. i++
  147. }
  148. return fmt.Sprintf("%.4g %s", sizef, units[i])
  149. }
  150. func Trunc(s string, maxlen int) string {
  151. if len(s) <= maxlen {
  152. return s
  153. }
  154. return s[:maxlen]
  155. }
  156. // Figure out the absolute path of our own binary
  157. func SelfPath() string {
  158. path, err := exec.LookPath(os.Args[0])
  159. if err != nil {
  160. panic(err)
  161. }
  162. path, err = filepath.Abs(path)
  163. if err != nil {
  164. panic(err)
  165. }
  166. return path
  167. }
  168. type NopWriter struct {
  169. }
  170. func (w *NopWriter) Write(buf []byte) (int, error) {
  171. return len(buf), nil
  172. }
  173. type nopWriteCloser struct {
  174. io.Writer
  175. }
  176. func (w *nopWriteCloser) Close() error { return nil }
  177. func NopWriteCloser(w io.Writer) io.WriteCloser {
  178. return &nopWriteCloser{w}
  179. }
  180. type bufReader struct {
  181. buf *bytes.Buffer
  182. reader io.Reader
  183. err error
  184. l sync.Mutex
  185. wait sync.Cond
  186. }
  187. func NewBufReader(r io.Reader) *bufReader {
  188. reader := &bufReader{
  189. buf: &bytes.Buffer{},
  190. reader: r,
  191. }
  192. reader.wait.L = &reader.l
  193. go reader.drain()
  194. return reader
  195. }
  196. func (r *bufReader) drain() {
  197. buf := make([]byte, 1024)
  198. for {
  199. n, err := r.reader.Read(buf)
  200. r.l.Lock()
  201. if err != nil {
  202. r.err = err
  203. } else {
  204. r.buf.Write(buf[0:n])
  205. }
  206. r.wait.Signal()
  207. r.l.Unlock()
  208. if err != nil {
  209. break
  210. }
  211. }
  212. }
  213. func (r *bufReader) Read(p []byte) (n int, err error) {
  214. r.l.Lock()
  215. defer r.l.Unlock()
  216. for {
  217. n, err = r.buf.Read(p)
  218. if n > 0 {
  219. return n, err
  220. }
  221. if r.err != nil {
  222. return 0, r.err
  223. }
  224. r.wait.Wait()
  225. }
  226. panic("unreachable")
  227. }
  228. func (r *bufReader) Close() error {
  229. closer, ok := r.reader.(io.ReadCloser)
  230. if !ok {
  231. return nil
  232. }
  233. return closer.Close()
  234. }
  235. type WriteBroadcaster struct {
  236. mu sync.Mutex
  237. writers map[io.WriteCloser]struct{}
  238. }
  239. func (w *WriteBroadcaster) AddWriter(writer io.WriteCloser) {
  240. w.mu.Lock()
  241. w.writers[writer] = struct{}{}
  242. w.mu.Unlock()
  243. }
  244. // FIXME: Is that function used?
  245. // FIXME: This relies on the concrete writer type used having equality operator
  246. func (w *WriteBroadcaster) RemoveWriter(writer io.WriteCloser) {
  247. w.mu.Lock()
  248. delete(w.writers, writer)
  249. w.mu.Unlock()
  250. }
  251. func (w *WriteBroadcaster) Write(p []byte) (n int, err error) {
  252. w.mu.Lock()
  253. defer w.mu.Unlock()
  254. for writer := range w.writers {
  255. if n, err := writer.Write(p); err != nil || n != len(p) {
  256. // On error, evict the writer
  257. delete(w.writers, writer)
  258. }
  259. }
  260. return len(p), nil
  261. }
  262. func (w *WriteBroadcaster) CloseWriters() error {
  263. w.mu.Lock()
  264. defer w.mu.Unlock()
  265. for writer := range w.writers {
  266. writer.Close()
  267. }
  268. w.writers = make(map[io.WriteCloser]struct{})
  269. return nil
  270. }
  271. func NewWriteBroadcaster() *WriteBroadcaster {
  272. return &WriteBroadcaster{writers: make(map[io.WriteCloser]struct{})}
  273. }
  274. func GetTotalUsedFds() int {
  275. if fds, err := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid())); err != nil {
  276. Debugf("Error opening /proc/%d/fd: %s", os.Getpid(), err)
  277. } else {
  278. return len(fds)
  279. }
  280. return -1
  281. }
  282. // TruncIndex allows the retrieval of string identifiers by any of their unique prefixes.
  283. // This is used to retrieve image and container IDs by more convenient shorthand prefixes.
  284. type TruncIndex struct {
  285. index *suffixarray.Index
  286. ids map[string]bool
  287. bytes []byte
  288. }
  289. func NewTruncIndex() *TruncIndex {
  290. return &TruncIndex{
  291. index: suffixarray.New([]byte{' '}),
  292. ids: make(map[string]bool),
  293. bytes: []byte{' '},
  294. }
  295. }
  296. func (idx *TruncIndex) Add(id string) error {
  297. if strings.Contains(id, " ") {
  298. return fmt.Errorf("Illegal character: ' '")
  299. }
  300. if _, exists := idx.ids[id]; exists {
  301. return fmt.Errorf("Id already exists: %s", id)
  302. }
  303. idx.ids[id] = true
  304. idx.bytes = append(idx.bytes, []byte(id+" ")...)
  305. idx.index = suffixarray.New(idx.bytes)
  306. return nil
  307. }
  308. func (idx *TruncIndex) Delete(id string) error {
  309. if _, exists := idx.ids[id]; !exists {
  310. return fmt.Errorf("No such id: %s", id)
  311. }
  312. before, after, err := idx.lookup(id)
  313. if err != nil {
  314. return err
  315. }
  316. delete(idx.ids, id)
  317. idx.bytes = append(idx.bytes[:before], idx.bytes[after:]...)
  318. idx.index = suffixarray.New(idx.bytes)
  319. return nil
  320. }
  321. func (idx *TruncIndex) lookup(s string) (int, int, error) {
  322. offsets := idx.index.Lookup([]byte(" "+s), -1)
  323. //log.Printf("lookup(%s): %v (index bytes: '%s')\n", s, offsets, idx.index.Bytes())
  324. if offsets == nil || len(offsets) == 0 || len(offsets) > 1 {
  325. return -1, -1, fmt.Errorf("No such id: %s", s)
  326. }
  327. offsetBefore := offsets[0] + 1
  328. offsetAfter := offsetBefore + strings.Index(string(idx.bytes[offsetBefore:]), " ")
  329. return offsetBefore, offsetAfter, nil
  330. }
  331. func (idx *TruncIndex) Get(s string) (string, error) {
  332. before, after, err := idx.lookup(s)
  333. //log.Printf("Get(%s) bytes=|%s| before=|%d| after=|%d|\n", s, idx.bytes, before, after)
  334. if err != nil {
  335. return "", err
  336. }
  337. return string(idx.bytes[before:after]), err
  338. }
  339. // TruncateID returns a shorthand version of a string identifier for convenience.
  340. // A collision with other shorthands is very unlikely, but possible.
  341. // In case of a collision a lookup with TruncIndex.Get() will fail, and the caller
  342. // will need to use a langer prefix, or the full-length Id.
  343. func TruncateID(id string) string {
  344. shortLen := 12
  345. if len(id) < shortLen {
  346. shortLen = len(id)
  347. }
  348. return id[:shortLen]
  349. }
  350. // Code c/c from io.Copy() modified to handle escape sequence
  351. func CopyEscapable(dst io.Writer, src io.ReadCloser) (written int64, err error) {
  352. buf := make([]byte, 32*1024)
  353. for {
  354. nr, er := src.Read(buf)
  355. if nr > 0 {
  356. // ---- Docker addition
  357. // char 16 is C-p
  358. if nr == 1 && buf[0] == 16 {
  359. nr, er = src.Read(buf)
  360. // char 17 is C-q
  361. if nr == 1 && buf[0] == 17 {
  362. if err := src.Close(); err != nil {
  363. return 0, err
  364. }
  365. return 0, io.EOF
  366. }
  367. }
  368. // ---- End of docker
  369. nw, ew := dst.Write(buf[0:nr])
  370. if nw > 0 {
  371. written += int64(nw)
  372. }
  373. if ew != nil {
  374. err = ew
  375. break
  376. }
  377. if nr != nw {
  378. err = io.ErrShortWrite
  379. break
  380. }
  381. }
  382. if er == io.EOF {
  383. break
  384. }
  385. if er != nil {
  386. err = er
  387. break
  388. }
  389. }
  390. return written, err
  391. }
  392. func HashData(src io.Reader) (string, error) {
  393. h := sha256.New()
  394. if _, err := io.Copy(h, src); err != nil {
  395. return "", err
  396. }
  397. return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil
  398. }
  399. type KernelVersionInfo struct {
  400. Kernel int
  401. Major int
  402. Minor int
  403. Flavor string
  404. }
  405. func (k *KernelVersionInfo) String() string {
  406. flavor := ""
  407. if len(k.Flavor) > 0 {
  408. flavor = fmt.Sprintf("-%s", k.Flavor)
  409. }
  410. return fmt.Sprintf("%d.%d.%d%s", k.Kernel, k.Major, k.Minor, flavor)
  411. }
  412. // Compare two KernelVersionInfo struct.
  413. // Returns -1 if a < b, = if a == b, 1 it a > b
  414. func CompareKernelVersion(a, b *KernelVersionInfo) int {
  415. if a.Kernel < b.Kernel {
  416. return -1
  417. } else if a.Kernel > b.Kernel {
  418. return 1
  419. }
  420. if a.Major < b.Major {
  421. return -1
  422. } else if a.Major > b.Major {
  423. return 1
  424. }
  425. if a.Minor < b.Minor {
  426. return -1
  427. } else if a.Minor > b.Minor {
  428. return 1
  429. }
  430. return 0
  431. }
  432. func FindCgroupMountpoint(cgroupType string) (string, error) {
  433. output, err := ioutil.ReadFile("/proc/mounts")
  434. if err != nil {
  435. return "", err
  436. }
  437. // /proc/mounts has 6 fields per line, one mount per line, e.g.
  438. // cgroup /sys/fs/cgroup/devices cgroup rw,relatime,devices 0 0
  439. for _, line := range strings.Split(string(output), "\n") {
  440. parts := strings.Split(line, " ")
  441. if len(parts) == 6 && parts[2] == "cgroup" {
  442. for _, opt := range strings.Split(parts[3], ",") {
  443. if opt == cgroupType {
  444. return parts[1], nil
  445. }
  446. }
  447. }
  448. }
  449. return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType)
  450. }
  451. func GetKernelVersion() (*KernelVersionInfo, error) {
  452. var (
  453. flavor string
  454. kernel, major, minor int
  455. err error
  456. )
  457. uts, err := uname()
  458. if err != nil {
  459. return nil, err
  460. }
  461. release := make([]byte, len(uts.Release))
  462. i := 0
  463. for _, c := range uts.Release {
  464. release[i] = byte(c)
  465. i++
  466. }
  467. // Remove the \x00 from the release for Atoi to parse correctly
  468. release = release[:bytes.IndexByte(release, 0)]
  469. tmp := strings.SplitN(string(release), "-", 2)
  470. tmp2 := strings.SplitN(tmp[0], ".", 3)
  471. if len(tmp2) > 0 {
  472. kernel, err = strconv.Atoi(tmp2[0])
  473. if err != nil {
  474. return nil, err
  475. }
  476. }
  477. if len(tmp2) > 1 {
  478. major, err = strconv.Atoi(tmp2[1])
  479. if err != nil {
  480. return nil, err
  481. }
  482. }
  483. if len(tmp2) > 2 {
  484. minor, err = strconv.Atoi(tmp2[2])
  485. if err != nil {
  486. return nil, err
  487. }
  488. }
  489. if len(tmp) == 2 {
  490. flavor = tmp[1]
  491. } else {
  492. flavor = ""
  493. }
  494. return &KernelVersionInfo{
  495. Kernel: kernel,
  496. Major: major,
  497. Minor: minor,
  498. Flavor: flavor,
  499. }, nil
  500. }
  501. func CopyDirectory(source, dest string) error {
  502. if output, err := exec.Command("cp", "-ra", source, dest).CombinedOutput(); err != nil {
  503. return fmt.Errorf("Error copy: %s (%s)", err, output)
  504. }
  505. return nil
  506. }
  507. type NopFlusher struct{}
  508. func (f *NopFlusher) Flush() {}
  509. type WriteFlusher struct {
  510. w io.Writer
  511. flusher http.Flusher
  512. }
  513. func (wf *WriteFlusher) Write(b []byte) (n int, err error) {
  514. n, err = wf.w.Write(b)
  515. wf.flusher.Flush()
  516. return n, err
  517. }
  518. func NewWriteFlusher(w io.Writer) *WriteFlusher {
  519. var flusher http.Flusher
  520. if f, ok := w.(http.Flusher); ok {
  521. flusher = f
  522. } else {
  523. flusher = &NopFlusher{}
  524. }
  525. return &WriteFlusher{w: w, flusher: flusher}
  526. }
  527. type JSONMessage struct {
  528. Status string `json:"status,omitempty"`
  529. Progress string `json:"progress,omitempty"`
  530. Error string `json:"error,omitempty"`
  531. }
  532. type StreamFormatter struct {
  533. json bool
  534. used bool
  535. }
  536. func NewStreamFormatter(json bool) *StreamFormatter {
  537. return &StreamFormatter{json, false}
  538. }
  539. func (sf *StreamFormatter) FormatStatus(format string, a ...interface{}) []byte {
  540. sf.used = true
  541. str := fmt.Sprintf(format, a...)
  542. if sf.json {
  543. b, err := json.Marshal(&JSONMessage{Status: str})
  544. if err != nil {
  545. return sf.FormatError(err)
  546. }
  547. return b
  548. }
  549. return []byte(str + "\r\n")
  550. }
  551. func (sf *StreamFormatter) FormatError(err error) []byte {
  552. sf.used = true
  553. if sf.json {
  554. if b, err := json.Marshal(&JSONMessage{Error: err.Error()}); err == nil {
  555. return b
  556. }
  557. return []byte("{\"error\":\"format error\"}")
  558. }
  559. return []byte("Error: " + err.Error() + "\r\n")
  560. }
  561. func (sf *StreamFormatter) FormatProgress(action, str string) []byte {
  562. sf.used = true
  563. if sf.json {
  564. b, err := json.Marshal(&JSONMessage{Status: action, Progress: str})
  565. if err != nil {
  566. return nil
  567. }
  568. return b
  569. }
  570. return []byte(action + " " + str + "\r")
  571. }
  572. func (sf *StreamFormatter) Used() bool {
  573. return sf.used
  574. }