utils.go 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622
  1. package utils
  2. import (
  3. "bytes"
  4. "crypto/sha256"
  5. "encoding/hex"
  6. "encoding/json"
  7. "errors"
  8. "fmt"
  9. "index/suffixarray"
  10. "io"
  11. "io/ioutil"
  12. "net/http"
  13. "os"
  14. "os/exec"
  15. "path/filepath"
  16. "runtime"
  17. "strconv"
  18. "strings"
  19. "sync"
  20. "time"
  21. )
  22. // Go is a basic promise implementation: it wraps calls a function in a goroutine,
  23. // and returns a channel which will later return the function's return value.
  24. func Go(f func() error) chan error {
  25. ch := make(chan error)
  26. go func() {
  27. ch <- f()
  28. }()
  29. return ch
  30. }
  31. // Request a given URL and return an io.Reader
  32. func Download(url string, stderr io.Writer) (*http.Response, error) {
  33. var resp *http.Response
  34. var err error
  35. if resp, err = http.Get(url); err != nil {
  36. return nil, err
  37. }
  38. if resp.StatusCode >= 400 {
  39. return nil, errors.New("Got HTTP status code >= 400: " + resp.Status)
  40. }
  41. return resp, nil
  42. }
  43. // Debug function, if the debug flag is set, then display. Do nothing otherwise
  44. // If Docker is in damon mode, also send the debug info on the socket
  45. func Debugf(format string, a ...interface{}) {
  46. if os.Getenv("DEBUG") != "" {
  47. // Retrieve the stack infos
  48. _, file, line, ok := runtime.Caller(1)
  49. if !ok {
  50. file = "<unknown>"
  51. line = -1
  52. } else {
  53. file = file[strings.LastIndex(file, "/")+1:]
  54. }
  55. fmt.Fprintf(os.Stderr, fmt.Sprintf("[debug] %s:%d %s\n", file, line, format), a...)
  56. }
  57. }
  58. // Reader with progress bar
  59. type progressReader struct {
  60. reader io.ReadCloser // Stream to read from
  61. output io.Writer // Where to send progress bar to
  62. readTotal int // Expected stream length (bytes)
  63. readProgress int // How much has been read so far (bytes)
  64. lastUpdate int // How many bytes read at least update
  65. template string // Template to print. Default "%v/%v (%v)"
  66. sf *StreamFormatter
  67. }
  68. func (r *progressReader) Read(p []byte) (n int, err error) {
  69. read, err := io.ReadCloser(r.reader).Read(p)
  70. r.readProgress += read
  71. updateEvery := 4096
  72. if r.readTotal > 0 {
  73. // Only update progress for every 1% read
  74. if increment := int(0.01 * float64(r.readTotal)); increment > updateEvery {
  75. updateEvery = increment
  76. }
  77. }
  78. if r.readProgress-r.lastUpdate > updateEvery || err != nil {
  79. if r.readTotal > 0 {
  80. fmt.Fprintf(r.output, r.template, r.readProgress, r.readTotal, fmt.Sprintf("%.0f%%", float64(r.readProgress)/float64(r.readTotal)*100))
  81. } else {
  82. fmt.Fprintf(r.output, r.template, r.readProgress, "?", "n/a")
  83. }
  84. r.lastUpdate = r.readProgress
  85. }
  86. // Send newline when complete
  87. if err != nil {
  88. r.output.Write(r.sf.FormatStatus(""))
  89. }
  90. return read, err
  91. }
  92. func (r *progressReader) Close() error {
  93. return io.ReadCloser(r.reader).Close()
  94. }
  95. func ProgressReader(r io.ReadCloser, size int, output io.Writer, template []byte, sf *StreamFormatter) *progressReader {
  96. tpl := string(template)
  97. if tpl == "" {
  98. tpl = string(sf.FormatProgress("", "%v/%v (%v)"))
  99. }
  100. return &progressReader{r, NewWriteFlusher(output), size, 0, 0, tpl, sf}
  101. }
  102. // HumanDuration returns a human-readable approximation of a duration
  103. // (eg. "About a minute", "4 hours ago", etc.)
  104. func HumanDuration(d time.Duration) string {
  105. if seconds := int(d.Seconds()); seconds < 1 {
  106. return "Less than a second"
  107. } else if seconds < 60 {
  108. return fmt.Sprintf("%d seconds", seconds)
  109. } else if minutes := int(d.Minutes()); minutes == 1 {
  110. return "About a minute"
  111. } else if minutes < 60 {
  112. return fmt.Sprintf("%d minutes", minutes)
  113. } else if hours := int(d.Hours()); hours == 1 {
  114. return "About an hour"
  115. } else if hours < 48 {
  116. return fmt.Sprintf("%d hours", hours)
  117. } else if hours < 24*7*2 {
  118. return fmt.Sprintf("%d days", hours/24)
  119. } else if hours < 24*30*3 {
  120. return fmt.Sprintf("%d weeks", hours/24/7)
  121. } else if hours < 24*365*2 {
  122. return fmt.Sprintf("%d months", hours/24/30)
  123. }
  124. return fmt.Sprintf("%d years", d.Hours()/24/365)
  125. }
  126. func Trunc(s string, maxlen int) string {
  127. if len(s) <= maxlen {
  128. return s
  129. }
  130. return s[:maxlen]
  131. }
  132. // Figure out the absolute path of our own binary
  133. func SelfPath() string {
  134. path, err := exec.LookPath(os.Args[0])
  135. if err != nil {
  136. panic(err)
  137. }
  138. path, err = filepath.Abs(path)
  139. if err != nil {
  140. panic(err)
  141. }
  142. return path
  143. }
  144. type NopWriter struct {
  145. }
  146. func (w *NopWriter) Write(buf []byte) (int, error) {
  147. return len(buf), nil
  148. }
  149. type nopWriteCloser struct {
  150. io.Writer
  151. }
  152. func (w *nopWriteCloser) Close() error { return nil }
  153. func NopWriteCloser(w io.Writer) io.WriteCloser {
  154. return &nopWriteCloser{w}
  155. }
  156. type bufReader struct {
  157. buf *bytes.Buffer
  158. reader io.Reader
  159. err error
  160. l sync.Mutex
  161. wait sync.Cond
  162. }
  163. func NewBufReader(r io.Reader) *bufReader {
  164. reader := &bufReader{
  165. buf: &bytes.Buffer{},
  166. reader: r,
  167. }
  168. reader.wait.L = &reader.l
  169. go reader.drain()
  170. return reader
  171. }
  172. func (r *bufReader) drain() {
  173. buf := make([]byte, 1024)
  174. for {
  175. n, err := r.reader.Read(buf)
  176. r.l.Lock()
  177. if err != nil {
  178. r.err = err
  179. } else {
  180. r.buf.Write(buf[0:n])
  181. }
  182. r.wait.Signal()
  183. r.l.Unlock()
  184. if err != nil {
  185. break
  186. }
  187. }
  188. }
  189. func (r *bufReader) Read(p []byte) (n int, err error) {
  190. r.l.Lock()
  191. defer r.l.Unlock()
  192. for {
  193. n, err = r.buf.Read(p)
  194. if n > 0 {
  195. return n, err
  196. }
  197. if r.err != nil {
  198. return 0, r.err
  199. }
  200. r.wait.Wait()
  201. }
  202. panic("unreachable")
  203. }
  204. func (r *bufReader) Close() error {
  205. closer, ok := r.reader.(io.ReadCloser)
  206. if !ok {
  207. return nil
  208. }
  209. return closer.Close()
  210. }
  211. type WriteBroadcaster struct {
  212. mu sync.Mutex
  213. writers map[io.WriteCloser]struct{}
  214. }
  215. func (w *WriteBroadcaster) AddWriter(writer io.WriteCloser) {
  216. w.mu.Lock()
  217. w.writers[writer] = struct{}{}
  218. w.mu.Unlock()
  219. }
  220. // FIXME: Is that function used?
  221. // FIXME: This relies on the concrete writer type used having equality operator
  222. func (w *WriteBroadcaster) RemoveWriter(writer io.WriteCloser) {
  223. w.mu.Lock()
  224. delete(w.writers, writer)
  225. w.mu.Unlock()
  226. }
  227. func (w *WriteBroadcaster) Write(p []byte) (n int, err error) {
  228. w.mu.Lock()
  229. defer w.mu.Unlock()
  230. for writer := range w.writers {
  231. if n, err := writer.Write(p); err != nil || n != len(p) {
  232. // On error, evict the writer
  233. delete(w.writers, writer)
  234. }
  235. }
  236. return len(p), nil
  237. }
  238. func (w *WriteBroadcaster) CloseWriters() error {
  239. w.mu.Lock()
  240. defer w.mu.Unlock()
  241. for writer := range w.writers {
  242. writer.Close()
  243. }
  244. w.writers = make(map[io.WriteCloser]struct{})
  245. return nil
  246. }
  247. func NewWriteBroadcaster() *WriteBroadcaster {
  248. return &WriteBroadcaster{writers: make(map[io.WriteCloser]struct{})}
  249. }
  250. func GetTotalUsedFds() int {
  251. if fds, err := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid())); err != nil {
  252. Debugf("Error opening /proc/%d/fd: %s", os.Getpid(), err)
  253. } else {
  254. return len(fds)
  255. }
  256. return -1
  257. }
  258. // TruncIndex allows the retrieval of string identifiers by any of their unique prefixes.
  259. // This is used to retrieve image and container IDs by more convenient shorthand prefixes.
  260. type TruncIndex struct {
  261. index *suffixarray.Index
  262. ids map[string]bool
  263. bytes []byte
  264. }
  265. func NewTruncIndex() *TruncIndex {
  266. return &TruncIndex{
  267. index: suffixarray.New([]byte{' '}),
  268. ids: make(map[string]bool),
  269. bytes: []byte{' '},
  270. }
  271. }
  272. func (idx *TruncIndex) Add(id string) error {
  273. if strings.Contains(id, " ") {
  274. return fmt.Errorf("Illegal character: ' '")
  275. }
  276. if _, exists := idx.ids[id]; exists {
  277. return fmt.Errorf("Id already exists: %s", id)
  278. }
  279. idx.ids[id] = true
  280. idx.bytes = append(idx.bytes, []byte(id+" ")...)
  281. idx.index = suffixarray.New(idx.bytes)
  282. return nil
  283. }
  284. func (idx *TruncIndex) Delete(id string) error {
  285. if _, exists := idx.ids[id]; !exists {
  286. return fmt.Errorf("No such id: %s", id)
  287. }
  288. before, after, err := idx.lookup(id)
  289. if err != nil {
  290. return err
  291. }
  292. delete(idx.ids, id)
  293. idx.bytes = append(idx.bytes[:before], idx.bytes[after:]...)
  294. idx.index = suffixarray.New(idx.bytes)
  295. return nil
  296. }
  297. func (idx *TruncIndex) lookup(s string) (int, int, error) {
  298. offsets := idx.index.Lookup([]byte(" "+s), -1)
  299. //log.Printf("lookup(%s): %v (index bytes: '%s')\n", s, offsets, idx.index.Bytes())
  300. if offsets == nil || len(offsets) == 0 || len(offsets) > 1 {
  301. return -1, -1, fmt.Errorf("No such id: %s", s)
  302. }
  303. offsetBefore := offsets[0] + 1
  304. offsetAfter := offsetBefore + strings.Index(string(idx.bytes[offsetBefore:]), " ")
  305. return offsetBefore, offsetAfter, nil
  306. }
  307. func (idx *TruncIndex) Get(s string) (string, error) {
  308. before, after, err := idx.lookup(s)
  309. //log.Printf("Get(%s) bytes=|%s| before=|%d| after=|%d|\n", s, idx.bytes, before, after)
  310. if err != nil {
  311. return "", err
  312. }
  313. return string(idx.bytes[before:after]), err
  314. }
  315. // TruncateID returns a shorthand version of a string identifier for convenience.
  316. // A collision with other shorthands is very unlikely, but possible.
  317. // In case of a collision a lookup with TruncIndex.Get() will fail, and the caller
  318. // will need to use a langer prefix, or the full-length Id.
  319. func TruncateID(id string) string {
  320. shortLen := 12
  321. if len(id) < shortLen {
  322. shortLen = len(id)
  323. }
  324. return id[:shortLen]
  325. }
  326. // Code c/c from io.Copy() modified to handle escape sequence
  327. func CopyEscapable(dst io.Writer, src io.ReadCloser) (written int64, err error) {
  328. buf := make([]byte, 32*1024)
  329. for {
  330. nr, er := src.Read(buf)
  331. if nr > 0 {
  332. // ---- Docker addition
  333. // char 16 is C-p
  334. if nr == 1 && buf[0] == 16 {
  335. nr, er = src.Read(buf)
  336. // char 17 is C-q
  337. if nr == 1 && buf[0] == 17 {
  338. if err := src.Close(); err != nil {
  339. return 0, err
  340. }
  341. return 0, io.EOF
  342. }
  343. }
  344. // ---- End of docker
  345. nw, ew := dst.Write(buf[0:nr])
  346. if nw > 0 {
  347. written += int64(nw)
  348. }
  349. if ew != nil {
  350. err = ew
  351. break
  352. }
  353. if nr != nw {
  354. err = io.ErrShortWrite
  355. break
  356. }
  357. }
  358. if er == io.EOF {
  359. break
  360. }
  361. if er != nil {
  362. err = er
  363. break
  364. }
  365. }
  366. return written, err
  367. }
  368. func HashData(src io.Reader) (string, error) {
  369. h := sha256.New()
  370. if _, err := io.Copy(h, src); err != nil {
  371. return "", err
  372. }
  373. return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil
  374. }
  375. type KernelVersionInfo struct {
  376. Kernel int
  377. Major int
  378. Minor int
  379. Flavor string
  380. }
  381. func (k *KernelVersionInfo) String() string {
  382. flavor := ""
  383. if len(k.Flavor) > 0 {
  384. flavor = fmt.Sprintf("-%s", k.Flavor)
  385. }
  386. return fmt.Sprintf("%d.%d.%d%s", k.Kernel, k.Major, k.Minor, flavor)
  387. }
  388. // Compare two KernelVersionInfo struct.
  389. // Returns -1 if a < b, = if a == b, 1 it a > b
  390. func CompareKernelVersion(a, b *KernelVersionInfo) int {
  391. if a.Kernel < b.Kernel {
  392. return -1
  393. } else if a.Kernel > b.Kernel {
  394. return 1
  395. }
  396. if a.Major < b.Major {
  397. return -1
  398. } else if a.Major > b.Major {
  399. return 1
  400. }
  401. if a.Minor < b.Minor {
  402. return -1
  403. } else if a.Minor > b.Minor {
  404. return 1
  405. }
  406. return 0
  407. }
  408. func FindCgroupMountpoint(cgroupType string) (string, error) {
  409. output, err := ioutil.ReadFile("/proc/mounts")
  410. if err != nil {
  411. return "", err
  412. }
  413. // /proc/mounts has 6 fields per line, one mount per line, e.g.
  414. // cgroup /sys/fs/cgroup/devices cgroup rw,relatime,devices 0 0
  415. for _, line := range strings.Split(string(output), "\n") {
  416. parts := strings.Split(line, " ")
  417. if len(parts) == 6 && parts[2] == "cgroup" {
  418. for _, opt := range strings.Split(parts[3], ",") {
  419. if opt == cgroupType {
  420. return parts[1], nil
  421. }
  422. }
  423. }
  424. }
  425. return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType)
  426. }
  427. func GetKernelVersion() (*KernelVersionInfo, error) {
  428. var (
  429. flavor string
  430. kernel, major, minor int
  431. err error
  432. )
  433. uts, err := uname()
  434. if err != nil {
  435. return nil, err
  436. }
  437. release := make([]byte, len(uts.Release))
  438. i := 0
  439. for _, c := range uts.Release {
  440. release[i] = byte(c)
  441. i++
  442. }
  443. // Remove the \x00 from the release for Atoi to parse correctly
  444. release = release[:bytes.IndexByte(release, 0)]
  445. tmp := strings.SplitN(string(release), "-", 2)
  446. tmp2 := strings.SplitN(tmp[0], ".", 3)
  447. if len(tmp2) > 0 {
  448. kernel, err = strconv.Atoi(tmp2[0])
  449. if err != nil {
  450. return nil, err
  451. }
  452. }
  453. if len(tmp2) > 1 {
  454. major, err = strconv.Atoi(tmp2[1])
  455. if err != nil {
  456. return nil, err
  457. }
  458. }
  459. if len(tmp2) > 2 {
  460. minor, err = strconv.Atoi(tmp2[2])
  461. if err != nil {
  462. return nil, err
  463. }
  464. }
  465. if len(tmp) == 2 {
  466. flavor = tmp[1]
  467. } else {
  468. flavor = ""
  469. }
  470. return &KernelVersionInfo{
  471. Kernel: kernel,
  472. Major: major,
  473. Minor: minor,
  474. Flavor: flavor,
  475. }, nil
  476. }
  477. func CopyDirectory(source, dest string) error {
  478. if output, err := exec.Command("cp", "-ra", source, dest).CombinedOutput(); err != nil {
  479. return fmt.Errorf("Error copy: %s (%s)", err, output)
  480. }
  481. return nil
  482. }
  483. type NopFlusher struct{}
  484. func (f *NopFlusher) Flush() {}
  485. type WriteFlusher struct {
  486. w io.Writer
  487. flusher http.Flusher
  488. }
  489. func (wf *WriteFlusher) Write(b []byte) (n int, err error) {
  490. n, err = wf.w.Write(b)
  491. wf.flusher.Flush()
  492. return n, err
  493. }
  494. func NewWriteFlusher(w io.Writer) *WriteFlusher {
  495. var flusher http.Flusher
  496. if f, ok := w.(http.Flusher); ok {
  497. flusher = f
  498. } else {
  499. flusher = &NopFlusher{}
  500. }
  501. return &WriteFlusher{w: w, flusher: flusher}
  502. }
  503. type JSONMessage struct {
  504. Status string `json:"status,omitempty"`
  505. Progress string `json:"progress,omitempty"`
  506. Error string `json:"error,omitempty"`
  507. }
  508. type StreamFormatter struct {
  509. json bool
  510. used bool
  511. }
  512. func NewStreamFormatter(json bool) *StreamFormatter {
  513. return &StreamFormatter{json, false}
  514. }
  515. func (sf *StreamFormatter) FormatStatus(format string, a ...interface{}) []byte {
  516. sf.used = true
  517. str := fmt.Sprintf(format, a...)
  518. if sf.json {
  519. b, err := json.Marshal(&JSONMessage{Status: str})
  520. if err != nil {
  521. return sf.FormatError(err)
  522. }
  523. return b
  524. }
  525. return []byte(str + "\r\n")
  526. }
  527. func (sf *StreamFormatter) FormatError(err error) []byte {
  528. sf.used = true
  529. if sf.json {
  530. if b, err := json.Marshal(&JSONMessage{Error: err.Error()}); err == nil {
  531. return b
  532. }
  533. return []byte("{\"error\":\"format error\"}")
  534. }
  535. return []byte("Error: " + err.Error() + "\r\n")
  536. }
  537. func (sf *StreamFormatter) FormatProgress(action, str string) []byte {
  538. sf.used = true
  539. if sf.json {
  540. b, err := json.Marshal(&JSONMessage{Status: action, Progress: str})
  541. if err != nil {
  542. return nil
  543. }
  544. return b
  545. }
  546. return []byte(action + " " + str + "\r")
  547. }
  548. func (sf *StreamFormatter) Used() bool {
  549. return sf.used
  550. }