utils.go 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682
  1. package utils
  2. import (
  3. "bytes"
  4. "crypto/sha256"
  5. "encoding/hex"
  6. "encoding/json"
  7. "errors"
  8. "fmt"
  9. "index/suffixarray"
  10. "io"
  11. "io/ioutil"
  12. "log"
  13. "net/http"
  14. "os"
  15. "os/exec"
  16. "path/filepath"
  17. "runtime"
  18. "strconv"
  19. "strings"
  20. "sync"
  21. "time"
  22. )
  23. // Go is a basic promise implementation: it wraps calls a function in a goroutine,
  24. // and returns a channel which will later return the function's return value.
  25. func Go(f func() error) chan error {
  26. ch := make(chan error)
  27. go func() {
  28. ch <- f()
  29. }()
  30. return ch
  31. }
  32. // Request a given URL and return an io.Reader
  33. func Download(url string, stderr io.Writer) (*http.Response, error) {
  34. var resp *http.Response
  35. var err error
  36. if resp, err = http.Get(url); err != nil {
  37. return nil, err
  38. }
  39. if resp.StatusCode >= 400 {
  40. return nil, errors.New("Got HTTP status code >= 400: " + resp.Status)
  41. }
  42. return resp, nil
  43. }
  44. // Debug function, if the debug flag is set, then display. Do nothing otherwise
  45. // If Docker is in damon mode, also send the debug info on the socket
  46. func Debugf(format string, a ...interface{}) {
  47. if os.Getenv("DEBUG") != "" {
  48. // Retrieve the stack infos
  49. _, file, line, ok := runtime.Caller(1)
  50. if !ok {
  51. file = "<unknown>"
  52. line = -1
  53. } else {
  54. file = file[strings.LastIndex(file, "/")+1:]
  55. }
  56. fmt.Fprintf(os.Stderr, fmt.Sprintf("[debug] %s:%d %s\n", file, line, format), a...)
  57. }
  58. }
  59. // Reader with progress bar
  60. type progressReader struct {
  61. reader io.ReadCloser // Stream to read from
  62. output io.Writer // Where to send progress bar to
  63. readTotal int // Expected stream length (bytes)
  64. readProgress int // How much has been read so far (bytes)
  65. lastUpdate int // How many bytes read at least update
  66. template string // Template to print. Default "%v/%v (%v)"
  67. sf *StreamFormatter
  68. }
  69. func (r *progressReader) Read(p []byte) (n int, err error) {
  70. read, err := io.ReadCloser(r.reader).Read(p)
  71. r.readProgress += read
  72. updateEvery := 4096
  73. if r.readTotal > 0 {
  74. // Only update progress for every 1% read
  75. if increment := int(0.01 * float64(r.readTotal)); increment > updateEvery {
  76. updateEvery = increment
  77. }
  78. }
  79. if r.readProgress-r.lastUpdate > updateEvery || err != nil {
  80. if r.readTotal > 0 {
  81. fmt.Fprintf(r.output, r.template, HumanSize(int64(r.readProgress)), HumanSize(int64(r.readTotal)), fmt.Sprintf("%.0f%%", float64(r.readProgress)/float64(r.readTotal)*100))
  82. } else {
  83. fmt.Fprintf(r.output, r.template, r.readProgress, "?", "n/a")
  84. }
  85. r.lastUpdate = r.readProgress
  86. }
  87. // Send newline when complete
  88. if err != nil {
  89. r.output.Write(r.sf.FormatStatus(""))
  90. }
  91. return read, err
  92. }
  93. func (r *progressReader) Close() error {
  94. return io.ReadCloser(r.reader).Close()
  95. }
  96. func ProgressReader(r io.ReadCloser, size int, output io.Writer, template []byte, sf *StreamFormatter) *progressReader {
  97. tpl := string(template)
  98. if tpl == "" {
  99. tpl = string(sf.FormatProgress("", "%v/%v (%v)"))
  100. }
  101. return &progressReader{r, NewWriteFlusher(output), size, 0, 0, tpl, sf}
  102. }
  103. // HumanDuration returns a human-readable approximation of a duration
  104. // (eg. "About a minute", "4 hours ago", etc.)
  105. func HumanDuration(d time.Duration) string {
  106. if seconds := int(d.Seconds()); seconds < 1 {
  107. return "Less than a second"
  108. } else if seconds < 60 {
  109. return fmt.Sprintf("%d seconds", seconds)
  110. } else if minutes := int(d.Minutes()); minutes == 1 {
  111. return "About a minute"
  112. } else if minutes < 60 {
  113. return fmt.Sprintf("%d minutes", minutes)
  114. } else if hours := int(d.Hours()); hours == 1 {
  115. return "About an hour"
  116. } else if hours < 48 {
  117. return fmt.Sprintf("%d hours", hours)
  118. } else if hours < 24*7*2 {
  119. return fmt.Sprintf("%d days", hours/24)
  120. } else if hours < 24*30*3 {
  121. return fmt.Sprintf("%d weeks", hours/24/7)
  122. } else if hours < 24*365*2 {
  123. return fmt.Sprintf("%d months", hours/24/30)
  124. }
  125. return fmt.Sprintf("%d years", d.Hours()/24/365)
  126. }
  127. // HumanSize returns a human-readable approximation of a size
  128. // using SI standard (eg. "44kB", "17MB")
  129. func HumanSize(size int64) string {
  130. i := 0
  131. var sizef float64
  132. sizef = float64(size)
  133. units := []string{"B", "kB", "MB", "GB", "TB", "PB", "EB", "ZB", "YB"}
  134. for sizef >= 1000.0 {
  135. sizef = sizef / 1000.0
  136. i++
  137. }
  138. return fmt.Sprintf("%.4g %s", sizef, units[i])
  139. }
  140. func Trunc(s string, maxlen int) string {
  141. if len(s) <= maxlen {
  142. return s
  143. }
  144. return s[:maxlen]
  145. }
  146. // Figure out the absolute path of our own binary
  147. func SelfPath() string {
  148. path, err := exec.LookPath(os.Args[0])
  149. if err != nil {
  150. panic(err)
  151. }
  152. path, err = filepath.Abs(path)
  153. if err != nil {
  154. panic(err)
  155. }
  156. return path
  157. }
  158. type NopWriter struct {
  159. }
  160. func (w *NopWriter) Write(buf []byte) (int, error) {
  161. return len(buf), nil
  162. }
  163. type nopWriteCloser struct {
  164. io.Writer
  165. }
  166. func (w *nopWriteCloser) Close() error { return nil }
  167. func NopWriteCloser(w io.Writer) io.WriteCloser {
  168. return &nopWriteCloser{w}
  169. }
  170. type bufReader struct {
  171. buf *bytes.Buffer
  172. reader io.Reader
  173. err error
  174. l sync.Mutex
  175. wait sync.Cond
  176. }
  177. func NewBufReader(r io.Reader) *bufReader {
  178. reader := &bufReader{
  179. buf: &bytes.Buffer{},
  180. reader: r,
  181. }
  182. reader.wait.L = &reader.l
  183. go reader.drain()
  184. return reader
  185. }
  186. func (r *bufReader) drain() {
  187. buf := make([]byte, 1024)
  188. for {
  189. n, err := r.reader.Read(buf)
  190. r.l.Lock()
  191. if err != nil {
  192. r.err = err
  193. } else {
  194. r.buf.Write(buf[0:n])
  195. }
  196. r.wait.Signal()
  197. r.l.Unlock()
  198. if err != nil {
  199. break
  200. }
  201. }
  202. }
  203. func (r *bufReader) Read(p []byte) (n int, err error) {
  204. r.l.Lock()
  205. defer r.l.Unlock()
  206. for {
  207. n, err = r.buf.Read(p)
  208. if n > 0 {
  209. return n, err
  210. }
  211. if r.err != nil {
  212. return 0, r.err
  213. }
  214. r.wait.Wait()
  215. }
  216. panic("unreachable")
  217. }
  218. func (r *bufReader) Close() error {
  219. closer, ok := r.reader.(io.ReadCloser)
  220. if !ok {
  221. return nil
  222. }
  223. return closer.Close()
  224. }
  225. type WriteBroadcaster struct {
  226. mu sync.Mutex
  227. writers map[io.WriteCloser]struct{}
  228. }
  229. func (w *WriteBroadcaster) AddWriter(writer io.WriteCloser) {
  230. w.mu.Lock()
  231. w.writers[writer] = struct{}{}
  232. w.mu.Unlock()
  233. }
  234. // FIXME: Is that function used?
  235. // FIXME: This relies on the concrete writer type used having equality operator
  236. func (w *WriteBroadcaster) RemoveWriter(writer io.WriteCloser) {
  237. w.mu.Lock()
  238. delete(w.writers, writer)
  239. w.mu.Unlock()
  240. }
  241. func (w *WriteBroadcaster) Write(p []byte) (n int, err error) {
  242. w.mu.Lock()
  243. defer w.mu.Unlock()
  244. for writer := range w.writers {
  245. if n, err := writer.Write(p); err != nil || n != len(p) {
  246. // On error, evict the writer
  247. delete(w.writers, writer)
  248. }
  249. }
  250. return len(p), nil
  251. }
  252. func (w *WriteBroadcaster) CloseWriters() error {
  253. w.mu.Lock()
  254. defer w.mu.Unlock()
  255. for writer := range w.writers {
  256. writer.Close()
  257. }
  258. w.writers = make(map[io.WriteCloser]struct{})
  259. return nil
  260. }
  261. func NewWriteBroadcaster() *WriteBroadcaster {
  262. return &WriteBroadcaster{writers: make(map[io.WriteCloser]struct{})}
  263. }
  264. func GetTotalUsedFds() int {
  265. if fds, err := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid())); err != nil {
  266. Debugf("Error opening /proc/%d/fd: %s", os.Getpid(), err)
  267. } else {
  268. return len(fds)
  269. }
  270. return -1
  271. }
  272. // TruncIndex allows the retrieval of string identifiers by any of their unique prefixes.
  273. // This is used to retrieve image and container IDs by more convenient shorthand prefixes.
  274. type TruncIndex struct {
  275. index *suffixarray.Index
  276. ids map[string]bool
  277. bytes []byte
  278. }
  279. func NewTruncIndex() *TruncIndex {
  280. return &TruncIndex{
  281. index: suffixarray.New([]byte{' '}),
  282. ids: make(map[string]bool),
  283. bytes: []byte{' '},
  284. }
  285. }
  286. func (idx *TruncIndex) Add(id string) error {
  287. if strings.Contains(id, " ") {
  288. return fmt.Errorf("Illegal character: ' '")
  289. }
  290. if _, exists := idx.ids[id]; exists {
  291. return fmt.Errorf("Id already exists: %s", id)
  292. }
  293. idx.ids[id] = true
  294. idx.bytes = append(idx.bytes, []byte(id+" ")...)
  295. idx.index = suffixarray.New(idx.bytes)
  296. return nil
  297. }
  298. func (idx *TruncIndex) Delete(id string) error {
  299. if _, exists := idx.ids[id]; !exists {
  300. return fmt.Errorf("No such id: %s", id)
  301. }
  302. before, after, err := idx.lookup(id)
  303. if err != nil {
  304. return err
  305. }
  306. delete(idx.ids, id)
  307. idx.bytes = append(idx.bytes[:before], idx.bytes[after:]...)
  308. idx.index = suffixarray.New(idx.bytes)
  309. return nil
  310. }
  311. func (idx *TruncIndex) lookup(s string) (int, int, error) {
  312. offsets := idx.index.Lookup([]byte(" "+s), -1)
  313. //log.Printf("lookup(%s): %v (index bytes: '%s')\n", s, offsets, idx.index.Bytes())
  314. if offsets == nil || len(offsets) == 0 || len(offsets) > 1 {
  315. return -1, -1, fmt.Errorf("No such id: %s", s)
  316. }
  317. offsetBefore := offsets[0] + 1
  318. offsetAfter := offsetBefore + strings.Index(string(idx.bytes[offsetBefore:]), " ")
  319. return offsetBefore, offsetAfter, nil
  320. }
  321. func (idx *TruncIndex) Get(s string) (string, error) {
  322. before, after, err := idx.lookup(s)
  323. //log.Printf("Get(%s) bytes=|%s| before=|%d| after=|%d|\n", s, idx.bytes, before, after)
  324. if err != nil {
  325. return "", err
  326. }
  327. return string(idx.bytes[before:after]), err
  328. }
  329. // TruncateID returns a shorthand version of a string identifier for convenience.
  330. // A collision with other shorthands is very unlikely, but possible.
  331. // In case of a collision a lookup with TruncIndex.Get() will fail, and the caller
  332. // will need to use a langer prefix, or the full-length Id.
  333. func TruncateID(id string) string {
  334. shortLen := 12
  335. if len(id) < shortLen {
  336. shortLen = len(id)
  337. }
  338. return id[:shortLen]
  339. }
  340. // Code c/c from io.Copy() modified to handle escape sequence
  341. func CopyEscapable(dst io.Writer, src io.ReadCloser) (written int64, err error) {
  342. buf := make([]byte, 32*1024)
  343. for {
  344. nr, er := src.Read(buf)
  345. if nr > 0 {
  346. // ---- Docker addition
  347. // char 16 is C-p
  348. if nr == 1 && buf[0] == 16 {
  349. nr, er = src.Read(buf)
  350. // char 17 is C-q
  351. if nr == 1 && buf[0] == 17 {
  352. if err := src.Close(); err != nil {
  353. return 0, err
  354. }
  355. return 0, io.EOF
  356. }
  357. }
  358. // ---- End of docker
  359. nw, ew := dst.Write(buf[0:nr])
  360. if nw > 0 {
  361. written += int64(nw)
  362. }
  363. if ew != nil {
  364. err = ew
  365. break
  366. }
  367. if nr != nw {
  368. err = io.ErrShortWrite
  369. break
  370. }
  371. }
  372. if er == io.EOF {
  373. break
  374. }
  375. if er != nil {
  376. err = er
  377. break
  378. }
  379. }
  380. return written, err
  381. }
  382. func HashData(src io.Reader) (string, error) {
  383. h := sha256.New()
  384. if _, err := io.Copy(h, src); err != nil {
  385. return "", err
  386. }
  387. return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil
  388. }
  389. type KernelVersionInfo struct {
  390. Kernel int
  391. Major int
  392. Minor int
  393. Flavor string
  394. }
  395. func (k *KernelVersionInfo) String() string {
  396. flavor := ""
  397. if len(k.Flavor) > 0 {
  398. flavor = fmt.Sprintf("-%s", k.Flavor)
  399. }
  400. return fmt.Sprintf("%d.%d.%d%s", k.Kernel, k.Major, k.Minor, flavor)
  401. }
  402. // Compare two KernelVersionInfo struct.
  403. // Returns -1 if a < b, = if a == b, 1 it a > b
  404. func CompareKernelVersion(a, b *KernelVersionInfo) int {
  405. if a.Kernel < b.Kernel {
  406. return -1
  407. } else if a.Kernel > b.Kernel {
  408. return 1
  409. }
  410. if a.Major < b.Major {
  411. return -1
  412. } else if a.Major > b.Major {
  413. return 1
  414. }
  415. if a.Minor < b.Minor {
  416. return -1
  417. } else if a.Minor > b.Minor {
  418. return 1
  419. }
  420. return 0
  421. }
  422. func FindCgroupMountpoint(cgroupType string) (string, error) {
  423. output, err := ioutil.ReadFile("/proc/mounts")
  424. if err != nil {
  425. return "", err
  426. }
  427. // /proc/mounts has 6 fields per line, one mount per line, e.g.
  428. // cgroup /sys/fs/cgroup/devices cgroup rw,relatime,devices 0 0
  429. for _, line := range strings.Split(string(output), "\n") {
  430. parts := strings.Split(line, " ")
  431. if len(parts) == 6 && parts[2] == "cgroup" {
  432. for _, opt := range strings.Split(parts[3], ",") {
  433. if opt == cgroupType {
  434. return parts[1], nil
  435. }
  436. }
  437. }
  438. }
  439. return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType)
  440. }
  441. func GetKernelVersion() (*KernelVersionInfo, error) {
  442. var (
  443. flavor string
  444. kernel, major, minor int
  445. err error
  446. )
  447. uts, err := uname()
  448. if err != nil {
  449. return nil, err
  450. }
  451. release := make([]byte, len(uts.Release))
  452. i := 0
  453. for _, c := range uts.Release {
  454. release[i] = byte(c)
  455. i++
  456. }
  457. // Remove the \x00 from the release for Atoi to parse correctly
  458. release = release[:bytes.IndexByte(release, 0)]
  459. tmp := strings.SplitN(string(release), "-", 2)
  460. tmp2 := strings.SplitN(tmp[0], ".", 3)
  461. if len(tmp2) > 0 {
  462. kernel, err = strconv.Atoi(tmp2[0])
  463. if err != nil {
  464. return nil, err
  465. }
  466. }
  467. if len(tmp2) > 1 {
  468. major, err = strconv.Atoi(tmp2[1])
  469. if err != nil {
  470. return nil, err
  471. }
  472. }
  473. if len(tmp2) > 2 {
  474. minor, err = strconv.Atoi(tmp2[2])
  475. if err != nil {
  476. return nil, err
  477. }
  478. }
  479. if len(tmp) == 2 {
  480. flavor = tmp[1]
  481. } else {
  482. flavor = ""
  483. }
  484. return &KernelVersionInfo{
  485. Kernel: kernel,
  486. Major: major,
  487. Minor: minor,
  488. Flavor: flavor,
  489. }, nil
  490. }
  491. // FIXME: this is deprecated by CopyWithTar in archive.go
  492. func CopyDirectory(source, dest string) error {
  493. if output, err := exec.Command("cp", "-ra", source, dest).CombinedOutput(); err != nil {
  494. return fmt.Errorf("Error copy: %s (%s)", err, output)
  495. }
  496. return nil
  497. }
  498. type NopFlusher struct{}
  499. func (f *NopFlusher) Flush() {}
  500. type WriteFlusher struct {
  501. w io.Writer
  502. flusher http.Flusher
  503. }
  504. func (wf *WriteFlusher) Write(b []byte) (n int, err error) {
  505. n, err = wf.w.Write(b)
  506. wf.flusher.Flush()
  507. return n, err
  508. }
  509. func NewWriteFlusher(w io.Writer) *WriteFlusher {
  510. var flusher http.Flusher
  511. if f, ok := w.(http.Flusher); ok {
  512. flusher = f
  513. } else {
  514. flusher = &NopFlusher{}
  515. }
  516. return &WriteFlusher{w: w, flusher: flusher}
  517. }
  518. type JSONMessage struct {
  519. Status string `json:"status,omitempty"`
  520. Progress string `json:"progress,omitempty"`
  521. Error string `json:"error,omitempty"`
  522. }
  523. type StreamFormatter struct {
  524. json bool
  525. used bool
  526. }
  527. func NewStreamFormatter(json bool) *StreamFormatter {
  528. return &StreamFormatter{json, false}
  529. }
  530. func (sf *StreamFormatter) FormatStatus(format string, a ...interface{}) []byte {
  531. sf.used = true
  532. str := fmt.Sprintf(format, a...)
  533. if sf.json {
  534. b, err := json.Marshal(&JSONMessage{Status: str})
  535. if err != nil {
  536. return sf.FormatError(err)
  537. }
  538. return b
  539. }
  540. return []byte(str + "\r\n")
  541. }
  542. func (sf *StreamFormatter) FormatError(err error) []byte {
  543. sf.used = true
  544. if sf.json {
  545. if b, err := json.Marshal(&JSONMessage{Error: err.Error()}); err == nil {
  546. return b
  547. }
  548. return []byte("{\"error\":\"format error\"}")
  549. }
  550. return []byte("Error: " + err.Error() + "\r\n")
  551. }
  552. func (sf *StreamFormatter) FormatProgress(action, str string) []byte {
  553. sf.used = true
  554. if sf.json {
  555. b, err := json.Marshal(&JSONMessage{Status: action, Progress: str})
  556. if err != nil {
  557. return nil
  558. }
  559. return b
  560. }
  561. return []byte(action + " " + str + "\r")
  562. }
  563. func (sf *StreamFormatter) Used() bool {
  564. return sf.used
  565. }
  566. func CheckLocalDns() bool {
  567. resolv, err := ioutil.ReadFile("/etc/resolv.conf")
  568. if err != nil {
  569. Debugf("Error openning resolv.conf: %s", err)
  570. return false
  571. }
  572. for _, ip := range []string{
  573. "127.0.0.1",
  574. "127.0.1.1",
  575. } {
  576. if strings.Contains(string(resolv), ip) {
  577. return true
  578. }
  579. }
  580. return false
  581. }
  582. func ParseHost(host string, port int, addr string) string {
  583. if strings.HasPrefix(addr, "unix://") {
  584. return addr
  585. }
  586. if strings.HasPrefix(addr, "tcp://") {
  587. addr = strings.TrimPrefix(addr, "tcp://")
  588. }
  589. if strings.Contains(addr, ":") {
  590. hostParts := strings.Split(addr, ":")
  591. if len(hostParts) != 2 {
  592. log.Fatal("Invalid bind address format.")
  593. os.Exit(-1)
  594. }
  595. if hostParts[0] != "" {
  596. host = hostParts[0]
  597. }
  598. if p, err := strconv.Atoi(hostParts[1]); err == nil {
  599. port = p
  600. }
  601. } else {
  602. host = addr
  603. }
  604. return fmt.Sprintf("tcp://%s:%d", host, port)
  605. }