utils.go 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579
  1. package utils
  2. import (
  3. "bytes"
  4. "crypto/sha256"
  5. "encoding/hex"
  6. "errors"
  7. "fmt"
  8. "index/suffixarray"
  9. "io"
  10. "io/ioutil"
  11. "net/http"
  12. "os"
  13. "os/exec"
  14. "path/filepath"
  15. "runtime"
  16. "strconv"
  17. "strings"
  18. "sync"
  19. "time"
  20. )
  21. // Go is a basic promise implementation: it wraps calls a function in a goroutine,
  22. // and returns a channel which will later return the function's return value.
  23. func Go(f func() error) chan error {
  24. ch := make(chan error)
  25. go func() {
  26. ch <- f()
  27. }()
  28. return ch
  29. }
  30. // Request a given URL and return an io.Reader
  31. func Download(url string, stderr io.Writer) (*http.Response, error) {
  32. var resp *http.Response
  33. var err error = nil
  34. if resp, err = http.Get(url); err != nil {
  35. return nil, err
  36. }
  37. if resp.StatusCode >= 400 {
  38. return nil, errors.New("Got HTTP status code >= 400: " + resp.Status)
  39. }
  40. return resp, nil
  41. }
  42. // Debug function, if the debug flag is set, then display. Do nothing otherwise
  43. // If Docker is in damon mode, also send the debug info on the socket
  44. func Debugf(format string, a ...interface{}) {
  45. if os.Getenv("DEBUG") != "" {
  46. // Retrieve the stack infos
  47. _, file, line, ok := runtime.Caller(1)
  48. if !ok {
  49. file = "<unknown>"
  50. line = -1
  51. } else {
  52. file = file[strings.LastIndex(file, "/")+1:]
  53. }
  54. fmt.Fprintf(os.Stderr, fmt.Sprintf("[debug] %s:%d %s\n", file, line, format), a...)
  55. }
  56. }
  57. // Reader with progress bar
  58. type progressReader struct {
  59. reader io.ReadCloser // Stream to read from
  60. output io.Writer // Where to send progress bar to
  61. readTotal int // Expected stream length (bytes)
  62. readProgress int // How much has been read so far (bytes)
  63. lastUpdate int // How many bytes read at least update
  64. template string // Template to print. Default "%v/%v (%v)"
  65. json bool
  66. }
  67. func (r *progressReader) Read(p []byte) (n int, err error) {
  68. read, err := io.ReadCloser(r.reader).Read(p)
  69. r.readProgress += read
  70. updateEvery := 4096
  71. if r.readTotal > 0 {
  72. // Only update progress for every 1% read
  73. if increment := int(0.01 * float64(r.readTotal)); increment > updateEvery {
  74. updateEvery = increment
  75. }
  76. }
  77. if r.readProgress-r.lastUpdate > updateEvery || err != nil {
  78. if r.readTotal > 0 {
  79. fmt.Fprintf(r.output, r.template, r.readProgress, r.readTotal, fmt.Sprintf("%.0f%%", float64(r.readProgress)/float64(r.readTotal)*100))
  80. } else {
  81. fmt.Fprintf(r.output, r.template, r.readProgress, "?", "n/a")
  82. }
  83. r.lastUpdate = r.readProgress
  84. }
  85. // Send newline when complete
  86. if err != nil {
  87. fmt.Fprintf(r.output, FormatStatus("", r.json))
  88. }
  89. return read, err
  90. }
  91. func (r *progressReader) Close() error {
  92. return io.ReadCloser(r.reader).Close()
  93. }
  94. func ProgressReader(r io.ReadCloser, size int, output io.Writer, template string, json bool) *progressReader {
  95. if template == "" {
  96. template = "%v/%v (%v)\r"
  97. }
  98. return &progressReader{r, NewWriteFlusher(output), size, 0, 0, template, json}
  99. }
  100. // HumanDuration returns a human-readable approximation of a duration
  101. // (eg. "About a minute", "4 hours ago", etc.)
  102. func HumanDuration(d time.Duration) string {
  103. if seconds := int(d.Seconds()); seconds < 1 {
  104. return "Less than a second"
  105. } else if seconds < 60 {
  106. return fmt.Sprintf("%d seconds", seconds)
  107. } else if minutes := int(d.Minutes()); minutes == 1 {
  108. return "About a minute"
  109. } else if minutes < 60 {
  110. return fmt.Sprintf("%d minutes", minutes)
  111. } else if hours := int(d.Hours()); hours == 1 {
  112. return "About an hour"
  113. } else if hours < 48 {
  114. return fmt.Sprintf("%d hours", hours)
  115. } else if hours < 24*7*2 {
  116. return fmt.Sprintf("%d days", hours/24)
  117. } else if hours < 24*30*3 {
  118. return fmt.Sprintf("%d weeks", hours/24/7)
  119. } else if hours < 24*365*2 {
  120. return fmt.Sprintf("%d months", hours/24/30)
  121. }
  122. return fmt.Sprintf("%d years", d.Hours()/24/365)
  123. }
  124. func Trunc(s string, maxlen int) string {
  125. if len(s) <= maxlen {
  126. return s
  127. }
  128. return s[:maxlen]
  129. }
  130. // Figure out the absolute path of our own binary
  131. func SelfPath() string {
  132. path, err := exec.LookPath(os.Args[0])
  133. if err != nil {
  134. panic(err)
  135. }
  136. path, err = filepath.Abs(path)
  137. if err != nil {
  138. panic(err)
  139. }
  140. return path
  141. }
  142. type NopWriter struct {
  143. }
  144. func (w *NopWriter) Write(buf []byte) (int, error) {
  145. return len(buf), nil
  146. }
  147. type nopWriteCloser struct {
  148. io.Writer
  149. }
  150. func (w *nopWriteCloser) Close() error { return nil }
  151. func NopWriteCloser(w io.Writer) io.WriteCloser {
  152. return &nopWriteCloser{w}
  153. }
  154. type bufReader struct {
  155. buf *bytes.Buffer
  156. reader io.Reader
  157. err error
  158. l sync.Mutex
  159. wait sync.Cond
  160. }
  161. func NewBufReader(r io.Reader) *bufReader {
  162. reader := &bufReader{
  163. buf: &bytes.Buffer{},
  164. reader: r,
  165. }
  166. reader.wait.L = &reader.l
  167. go reader.drain()
  168. return reader
  169. }
  170. func (r *bufReader) drain() {
  171. buf := make([]byte, 1024)
  172. for {
  173. n, err := r.reader.Read(buf)
  174. r.l.Lock()
  175. if err != nil {
  176. r.err = err
  177. } else {
  178. r.buf.Write(buf[0:n])
  179. }
  180. r.wait.Signal()
  181. r.l.Unlock()
  182. if err != nil {
  183. break
  184. }
  185. }
  186. }
  187. func (r *bufReader) Read(p []byte) (n int, err error) {
  188. r.l.Lock()
  189. defer r.l.Unlock()
  190. for {
  191. n, err = r.buf.Read(p)
  192. if n > 0 {
  193. return n, err
  194. }
  195. if r.err != nil {
  196. return 0, r.err
  197. }
  198. r.wait.Wait()
  199. }
  200. panic("unreachable")
  201. }
  202. func (r *bufReader) Close() error {
  203. closer, ok := r.reader.(io.ReadCloser)
  204. if !ok {
  205. return nil
  206. }
  207. return closer.Close()
  208. }
  209. type WriteBroadcaster struct {
  210. mu sync.Mutex
  211. writers map[io.WriteCloser]struct{}
  212. }
  213. func (w *WriteBroadcaster) AddWriter(writer io.WriteCloser) {
  214. w.mu.Lock()
  215. w.writers[writer] = struct{}{}
  216. w.mu.Unlock()
  217. }
  218. // FIXME: Is that function used?
  219. // FIXME: This relies on the concrete writer type used having equality operator
  220. func (w *WriteBroadcaster) RemoveWriter(writer io.WriteCloser) {
  221. w.mu.Lock()
  222. delete(w.writers, writer)
  223. w.mu.Unlock()
  224. }
  225. func (w *WriteBroadcaster) Write(p []byte) (n int, err error) {
  226. w.mu.Lock()
  227. defer w.mu.Unlock()
  228. for writer := range w.writers {
  229. if n, err := writer.Write(p); err != nil || n != len(p) {
  230. // On error, evict the writer
  231. delete(w.writers, writer)
  232. }
  233. }
  234. return len(p), nil
  235. }
  236. func (w *WriteBroadcaster) CloseWriters() error {
  237. w.mu.Lock()
  238. defer w.mu.Unlock()
  239. for writer := range w.writers {
  240. writer.Close()
  241. }
  242. w.writers = make(map[io.WriteCloser]struct{})
  243. return nil
  244. }
  245. func NewWriteBroadcaster() *WriteBroadcaster {
  246. return &WriteBroadcaster{writers: make(map[io.WriteCloser]struct{})}
  247. }
  248. func GetTotalUsedFds() int {
  249. if fds, err := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid())); err != nil {
  250. Debugf("Error opening /proc/%d/fd: %s", os.Getpid(), err)
  251. } else {
  252. return len(fds)
  253. }
  254. return -1
  255. }
  256. // TruncIndex allows the retrieval of string identifiers by any of their unique prefixes.
  257. // This is used to retrieve image and container IDs by more convenient shorthand prefixes.
  258. type TruncIndex struct {
  259. index *suffixarray.Index
  260. ids map[string]bool
  261. bytes []byte
  262. }
  263. func NewTruncIndex() *TruncIndex {
  264. return &TruncIndex{
  265. index: suffixarray.New([]byte{' '}),
  266. ids: make(map[string]bool),
  267. bytes: []byte{' '},
  268. }
  269. }
  270. func (idx *TruncIndex) Add(id string) error {
  271. if strings.Contains(id, " ") {
  272. return fmt.Errorf("Illegal character: ' '")
  273. }
  274. if _, exists := idx.ids[id]; exists {
  275. return fmt.Errorf("Id already exists: %s", id)
  276. }
  277. idx.ids[id] = true
  278. idx.bytes = append(idx.bytes, []byte(id+" ")...)
  279. idx.index = suffixarray.New(idx.bytes)
  280. return nil
  281. }
  282. func (idx *TruncIndex) Delete(id string) error {
  283. if _, exists := idx.ids[id]; !exists {
  284. return fmt.Errorf("No such id: %s", id)
  285. }
  286. before, after, err := idx.lookup(id)
  287. if err != nil {
  288. return err
  289. }
  290. delete(idx.ids, id)
  291. idx.bytes = append(idx.bytes[:before], idx.bytes[after:]...)
  292. idx.index = suffixarray.New(idx.bytes)
  293. return nil
  294. }
  295. func (idx *TruncIndex) lookup(s string) (int, int, error) {
  296. offsets := idx.index.Lookup([]byte(" "+s), -1)
  297. //log.Printf("lookup(%s): %v (index bytes: '%s')\n", s, offsets, idx.index.Bytes())
  298. if offsets == nil || len(offsets) == 0 || len(offsets) > 1 {
  299. return -1, -1, fmt.Errorf("No such id: %s", s)
  300. }
  301. offsetBefore := offsets[0] + 1
  302. offsetAfter := offsetBefore + strings.Index(string(idx.bytes[offsetBefore:]), " ")
  303. return offsetBefore, offsetAfter, nil
  304. }
  305. func (idx *TruncIndex) Get(s string) (string, error) {
  306. before, after, err := idx.lookup(s)
  307. //log.Printf("Get(%s) bytes=|%s| before=|%d| after=|%d|\n", s, idx.bytes, before, after)
  308. if err != nil {
  309. return "", err
  310. }
  311. return string(idx.bytes[before:after]), err
  312. }
  313. // TruncateId returns a shorthand version of a string identifier for convenience.
  314. // A collision with other shorthands is very unlikely, but possible.
  315. // In case of a collision a lookup with TruncIndex.Get() will fail, and the caller
  316. // will need to use a langer prefix, or the full-length Id.
  317. func TruncateId(id string) string {
  318. shortLen := 12
  319. if len(id) < shortLen {
  320. shortLen = len(id)
  321. }
  322. return id[:shortLen]
  323. }
  324. // Code c/c from io.Copy() modified to handle escape sequence
  325. func CopyEscapable(dst io.Writer, src io.ReadCloser) (written int64, err error) {
  326. buf := make([]byte, 32*1024)
  327. for {
  328. nr, er := src.Read(buf)
  329. if nr > 0 {
  330. // ---- Docker addition
  331. // char 16 is C-p
  332. if nr == 1 && buf[0] == 16 {
  333. nr, er = src.Read(buf)
  334. // char 17 is C-q
  335. if nr == 1 && buf[0] == 17 {
  336. if err := src.Close(); err != nil {
  337. return 0, err
  338. }
  339. return 0, io.EOF
  340. }
  341. }
  342. // ---- End of docker
  343. nw, ew := dst.Write(buf[0:nr])
  344. if nw > 0 {
  345. written += int64(nw)
  346. }
  347. if ew != nil {
  348. err = ew
  349. break
  350. }
  351. if nr != nw {
  352. err = io.ErrShortWrite
  353. break
  354. }
  355. }
  356. if er == io.EOF {
  357. break
  358. }
  359. if er != nil {
  360. err = er
  361. break
  362. }
  363. }
  364. return written, err
  365. }
  366. func HashData(src io.Reader) (string, error) {
  367. h := sha256.New()
  368. if _, err := io.Copy(h, src); err != nil {
  369. return "", err
  370. }
  371. return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil
  372. }
  373. type KernelVersionInfo struct {
  374. Kernel int
  375. Major int
  376. Minor int
  377. Flavor string
  378. }
  379. func (k *KernelVersionInfo) String() string {
  380. flavor := ""
  381. if len(k.Flavor) > 0 {
  382. flavor = fmt.Sprintf("-%s", k.Flavor)
  383. }
  384. return fmt.Sprintf("%d.%d.%d%s", k.Kernel, k.Major, k.Minor, flavor)
  385. }
  386. // Compare two KernelVersionInfo struct.
  387. // Returns -1 if a < b, = if a == b, 1 it a > b
  388. func CompareKernelVersion(a, b *KernelVersionInfo) int {
  389. if a.Kernel < b.Kernel {
  390. return -1
  391. } else if a.Kernel > b.Kernel {
  392. return 1
  393. }
  394. if a.Major < b.Major {
  395. return -1
  396. } else if a.Major > b.Major {
  397. return 1
  398. }
  399. if a.Minor < b.Minor {
  400. return -1
  401. } else if a.Minor > b.Minor {
  402. return 1
  403. }
  404. return 0
  405. }
  406. func FindCgroupMountpoint(cgroupType string) (string, error) {
  407. output, err := ioutil.ReadFile("/proc/mounts")
  408. if err != nil {
  409. return "", err
  410. }
  411. // /proc/mounts has 6 fields per line, one mount per line, e.g.
  412. // cgroup /sys/fs/cgroup/devices cgroup rw,relatime,devices 0 0
  413. for _, line := range strings.Split(string(output), "\n") {
  414. parts := strings.Split(line, " ")
  415. if len(parts) == 6 && parts[2] == "cgroup" {
  416. for _, opt := range strings.Split(parts[3], ",") {
  417. if opt == cgroupType {
  418. return parts[1], nil
  419. }
  420. }
  421. }
  422. }
  423. return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType)
  424. }
  425. func GetKernelVersion() (*KernelVersionInfo, error) {
  426. var (
  427. flavor string
  428. kernel, major, minor int
  429. err error
  430. )
  431. uts, err := uname()
  432. if err != nil {
  433. return nil, err
  434. }
  435. release := make([]byte, len(uts.Release))
  436. i := 0
  437. for _, c := range uts.Release {
  438. release[i] = byte(c)
  439. i++
  440. }
  441. // Remove the \x00 from the release for Atoi to parse correctly
  442. release = release[:bytes.IndexByte(release, 0)]
  443. tmp := strings.SplitN(string(release), "-", 2)
  444. tmp2 := strings.SplitN(tmp[0], ".", 3)
  445. if len(tmp2) > 0 {
  446. kernel, err = strconv.Atoi(tmp2[0])
  447. if err != nil {
  448. return nil, err
  449. }
  450. }
  451. if len(tmp2) > 1 {
  452. major, err = strconv.Atoi(tmp2[1])
  453. if err != nil {
  454. return nil, err
  455. }
  456. }
  457. if len(tmp2) > 2 {
  458. minor, err = strconv.Atoi(tmp2[2])
  459. if err != nil {
  460. return nil, err
  461. }
  462. }
  463. if len(tmp) == 2 {
  464. flavor = tmp[1]
  465. } else {
  466. flavor = ""
  467. }
  468. return &KernelVersionInfo{
  469. Kernel: kernel,
  470. Major: major,
  471. Minor: minor,
  472. Flavor: flavor,
  473. }, nil
  474. }
  475. func CopyDirectory(source, dest string) error {
  476. if output, err := exec.Command("cp", "-ra", source, dest).CombinedOutput(); err != nil {
  477. return fmt.Errorf("Error copy: %s (%s)", err, output)
  478. }
  479. return nil
  480. }
  481. type NopFlusher struct{}
  482. func (f *NopFlusher) Flush() {}
  483. type WriteFlusher struct {
  484. w io.Writer
  485. flusher http.Flusher
  486. }
  487. func (wf *WriteFlusher) Write(b []byte) (n int, err error) {
  488. n, err = wf.w.Write(b)
  489. wf.flusher.Flush()
  490. return n, err
  491. }
  492. func NewWriteFlusher(w io.Writer) *WriteFlusher {
  493. var flusher http.Flusher
  494. if f, ok := w.(http.Flusher); ok {
  495. flusher = f
  496. } else {
  497. flusher = &NopFlusher{}
  498. }
  499. return &WriteFlusher{w: w, flusher: flusher}
  500. }
  501. func FormatStatus(str string, json bool) string {
  502. if json {
  503. return "{\"status\" : \"" + str + "\"}"
  504. }
  505. return str + "\r\n"
  506. }
  507. func FormatProgress(str string, json bool) string {
  508. if json {
  509. return "{\"progress\" : \"" + str + "\"}"
  510. }
  511. return "Downloading " + str + "\r"
  512. }