utils.go 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537
  1. package utils
  2. import (
  3. "bufio"
  4. "bytes"
  5. "crypto/sha1"
  6. "crypto/sha256"
  7. "encoding/hex"
  8. "fmt"
  9. "io"
  10. "io/ioutil"
  11. "net/http"
  12. "os"
  13. "os/exec"
  14. "path/filepath"
  15. "regexp"
  16. "runtime"
  17. "strings"
  18. "sync"
  19. log "github.com/Sirupsen/logrus"
  20. "github.com/docker/docker/autogen/dockerversion"
  21. "github.com/docker/docker/pkg/archive"
  22. "github.com/docker/docker/pkg/common"
  23. "github.com/docker/docker/pkg/fileutils"
  24. "github.com/docker/docker/pkg/ioutils"
  25. )
  26. type KeyValuePair struct {
  27. Key string
  28. Value string
  29. }
  30. var (
  31. validHex = regexp.MustCompile(`^([a-f0-9]{64})$`)
  32. )
  33. // Request a given URL and return an io.Reader
  34. func Download(url string) (resp *http.Response, err error) {
  35. if resp, err = http.Get(url); err != nil {
  36. return nil, err
  37. }
  38. if resp.StatusCode >= 400 {
  39. return nil, fmt.Errorf("Got HTTP status code >= 400: %s", resp.Status)
  40. }
  41. return resp, nil
  42. }
  43. func Trunc(s string, maxlen int) string {
  44. if len(s) <= maxlen {
  45. return s
  46. }
  47. return s[:maxlen]
  48. }
  49. // Figure out the absolute path of our own binary (if it's still around).
  50. func SelfPath() string {
  51. path, err := exec.LookPath(os.Args[0])
  52. if err != nil {
  53. if os.IsNotExist(err) {
  54. return ""
  55. }
  56. if execErr, ok := err.(*exec.Error); ok && os.IsNotExist(execErr.Err) {
  57. return ""
  58. }
  59. panic(err)
  60. }
  61. path, err = filepath.Abs(path)
  62. if err != nil {
  63. if os.IsNotExist(err) {
  64. return ""
  65. }
  66. panic(err)
  67. }
  68. return path
  69. }
  70. func dockerInitSha1(target string) string {
  71. f, err := os.Open(target)
  72. if err != nil {
  73. return ""
  74. }
  75. defer f.Close()
  76. h := sha1.New()
  77. _, err = io.Copy(h, f)
  78. if err != nil {
  79. return ""
  80. }
  81. return hex.EncodeToString(h.Sum(nil))
  82. }
  83. func isValidDockerInitPath(target string, selfPath string) bool { // target and selfPath should be absolute (InitPath and SelfPath already do this)
  84. if target == "" {
  85. return false
  86. }
  87. if dockerversion.IAMSTATIC == "true" {
  88. if selfPath == "" {
  89. return false
  90. }
  91. if target == selfPath {
  92. return true
  93. }
  94. targetFileInfo, err := os.Lstat(target)
  95. if err != nil {
  96. return false
  97. }
  98. selfPathFileInfo, err := os.Lstat(selfPath)
  99. if err != nil {
  100. return false
  101. }
  102. return os.SameFile(targetFileInfo, selfPathFileInfo)
  103. }
  104. return dockerversion.INITSHA1 != "" && dockerInitSha1(target) == dockerversion.INITSHA1
  105. }
  106. // Figure out the path of our dockerinit (which may be SelfPath())
  107. func DockerInitPath(localCopy string) string {
  108. selfPath := SelfPath()
  109. if isValidDockerInitPath(selfPath, selfPath) {
  110. // if we're valid, don't bother checking anything else
  111. return selfPath
  112. }
  113. var possibleInits = []string{
  114. localCopy,
  115. dockerversion.INITPATH,
  116. filepath.Join(filepath.Dir(selfPath), "dockerinit"),
  117. // FHS 3.0 Draft: "/usr/libexec includes internal binaries that are not intended to be executed directly by users or shell scripts. Applications may use a single subdirectory under /usr/libexec."
  118. // http://www.linuxbase.org/betaspecs/fhs/fhs.html#usrlibexec
  119. "/usr/libexec/docker/dockerinit",
  120. "/usr/local/libexec/docker/dockerinit",
  121. // FHS 2.3: "/usr/lib includes object files, libraries, and internal binaries that are not intended to be executed directly by users or shell scripts."
  122. // http://refspecs.linuxfoundation.org/FHS_2.3/fhs-2.3.html#USRLIBLIBRARIESFORPROGRAMMINGANDPA
  123. "/usr/lib/docker/dockerinit",
  124. "/usr/local/lib/docker/dockerinit",
  125. }
  126. for _, dockerInit := range possibleInits {
  127. if dockerInit == "" {
  128. continue
  129. }
  130. path, err := exec.LookPath(dockerInit)
  131. if err == nil {
  132. path, err = filepath.Abs(path)
  133. if err != nil {
  134. // LookPath already validated that this file exists and is executable (following symlinks), so how could Abs fail?
  135. panic(err)
  136. }
  137. if isValidDockerInitPath(path, selfPath) {
  138. return path
  139. }
  140. }
  141. }
  142. return ""
  143. }
  144. func GetTotalUsedFds() int {
  145. if fds, err := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid())); err != nil {
  146. log.Errorf("Error opening /proc/%d/fd: %s", os.Getpid(), err)
  147. } else {
  148. return len(fds)
  149. }
  150. return -1
  151. }
  152. func ValidateID(id string) error {
  153. if ok := validHex.MatchString(id); !ok {
  154. err := fmt.Errorf("image ID '%s' is invalid", id)
  155. return err
  156. }
  157. return nil
  158. }
  159. // Code c/c from io.Copy() modified to handle escape sequence
  160. func CopyEscapable(dst io.Writer, src io.ReadCloser) (written int64, err error) {
  161. buf := make([]byte, 32*1024)
  162. for {
  163. nr, er := src.Read(buf)
  164. if nr > 0 {
  165. // ---- Docker addition
  166. // char 16 is C-p
  167. if nr == 1 && buf[0] == 16 {
  168. nr, er = src.Read(buf)
  169. // char 17 is C-q
  170. if nr == 1 && buf[0] == 17 {
  171. if err := src.Close(); err != nil {
  172. return 0, err
  173. }
  174. return 0, nil
  175. }
  176. }
  177. // ---- End of docker
  178. nw, ew := dst.Write(buf[0:nr])
  179. if nw > 0 {
  180. written += int64(nw)
  181. }
  182. if ew != nil {
  183. err = ew
  184. break
  185. }
  186. if nr != nw {
  187. err = io.ErrShortWrite
  188. break
  189. }
  190. }
  191. if er == io.EOF {
  192. break
  193. }
  194. if er != nil {
  195. err = er
  196. break
  197. }
  198. }
  199. return written, err
  200. }
  201. func HashData(src io.Reader) (string, error) {
  202. h := sha256.New()
  203. if _, err := io.Copy(h, src); err != nil {
  204. return "", err
  205. }
  206. return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil
  207. }
  208. type WriteFlusher struct {
  209. sync.Mutex
  210. w io.Writer
  211. flusher http.Flusher
  212. }
  213. func (wf *WriteFlusher) Write(b []byte) (n int, err error) {
  214. wf.Lock()
  215. defer wf.Unlock()
  216. n, err = wf.w.Write(b)
  217. wf.flusher.Flush()
  218. return n, err
  219. }
  220. // Flush the stream immediately.
  221. func (wf *WriteFlusher) Flush() {
  222. wf.Lock()
  223. defer wf.Unlock()
  224. wf.flusher.Flush()
  225. }
  226. func NewWriteFlusher(w io.Writer) *WriteFlusher {
  227. var flusher http.Flusher
  228. if f, ok := w.(http.Flusher); ok {
  229. flusher = f
  230. } else {
  231. flusher = &ioutils.NopFlusher{}
  232. }
  233. return &WriteFlusher{w: w, flusher: flusher}
  234. }
  235. func NewHTTPRequestError(msg string, res *http.Response) error {
  236. return &JSONError{
  237. Message: msg,
  238. Code: res.StatusCode,
  239. }
  240. }
  241. // An StatusError reports an unsuccessful exit by a command.
  242. type StatusError struct {
  243. Status string
  244. StatusCode int
  245. }
  246. func (e *StatusError) Error() string {
  247. return fmt.Sprintf("Status: %s, Code: %d", e.Status, e.StatusCode)
  248. }
  249. func quote(word string, buf *bytes.Buffer) {
  250. // Bail out early for "simple" strings
  251. if word != "" && !strings.ContainsAny(word, "\\'\"`${[|&;<>()~*?! \t\n") {
  252. buf.WriteString(word)
  253. return
  254. }
  255. buf.WriteString("'")
  256. for i := 0; i < len(word); i++ {
  257. b := word[i]
  258. if b == '\'' {
  259. // Replace literal ' with a close ', a \', and a open '
  260. buf.WriteString("'\\''")
  261. } else {
  262. buf.WriteByte(b)
  263. }
  264. }
  265. buf.WriteString("'")
  266. }
  267. // Take a list of strings and escape them so they will be handled right
  268. // when passed as arguments to an program via a shell
  269. func ShellQuoteArguments(args []string) string {
  270. var buf bytes.Buffer
  271. for i, arg := range args {
  272. if i != 0 {
  273. buf.WriteByte(' ')
  274. }
  275. quote(arg, &buf)
  276. }
  277. return buf.String()
  278. }
  279. var globalTestID string
  280. // TestDirectory creates a new temporary directory and returns its path.
  281. // The contents of directory at path `templateDir` is copied into the
  282. // new directory.
  283. func TestDirectory(templateDir string) (dir string, err error) {
  284. if globalTestID == "" {
  285. globalTestID = common.RandomString()[:4]
  286. }
  287. prefix := fmt.Sprintf("docker-test%s-%s-", globalTestID, GetCallerName(2))
  288. if prefix == "" {
  289. prefix = "docker-test-"
  290. }
  291. dir, err = ioutil.TempDir("", prefix)
  292. if err = os.Remove(dir); err != nil {
  293. return
  294. }
  295. if templateDir != "" {
  296. if err = archive.CopyWithTar(templateDir, dir); err != nil {
  297. return
  298. }
  299. }
  300. return
  301. }
  302. // GetCallerName introspects the call stack and returns the name of the
  303. // function `depth` levels down in the stack.
  304. func GetCallerName(depth int) string {
  305. // Use the caller function name as a prefix.
  306. // This helps trace temp directories back to their test.
  307. pc, _, _, _ := runtime.Caller(depth + 1)
  308. callerLongName := runtime.FuncForPC(pc).Name()
  309. parts := strings.Split(callerLongName, ".")
  310. callerShortName := parts[len(parts)-1]
  311. return callerShortName
  312. }
  313. func CopyFile(src, dst string) (int64, error) {
  314. if src == dst {
  315. return 0, nil
  316. }
  317. sf, err := os.Open(src)
  318. if err != nil {
  319. return 0, err
  320. }
  321. defer sf.Close()
  322. if err := os.Remove(dst); err != nil && !os.IsNotExist(err) {
  323. return 0, err
  324. }
  325. df, err := os.Create(dst)
  326. if err != nil {
  327. return 0, err
  328. }
  329. defer df.Close()
  330. return io.Copy(df, sf)
  331. }
  332. // ReplaceOrAppendValues returns the defaults with the overrides either
  333. // replaced by env key or appended to the list
  334. func ReplaceOrAppendEnvValues(defaults, overrides []string) []string {
  335. cache := make(map[string]int, len(defaults))
  336. for i, e := range defaults {
  337. parts := strings.SplitN(e, "=", 2)
  338. cache[parts[0]] = i
  339. }
  340. for _, value := range overrides {
  341. // Values w/o = means they want this env to be removed/unset.
  342. if !strings.Contains(value, "=") {
  343. if i, exists := cache[value]; exists {
  344. defaults[i] = "" // Used to indicate it should be removed
  345. }
  346. continue
  347. }
  348. // Just do a normal set/update
  349. parts := strings.SplitN(value, "=", 2)
  350. if i, exists := cache[parts[0]]; exists {
  351. defaults[i] = value
  352. } else {
  353. defaults = append(defaults, value)
  354. }
  355. }
  356. // Now remove all entries that we want to "unset"
  357. for i := 0; i < len(defaults); i++ {
  358. if defaults[i] == "" {
  359. defaults = append(defaults[:i], defaults[i+1:]...)
  360. i--
  361. }
  362. }
  363. return defaults
  364. }
  365. func DoesEnvExist(name string) bool {
  366. for _, entry := range os.Environ() {
  367. parts := strings.SplitN(entry, "=", 2)
  368. if parts[0] == name {
  369. return true
  370. }
  371. }
  372. return false
  373. }
  374. // ReadSymlinkedDirectory returns the target directory of a symlink.
  375. // The target of the symbolic link may not be a file.
  376. func ReadSymlinkedDirectory(path string) (string, error) {
  377. var realPath string
  378. var err error
  379. if realPath, err = filepath.Abs(path); err != nil {
  380. return "", fmt.Errorf("unable to get absolute path for %s: %s", path, err)
  381. }
  382. if realPath, err = filepath.EvalSymlinks(realPath); err != nil {
  383. return "", fmt.Errorf("failed to canonicalise path for %s: %s", path, err)
  384. }
  385. realPathInfo, err := os.Stat(realPath)
  386. if err != nil {
  387. return "", fmt.Errorf("failed to stat target '%s' of '%s': %s", realPath, path, err)
  388. }
  389. if !realPathInfo.Mode().IsDir() {
  390. return "", fmt.Errorf("canonical path points to a file '%s'", realPath)
  391. }
  392. return realPath, nil
  393. }
  394. // ValidateContextDirectory checks if all the contents of the directory
  395. // can be read and returns an error if some files can't be read
  396. // symlinks which point to non-existing files don't trigger an error
  397. func ValidateContextDirectory(srcPath string, excludes []string) error {
  398. return filepath.Walk(filepath.Join(srcPath, "."), func(filePath string, f os.FileInfo, err error) error {
  399. // skip this directory/file if it's not in the path, it won't get added to the context
  400. if relFilePath, err := filepath.Rel(srcPath, filePath); err != nil {
  401. return err
  402. } else if skip, err := fileutils.Matches(relFilePath, excludes); err != nil {
  403. return err
  404. } else if skip {
  405. if f.IsDir() {
  406. return filepath.SkipDir
  407. }
  408. return nil
  409. }
  410. if err != nil {
  411. if os.IsPermission(err) {
  412. return fmt.Errorf("can't stat '%s'", filePath)
  413. }
  414. if os.IsNotExist(err) {
  415. return nil
  416. }
  417. return err
  418. }
  419. // skip checking if symlinks point to non-existing files, such symlinks can be useful
  420. // also skip named pipes, because they hanging on open
  421. if f.Mode()&(os.ModeSymlink|os.ModeNamedPipe) != 0 {
  422. return nil
  423. }
  424. if !f.IsDir() {
  425. currentFile, err := os.Open(filePath)
  426. if err != nil && os.IsPermission(err) {
  427. return fmt.Errorf("no permission to read from '%s'", filePath)
  428. }
  429. currentFile.Close()
  430. }
  431. return nil
  432. })
  433. }
  434. func StringsContainsNoCase(slice []string, s string) bool {
  435. for _, ss := range slice {
  436. if strings.ToLower(s) == strings.ToLower(ss) {
  437. return true
  438. }
  439. }
  440. return false
  441. }
  442. // Reads a .dockerignore file and returns the list of file patterns
  443. // to ignore. Note this will trim whitespace from each line as well
  444. // as use GO's "clean" func to get the shortest/cleanest path for each.
  445. func ReadDockerIgnore(path string) ([]string, error) {
  446. // Note that a missing .dockerignore file isn't treated as an error
  447. reader, err := os.Open(path)
  448. if err != nil {
  449. if !os.IsNotExist(err) {
  450. return nil, fmt.Errorf("Error reading '%s': %v", path, err)
  451. }
  452. return nil, nil
  453. }
  454. defer reader.Close()
  455. scanner := bufio.NewScanner(reader)
  456. var excludes []string
  457. for scanner.Scan() {
  458. pattern := strings.TrimSpace(scanner.Text())
  459. if pattern == "" {
  460. continue
  461. }
  462. pattern = filepath.Clean(pattern)
  463. excludes = append(excludes, pattern)
  464. }
  465. if err = scanner.Err(); err != nil {
  466. return nil, fmt.Errorf("Error reading '%s': %v", path, err)
  467. }
  468. return excludes, nil
  469. }
  470. // Wrap a concrete io.Writer and hold a count of the number
  471. // of bytes written to the writer during a "session".
  472. // This can be convenient when write return is masked
  473. // (e.g., json.Encoder.Encode())
  474. type WriteCounter struct {
  475. Count int64
  476. Writer io.Writer
  477. }
  478. func NewWriteCounter(w io.Writer) *WriteCounter {
  479. return &WriteCounter{
  480. Writer: w,
  481. }
  482. }
  483. func (wc *WriteCounter) Write(p []byte) (count int, err error) {
  484. count, err = wc.Writer.Write(p)
  485. wc.Count += int64(count)
  486. return
  487. }