internals.go 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575
  1. package dockerfile
  2. // internals for handling commands. Covers many areas and a lot of
  3. // non-contiguous functionality. Please read the comments.
  4. import (
  5. "crypto/sha256"
  6. "encoding/hex"
  7. "fmt"
  8. "io"
  9. "os"
  10. "path"
  11. "path/filepath"
  12. "runtime"
  13. "strconv"
  14. "strings"
  15. "github.com/docker/docker/api/types"
  16. "github.com/docker/docker/api/types/backend"
  17. "github.com/docker/docker/api/types/container"
  18. "github.com/docker/docker/image"
  19. "github.com/docker/docker/pkg/archive"
  20. "github.com/docker/docker/pkg/chrootarchive"
  21. "github.com/docker/docker/pkg/containerfs"
  22. "github.com/docker/docker/pkg/idtools"
  23. "github.com/docker/docker/pkg/stringid"
  24. "github.com/docker/docker/pkg/symlink"
  25. "github.com/docker/docker/pkg/system"
  26. "github.com/docker/go-connections/nat"
  27. lcUser "github.com/opencontainers/runc/libcontainer/user"
  28. "github.com/pkg/errors"
  29. )
  30. // Archiver defines an interface for copying files from one destination to
  31. // another using Tar/Untar.
  32. type Archiver interface {
  33. TarUntar(src, dst string) error
  34. UntarPath(src, dst string) error
  35. CopyWithTar(src, dst string) error
  36. CopyFileWithTar(src, dst string) error
  37. IDMappings() *idtools.IDMappings
  38. }
  39. // The builder will use the following interfaces if the container fs implements
  40. // these for optimized copies to and from the container.
  41. type extractor interface {
  42. ExtractArchive(src io.Reader, dst string, opts *archive.TarOptions) error
  43. }
  44. type archiver interface {
  45. ArchivePath(src string, opts *archive.TarOptions) (io.ReadCloser, error)
  46. }
  47. // helper functions to get tar/untar func
  48. func untarFunc(i interface{}) containerfs.UntarFunc {
  49. if ea, ok := i.(extractor); ok {
  50. return ea.ExtractArchive
  51. }
  52. return chrootarchive.Untar
  53. }
  54. func tarFunc(i interface{}) containerfs.TarFunc {
  55. if ap, ok := i.(archiver); ok {
  56. return ap.ArchivePath
  57. }
  58. return archive.TarWithOptions
  59. }
  60. func (b *Builder) getArchiver(src, dst containerfs.Driver) Archiver {
  61. t, u := tarFunc(src), untarFunc(dst)
  62. return &containerfs.Archiver{
  63. SrcDriver: src,
  64. DstDriver: dst,
  65. Tar: t,
  66. Untar: u,
  67. IDMappingsVar: b.idMappings,
  68. }
  69. }
  70. func (b *Builder) commit(dispatchState *dispatchState, comment string) error {
  71. if b.disableCommit {
  72. return nil
  73. }
  74. if !dispatchState.hasFromImage() {
  75. return errors.New("Please provide a source image with `from` prior to commit")
  76. }
  77. optionsPlatform := system.ParsePlatform(b.options.Platform)
  78. runConfigWithCommentCmd := copyRunConfig(dispatchState.runConfig, withCmdComment(comment, optionsPlatform.OS))
  79. hit, err := b.probeCache(dispatchState, runConfigWithCommentCmd)
  80. if err != nil || hit {
  81. return err
  82. }
  83. id, err := b.create(runConfigWithCommentCmd)
  84. if err != nil {
  85. return err
  86. }
  87. return b.commitContainer(dispatchState, id, runConfigWithCommentCmd)
  88. }
  89. func (b *Builder) commitContainer(dispatchState *dispatchState, id string, containerConfig *container.Config) error {
  90. if b.disableCommit {
  91. return nil
  92. }
  93. commitCfg := &backend.ContainerCommitConfig{
  94. ContainerCommitConfig: types.ContainerCommitConfig{
  95. Author: dispatchState.maintainer,
  96. Pause: true,
  97. // TODO: this should be done by Commit()
  98. Config: copyRunConfig(dispatchState.runConfig),
  99. },
  100. ContainerConfig: containerConfig,
  101. }
  102. // Commit the container
  103. imageID, err := b.docker.Commit(id, commitCfg)
  104. if err != nil {
  105. return err
  106. }
  107. dispatchState.imageID = imageID
  108. return nil
  109. }
  110. func (b *Builder) exportImage(state *dispatchState, imageMount *imageMount, runConfig *container.Config) error {
  111. optionsPlatform := system.ParsePlatform(b.options.Platform)
  112. newLayer, err := imageMount.Layer().Commit(optionsPlatform.OS)
  113. if err != nil {
  114. return err
  115. }
  116. // add an image mount without an image so the layer is properly unmounted
  117. // if there is an error before we can add the full mount with image
  118. b.imageSources.Add(newImageMount(nil, newLayer))
  119. parentImage, ok := imageMount.Image().(*image.Image)
  120. if !ok {
  121. return errors.Errorf("unexpected image type")
  122. }
  123. newImage := image.NewChildImage(parentImage, image.ChildConfig{
  124. Author: state.maintainer,
  125. ContainerConfig: runConfig,
  126. DiffID: newLayer.DiffID(),
  127. Config: copyRunConfig(state.runConfig),
  128. }, parentImage.OS)
  129. // TODO: it seems strange to marshal this here instead of just passing in the
  130. // image struct
  131. config, err := newImage.MarshalJSON()
  132. if err != nil {
  133. return errors.Wrap(err, "failed to encode image config")
  134. }
  135. exportedImage, err := b.docker.CreateImage(config, state.imageID, parentImage.OS)
  136. if err != nil {
  137. return errors.Wrapf(err, "failed to export image")
  138. }
  139. state.imageID = exportedImage.ImageID()
  140. b.imageSources.Add(newImageMount(exportedImage, newLayer))
  141. return nil
  142. }
  143. func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error {
  144. srcHash := getSourceHashFromInfos(inst.infos)
  145. var chownComment string
  146. if inst.chownStr != "" {
  147. chownComment = fmt.Sprintf("--chown=%s", inst.chownStr)
  148. }
  149. commentStr := fmt.Sprintf("%s %s%s in %s ", inst.cmdName, chownComment, srcHash, inst.dest)
  150. // TODO: should this have been using origPaths instead of srcHash in the comment?
  151. optionsPlatform := system.ParsePlatform(b.options.Platform)
  152. runConfigWithCommentCmd := copyRunConfig(
  153. state.runConfig,
  154. withCmdCommentString(commentStr, optionsPlatform.OS))
  155. hit, err := b.probeCache(state, runConfigWithCommentCmd)
  156. if err != nil || hit {
  157. return err
  158. }
  159. imageMount, err := b.imageSources.Get(state.imageID, true)
  160. if err != nil {
  161. return errors.Wrapf(err, "failed to get destination image %q", state.imageID)
  162. }
  163. destInfo, err := createDestInfo(state.runConfig.WorkingDir, inst, imageMount, b.options.Platform)
  164. if err != nil {
  165. return err
  166. }
  167. chownPair := b.idMappings.RootPair()
  168. // if a chown was requested, perform the steps to get the uid, gid
  169. // translated (if necessary because of user namespaces), and replace
  170. // the root pair with the chown pair for copy operations
  171. if inst.chownStr != "" {
  172. chownPair, err = parseChownFlag(inst.chownStr, destInfo.root.Path(), b.idMappings)
  173. if err != nil {
  174. return errors.Wrapf(err, "unable to convert uid/gid chown string to host mapping")
  175. }
  176. }
  177. for _, info := range inst.infos {
  178. opts := copyFileOptions{
  179. decompress: inst.allowLocalDecompression,
  180. archiver: b.getArchiver(info.root, destInfo.root),
  181. chownPair: chownPair,
  182. }
  183. if err := performCopyForInfo(destInfo, info, opts); err != nil {
  184. return errors.Wrapf(err, "failed to copy files")
  185. }
  186. }
  187. return b.exportImage(state, imageMount, runConfigWithCommentCmd)
  188. }
  189. func parseChownFlag(chown, ctrRootPath string, idMappings *idtools.IDMappings) (idtools.IDPair, error) {
  190. var userStr, grpStr string
  191. parts := strings.Split(chown, ":")
  192. if len(parts) > 2 {
  193. return idtools.IDPair{}, errors.New("invalid chown string format: " + chown)
  194. }
  195. if len(parts) == 1 {
  196. // if no group specified, use the user spec as group as well
  197. userStr, grpStr = parts[0], parts[0]
  198. } else {
  199. userStr, grpStr = parts[0], parts[1]
  200. }
  201. passwdPath, err := symlink.FollowSymlinkInScope(filepath.Join(ctrRootPath, "etc", "passwd"), ctrRootPath)
  202. if err != nil {
  203. return idtools.IDPair{}, errors.Wrapf(err, "can't resolve /etc/passwd path in container rootfs")
  204. }
  205. groupPath, err := symlink.FollowSymlinkInScope(filepath.Join(ctrRootPath, "etc", "group"), ctrRootPath)
  206. if err != nil {
  207. return idtools.IDPair{}, errors.Wrapf(err, "can't resolve /etc/group path in container rootfs")
  208. }
  209. uid, err := lookupUser(userStr, passwdPath)
  210. if err != nil {
  211. return idtools.IDPair{}, errors.Wrapf(err, "can't find uid for user "+userStr)
  212. }
  213. gid, err := lookupGroup(grpStr, groupPath)
  214. if err != nil {
  215. return idtools.IDPair{}, errors.Wrapf(err, "can't find gid for group "+grpStr)
  216. }
  217. // convert as necessary because of user namespaces
  218. chownPair, err := idMappings.ToHost(idtools.IDPair{UID: uid, GID: gid})
  219. if err != nil {
  220. return idtools.IDPair{}, errors.Wrapf(err, "unable to convert uid/gid to host mapping")
  221. }
  222. return chownPair, nil
  223. }
  224. func lookupUser(userStr, filepath string) (int, error) {
  225. // if the string is actually a uid integer, parse to int and return
  226. // as we don't need to translate with the help of files
  227. uid, err := strconv.Atoi(userStr)
  228. if err == nil {
  229. return uid, nil
  230. }
  231. users, err := lcUser.ParsePasswdFileFilter(filepath, func(u lcUser.User) bool {
  232. return u.Name == userStr
  233. })
  234. if err != nil {
  235. return 0, err
  236. }
  237. if len(users) == 0 {
  238. return 0, errors.New("no such user: " + userStr)
  239. }
  240. return users[0].Uid, nil
  241. }
  242. func lookupGroup(groupStr, filepath string) (int, error) {
  243. // if the string is actually a gid integer, parse to int and return
  244. // as we don't need to translate with the help of files
  245. gid, err := strconv.Atoi(groupStr)
  246. if err == nil {
  247. return gid, nil
  248. }
  249. groups, err := lcUser.ParseGroupFileFilter(filepath, func(g lcUser.Group) bool {
  250. return g.Name == groupStr
  251. })
  252. if err != nil {
  253. return 0, err
  254. }
  255. if len(groups) == 0 {
  256. return 0, errors.New("no such group: " + groupStr)
  257. }
  258. return groups[0].Gid, nil
  259. }
  260. func createDestInfo(workingDir string, inst copyInstruction, imageMount *imageMount, platform string) (copyInfo, error) {
  261. // Twiddle the destination when it's a relative path - meaning, make it
  262. // relative to the WORKINGDIR
  263. dest, err := normalizeDest(workingDir, inst.dest, platform)
  264. if err != nil {
  265. return copyInfo{}, errors.Wrapf(err, "invalid %s", inst.cmdName)
  266. }
  267. destMount, err := imageMount.Source()
  268. if err != nil {
  269. return copyInfo{}, errors.Wrapf(err, "failed to mount copy source")
  270. }
  271. return newCopyInfoFromSource(destMount, dest, ""), nil
  272. }
  273. // normalizeDest normalises the destination of a COPY/ADD command in a
  274. // platform semantically consistent way.
  275. func normalizeDest(workingDir, requested string, platform string) (string, error) {
  276. dest := fromSlash(requested, platform)
  277. endsInSlash := strings.HasSuffix(dest, string(separator(platform)))
  278. if platform != "windows" {
  279. if !path.IsAbs(requested) {
  280. dest = path.Join("/", filepath.ToSlash(workingDir), dest)
  281. // Make sure we preserve any trailing slash
  282. if endsInSlash {
  283. dest += "/"
  284. }
  285. }
  286. return dest, nil
  287. }
  288. // We are guaranteed that the working directory is already consistent,
  289. // However, Windows also has, for now, the limitation that ADD/COPY can
  290. // only be done to the system drive, not any drives that might be present
  291. // as a result of a bind mount.
  292. //
  293. // So... if the path requested is Linux-style absolute (/foo or \\foo),
  294. // we assume it is the system drive. If it is a Windows-style absolute
  295. // (DRIVE:\\foo), error if DRIVE is not C. And finally, ensure we
  296. // strip any configured working directories drive letter so that it
  297. // can be subsequently legitimately converted to a Windows volume-style
  298. // pathname.
  299. // Not a typo - filepath.IsAbs, not system.IsAbs on this next check as
  300. // we only want to validate where the DriveColon part has been supplied.
  301. if filepath.IsAbs(dest) {
  302. if strings.ToUpper(string(dest[0])) != "C" {
  303. return "", fmt.Errorf("Windows does not support destinations not on the system drive (C:)")
  304. }
  305. dest = dest[2:] // Strip the drive letter
  306. }
  307. // Cannot handle relative where WorkingDir is not the system drive.
  308. if len(workingDir) > 0 {
  309. if ((len(workingDir) > 1) && !system.IsAbs(workingDir[2:])) || (len(workingDir) == 1) {
  310. return "", fmt.Errorf("Current WorkingDir %s is not platform consistent", workingDir)
  311. }
  312. if !system.IsAbs(dest) {
  313. if string(workingDir[0]) != "C" {
  314. return "", fmt.Errorf("Windows does not support relative paths when WORKDIR is not the system drive")
  315. }
  316. dest = filepath.Join(string(os.PathSeparator), workingDir[2:], dest)
  317. // Make sure we preserve any trailing slash
  318. if endsInSlash {
  319. dest += string(os.PathSeparator)
  320. }
  321. }
  322. }
  323. return dest, nil
  324. }
  325. // For backwards compat, if there's just one info then use it as the
  326. // cache look-up string, otherwise hash 'em all into one
  327. func getSourceHashFromInfos(infos []copyInfo) string {
  328. if len(infos) == 1 {
  329. return infos[0].hash
  330. }
  331. var hashs []string
  332. for _, info := range infos {
  333. hashs = append(hashs, info.hash)
  334. }
  335. return hashStringSlice("multi", hashs)
  336. }
  337. func hashStringSlice(prefix string, slice []string) string {
  338. hasher := sha256.New()
  339. hasher.Write([]byte(strings.Join(slice, ",")))
  340. return prefix + ":" + hex.EncodeToString(hasher.Sum(nil))
  341. }
  342. type runConfigModifier func(*container.Config)
  343. func withCmd(cmd []string) runConfigModifier {
  344. return func(runConfig *container.Config) {
  345. runConfig.Cmd = cmd
  346. }
  347. }
  348. // withCmdComment sets Cmd to a nop comment string. See withCmdCommentString for
  349. // why there are two almost identical versions of this.
  350. func withCmdComment(comment string, platform string) runConfigModifier {
  351. return func(runConfig *container.Config) {
  352. runConfig.Cmd = append(getShell(runConfig, platform), "#(nop) ", comment)
  353. }
  354. }
  355. // withCmdCommentString exists to maintain compatibility with older versions.
  356. // A few instructions (workdir, copy, add) used a nop comment that is a single arg
  357. // where as all the other instructions used a two arg comment string. This
  358. // function implements the single arg version.
  359. func withCmdCommentString(comment string, platform string) runConfigModifier {
  360. return func(runConfig *container.Config) {
  361. runConfig.Cmd = append(getShell(runConfig, platform), "#(nop) "+comment)
  362. }
  363. }
  364. func withEnv(env []string) runConfigModifier {
  365. return func(runConfig *container.Config) {
  366. runConfig.Env = env
  367. }
  368. }
  369. // withEntrypointOverride sets an entrypoint on runConfig if the command is
  370. // not empty. The entrypoint is left unmodified if command is empty.
  371. //
  372. // The dockerfile RUN instruction expect to run without an entrypoint
  373. // so the runConfig entrypoint needs to be modified accordingly. ContainerCreate
  374. // will change a []string{""} entrypoint to nil, so we probe the cache with the
  375. // nil entrypoint.
  376. func withEntrypointOverride(cmd []string, entrypoint []string) runConfigModifier {
  377. return func(runConfig *container.Config) {
  378. if len(cmd) > 0 {
  379. runConfig.Entrypoint = entrypoint
  380. }
  381. }
  382. }
  383. func copyRunConfig(runConfig *container.Config, modifiers ...runConfigModifier) *container.Config {
  384. copy := *runConfig
  385. copy.Cmd = copyStringSlice(runConfig.Cmd)
  386. copy.Env = copyStringSlice(runConfig.Env)
  387. copy.Entrypoint = copyStringSlice(runConfig.Entrypoint)
  388. copy.OnBuild = copyStringSlice(runConfig.OnBuild)
  389. copy.Shell = copyStringSlice(runConfig.Shell)
  390. if copy.Volumes != nil {
  391. copy.Volumes = make(map[string]struct{}, len(runConfig.Volumes))
  392. for k, v := range runConfig.Volumes {
  393. copy.Volumes[k] = v
  394. }
  395. }
  396. if copy.ExposedPorts != nil {
  397. copy.ExposedPorts = make(nat.PortSet, len(runConfig.ExposedPorts))
  398. for k, v := range runConfig.ExposedPorts {
  399. copy.ExposedPorts[k] = v
  400. }
  401. }
  402. if copy.Labels != nil {
  403. copy.Labels = make(map[string]string, len(runConfig.Labels))
  404. for k, v := range runConfig.Labels {
  405. copy.Labels[k] = v
  406. }
  407. }
  408. for _, modifier := range modifiers {
  409. modifier(&copy)
  410. }
  411. return &copy
  412. }
  413. func copyStringSlice(orig []string) []string {
  414. if orig == nil {
  415. return nil
  416. }
  417. return append([]string{}, orig...)
  418. }
  419. // getShell is a helper function which gets the right shell for prefixing the
  420. // shell-form of RUN, ENTRYPOINT and CMD instructions
  421. func getShell(c *container.Config, os string) []string {
  422. if 0 == len(c.Shell) {
  423. return append([]string{}, defaultShellForOS(os)[:]...)
  424. }
  425. return append([]string{}, c.Shell[:]...)
  426. }
  427. func (b *Builder) probeCache(dispatchState *dispatchState, runConfig *container.Config) (bool, error) {
  428. cachedID, err := b.imageProber.Probe(dispatchState.imageID, runConfig)
  429. if cachedID == "" || err != nil {
  430. return false, err
  431. }
  432. fmt.Fprint(b.Stdout, " ---> Using cache\n")
  433. dispatchState.imageID = cachedID
  434. return true, nil
  435. }
  436. var defaultLogConfig = container.LogConfig{Type: "none"}
  437. func (b *Builder) probeAndCreate(dispatchState *dispatchState, runConfig *container.Config) (string, error) {
  438. if hit, err := b.probeCache(dispatchState, runConfig); err != nil || hit {
  439. return "", err
  440. }
  441. // Set a log config to override any default value set on the daemon
  442. hostConfig := &container.HostConfig{LogConfig: defaultLogConfig}
  443. optionsPlatform := system.ParsePlatform(b.options.Platform)
  444. container, err := b.containerManager.Create(runConfig, hostConfig, optionsPlatform.OS)
  445. return container.ID, err
  446. }
  447. func (b *Builder) create(runConfig *container.Config) (string, error) {
  448. hostConfig := hostConfigFromOptions(b.options)
  449. optionsPlatform := system.ParsePlatform(b.options.Platform)
  450. container, err := b.containerManager.Create(runConfig, hostConfig, optionsPlatform.OS)
  451. if err != nil {
  452. return "", err
  453. }
  454. // TODO: could this be moved into containerManager.Create() ?
  455. for _, warning := range container.Warnings {
  456. fmt.Fprintf(b.Stdout, " ---> [Warning] %s\n", warning)
  457. }
  458. fmt.Fprintf(b.Stdout, " ---> Running in %s\n", stringid.TruncateID(container.ID))
  459. return container.ID, nil
  460. }
  461. func hostConfigFromOptions(options *types.ImageBuildOptions) *container.HostConfig {
  462. resources := container.Resources{
  463. CgroupParent: options.CgroupParent,
  464. CPUShares: options.CPUShares,
  465. CPUPeriod: options.CPUPeriod,
  466. CPUQuota: options.CPUQuota,
  467. CpusetCpus: options.CPUSetCPUs,
  468. CpusetMems: options.CPUSetMems,
  469. Memory: options.Memory,
  470. MemorySwap: options.MemorySwap,
  471. Ulimits: options.Ulimits,
  472. }
  473. hc := &container.HostConfig{
  474. SecurityOpt: options.SecurityOpt,
  475. Isolation: options.Isolation,
  476. ShmSize: options.ShmSize,
  477. Resources: resources,
  478. NetworkMode: container.NetworkMode(options.NetworkMode),
  479. // Set a log config to override any default value set on the daemon
  480. LogConfig: defaultLogConfig,
  481. ExtraHosts: options.ExtraHosts,
  482. }
  483. // For WCOW, the default of 20GB hard-coded in the platform
  484. // is too small for builder scenarios where many users are
  485. // using RUN statements to install large amounts of data.
  486. // Use 127GB as that's the default size of a VHD in Hyper-V.
  487. if runtime.GOOS == "windows" && options.Platform == "windows" {
  488. hc.StorageOpt = make(map[string]string)
  489. hc.StorageOpt["size"] = "127GB"
  490. }
  491. return hc
  492. }
  493. // fromSlash works like filepath.FromSlash but with a given OS platform field
  494. func fromSlash(path, platform string) string {
  495. if platform == "windows" {
  496. return strings.Replace(path, "/", "\\", -1)
  497. }
  498. return path
  499. }
  500. // separator returns a OS path separator for the given OS platform
  501. func separator(platform string) byte {
  502. if platform == "windows" {
  503. return '\\'
  504. }
  505. return '/'
  506. }