internals.go 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515
  1. package dockerfile // import "github.com/docker/docker/builder/dockerfile"
  2. // internals for handling commands. Covers many areas and a lot of
  3. // non-contiguous functionality. Please read the comments.
  4. import (
  5. "crypto/sha256"
  6. "encoding/hex"
  7. "fmt"
  8. "io"
  9. "os"
  10. "path"
  11. "path/filepath"
  12. "runtime"
  13. "strings"
  14. "github.com/docker/docker/api/types"
  15. "github.com/docker/docker/api/types/backend"
  16. "github.com/docker/docker/api/types/container"
  17. "github.com/docker/docker/builder"
  18. "github.com/docker/docker/image"
  19. "github.com/docker/docker/pkg/archive"
  20. "github.com/docker/docker/pkg/chrootarchive"
  21. "github.com/docker/docker/pkg/containerfs"
  22. "github.com/docker/docker/pkg/idtools"
  23. "github.com/docker/docker/pkg/stringid"
  24. "github.com/docker/docker/pkg/system"
  25. "github.com/docker/go-connections/nat"
  26. specs "github.com/opencontainers/image-spec/specs-go/v1"
  27. "github.com/pkg/errors"
  28. "github.com/sirupsen/logrus"
  29. )
  30. // Archiver defines an interface for copying files from one destination to
  31. // another using Tar/Untar.
  32. type Archiver interface {
  33. TarUntar(src, dst string) error
  34. UntarPath(src, dst string) error
  35. CopyWithTar(src, dst string) error
  36. CopyFileWithTar(src, dst string) error
  37. IdentityMapping() *idtools.IdentityMapping
  38. }
  39. // The builder will use the following interfaces if the container fs implements
  40. // these for optimized copies to and from the container.
  41. type extractor interface {
  42. ExtractArchive(src io.Reader, dst string, opts *archive.TarOptions) error
  43. }
  44. type archiver interface {
  45. ArchivePath(src string, opts *archive.TarOptions) (io.ReadCloser, error)
  46. }
  47. // helper functions to get tar/untar func
  48. func untarFunc(i interface{}) containerfs.UntarFunc {
  49. if ea, ok := i.(extractor); ok {
  50. return ea.ExtractArchive
  51. }
  52. return chrootarchive.Untar
  53. }
  54. func tarFunc(i interface{}) containerfs.TarFunc {
  55. if ap, ok := i.(archiver); ok {
  56. return ap.ArchivePath
  57. }
  58. return archive.TarWithOptions
  59. }
  60. func (b *Builder) getArchiver(src, dst containerfs.Driver) Archiver {
  61. t, u := tarFunc(src), untarFunc(dst)
  62. return &containerfs.Archiver{
  63. SrcDriver: src,
  64. DstDriver: dst,
  65. Tar: t,
  66. Untar: u,
  67. IDMapping: b.idMapping,
  68. }
  69. }
  70. func (b *Builder) commit(dispatchState *dispatchState, comment string) error {
  71. if b.disableCommit {
  72. return nil
  73. }
  74. if !dispatchState.hasFromImage() {
  75. return errors.New("Please provide a source image with `from` prior to commit")
  76. }
  77. runConfigWithCommentCmd := copyRunConfig(dispatchState.runConfig, withCmdComment(comment, dispatchState.operatingSystem))
  78. id, err := b.probeAndCreate(dispatchState, runConfigWithCommentCmd)
  79. if err != nil || id == "" {
  80. return err
  81. }
  82. return b.commitContainer(dispatchState, id, runConfigWithCommentCmd)
  83. }
  84. func (b *Builder) commitContainer(dispatchState *dispatchState, id string, containerConfig *container.Config) error {
  85. if b.disableCommit {
  86. return nil
  87. }
  88. commitCfg := backend.CommitConfig{
  89. Author: dispatchState.maintainer,
  90. // TODO: this copy should be done by Commit()
  91. Config: copyRunConfig(dispatchState.runConfig),
  92. ContainerConfig: containerConfig,
  93. ContainerID: id,
  94. }
  95. imageID, err := b.docker.CommitBuildStep(commitCfg)
  96. dispatchState.imageID = string(imageID)
  97. return err
  98. }
  99. func (b *Builder) exportImage(state *dispatchState, layer builder.RWLayer, parent builder.Image, runConfig *container.Config) error {
  100. newLayer, err := layer.Commit()
  101. if err != nil {
  102. return err
  103. }
  104. parentImage, ok := parent.(*image.Image)
  105. if !ok {
  106. return errors.Errorf("unexpected image type")
  107. }
  108. platform := &specs.Platform{
  109. OS: parentImage.OS,
  110. Architecture: parentImage.Architecture,
  111. Variant: parentImage.Variant,
  112. }
  113. // add an image mount without an image so the layer is properly unmounted
  114. // if there is an error before we can add the full mount with image
  115. b.imageSources.Add(newImageMount(nil, newLayer), platform)
  116. newImage := image.NewChildImage(parentImage, image.ChildConfig{
  117. Author: state.maintainer,
  118. ContainerConfig: runConfig,
  119. DiffID: newLayer.DiffID(),
  120. Config: copyRunConfig(state.runConfig),
  121. }, parentImage.OS)
  122. // TODO: it seems strange to marshal this here instead of just passing in the
  123. // image struct
  124. config, err := newImage.MarshalJSON()
  125. if err != nil {
  126. return errors.Wrap(err, "failed to encode image config")
  127. }
  128. exportedImage, err := b.docker.CreateImage(config, state.imageID)
  129. if err != nil {
  130. return errors.Wrapf(err, "failed to export image")
  131. }
  132. state.imageID = exportedImage.ImageID()
  133. b.imageSources.Add(newImageMount(exportedImage, newLayer), platform)
  134. return nil
  135. }
  136. func (b *Builder) performCopy(req dispatchRequest, inst copyInstruction) error {
  137. state := req.state
  138. srcHash := getSourceHashFromInfos(inst.infos)
  139. var chownComment string
  140. if inst.chownStr != "" {
  141. chownComment = fmt.Sprintf("--chown=%s", inst.chownStr)
  142. }
  143. commentStr := fmt.Sprintf("%s %s%s in %s ", inst.cmdName, chownComment, srcHash, inst.dest)
  144. // TODO: should this have been using origPaths instead of srcHash in the comment?
  145. runConfigWithCommentCmd := copyRunConfig(
  146. state.runConfig,
  147. withCmdCommentString(commentStr, state.operatingSystem))
  148. hit, err := b.probeCache(state, runConfigWithCommentCmd)
  149. if err != nil || hit {
  150. return err
  151. }
  152. imageMount, err := b.imageSources.Get(state.imageID, true, req.builder.platform)
  153. if err != nil {
  154. return errors.Wrapf(err, "failed to get destination image %q", state.imageID)
  155. }
  156. rwLayer, err := imageMount.NewRWLayer()
  157. if err != nil {
  158. return err
  159. }
  160. defer rwLayer.Release()
  161. destInfo, err := createDestInfo(state.runConfig.WorkingDir, inst, rwLayer, state.operatingSystem)
  162. if err != nil {
  163. return err
  164. }
  165. identity := b.idMapping.RootPair()
  166. // if a chown was requested, perform the steps to get the uid, gid
  167. // translated (if necessary because of user namespaces), and replace
  168. // the root pair with the chown pair for copy operations
  169. if inst.chownStr != "" {
  170. identity, err = parseChownFlag(b, state, inst.chownStr, destInfo.root.Path(), b.idMapping)
  171. if err != nil {
  172. if b.options.Platform != "windows" {
  173. return errors.Wrapf(err, "unable to convert uid/gid chown string to host mapping")
  174. }
  175. return errors.Wrapf(err, "unable to map container user account name to SID")
  176. }
  177. }
  178. for _, info := range inst.infos {
  179. opts := copyFileOptions{
  180. decompress: inst.allowLocalDecompression,
  181. archiver: b.getArchiver(info.root, destInfo.root),
  182. }
  183. if !inst.preserveOwnership {
  184. opts.identity = &identity
  185. }
  186. if err := performCopyForInfo(destInfo, info, opts); err != nil {
  187. return errors.Wrapf(err, "failed to copy files")
  188. }
  189. }
  190. return b.exportImage(state, rwLayer, imageMount.Image(), runConfigWithCommentCmd)
  191. }
  192. func createDestInfo(workingDir string, inst copyInstruction, rwLayer builder.RWLayer, platform string) (copyInfo, error) {
  193. // Twiddle the destination when it's a relative path - meaning, make it
  194. // relative to the WORKINGDIR
  195. dest, err := normalizeDest(workingDir, inst.dest, platform)
  196. if err != nil {
  197. return copyInfo{}, errors.Wrapf(err, "invalid %s", inst.cmdName)
  198. }
  199. return copyInfo{root: rwLayer.Root(), path: dest}, nil
  200. }
  201. // normalizeDest normalises the destination of a COPY/ADD command in a
  202. // platform semantically consistent way.
  203. func normalizeDest(workingDir, requested string, platform string) (string, error) {
  204. dest := fromSlash(requested, platform)
  205. endsInSlash := strings.HasSuffix(dest, string(separator(platform)))
  206. if platform != "windows" {
  207. if !path.IsAbs(requested) {
  208. dest = path.Join("/", filepath.ToSlash(workingDir), dest)
  209. // Make sure we preserve any trailing slash
  210. if endsInSlash {
  211. dest += "/"
  212. }
  213. }
  214. return dest, nil
  215. }
  216. // We are guaranteed that the working directory is already consistent,
  217. // However, Windows also has, for now, the limitation that ADD/COPY can
  218. // only be done to the system drive, not any drives that might be present
  219. // as a result of a bind mount.
  220. //
  221. // So... if the path requested is Linux-style absolute (/foo or \\foo),
  222. // we assume it is the system drive. If it is a Windows-style absolute
  223. // (DRIVE:\\foo), error if DRIVE is not C. And finally, ensure we
  224. // strip any configured working directories drive letter so that it
  225. // can be subsequently legitimately converted to a Windows volume-style
  226. // pathname.
  227. // Not a typo - filepath.IsAbs, not system.IsAbs on this next check as
  228. // we only want to validate where the DriveColon part has been supplied.
  229. if filepath.IsAbs(dest) {
  230. if strings.ToUpper(string(dest[0])) != "C" {
  231. return "", fmt.Errorf("Windows does not support destinations not on the system drive (C:)")
  232. }
  233. dest = dest[2:] // Strip the drive letter
  234. }
  235. // Cannot handle relative where WorkingDir is not the system drive.
  236. if len(workingDir) > 0 {
  237. if ((len(workingDir) > 1) && !system.IsAbs(workingDir[2:])) || (len(workingDir) == 1) {
  238. return "", fmt.Errorf("Current WorkingDir %s is not platform consistent", workingDir)
  239. }
  240. if !system.IsAbs(dest) {
  241. if string(workingDir[0]) != "C" {
  242. return "", fmt.Errorf("Windows does not support relative paths when WORKDIR is not the system drive")
  243. }
  244. dest = filepath.Join(string(os.PathSeparator), workingDir[2:], dest)
  245. // Make sure we preserve any trailing slash
  246. if endsInSlash {
  247. dest += string(os.PathSeparator)
  248. }
  249. }
  250. }
  251. return dest, nil
  252. }
  253. // For backwards compat, if there's just one info then use it as the
  254. // cache look-up string, otherwise hash 'em all into one
  255. func getSourceHashFromInfos(infos []copyInfo) string {
  256. if len(infos) == 1 {
  257. return infos[0].hash
  258. }
  259. var hashs []string
  260. for _, info := range infos {
  261. hashs = append(hashs, info.hash)
  262. }
  263. return hashStringSlice("multi", hashs)
  264. }
  265. func hashStringSlice(prefix string, slice []string) string {
  266. hasher := sha256.New()
  267. hasher.Write([]byte(strings.Join(slice, ",")))
  268. return prefix + ":" + hex.EncodeToString(hasher.Sum(nil))
  269. }
  270. type runConfigModifier func(*container.Config)
  271. func withCmd(cmd []string) runConfigModifier {
  272. return func(runConfig *container.Config) {
  273. runConfig.Cmd = cmd
  274. }
  275. }
  276. func withArgsEscaped(argsEscaped bool) runConfigModifier {
  277. return func(runConfig *container.Config) {
  278. runConfig.ArgsEscaped = argsEscaped
  279. }
  280. }
  281. // withCmdComment sets Cmd to a nop comment string. See withCmdCommentString for
  282. // why there are two almost identical versions of this.
  283. func withCmdComment(comment string, platform string) runConfigModifier {
  284. return func(runConfig *container.Config) {
  285. runConfig.Cmd = append(getShell(runConfig, platform), "#(nop) ", comment)
  286. }
  287. }
  288. // withCmdCommentString exists to maintain compatibility with older versions.
  289. // A few instructions (workdir, copy, add) used a nop comment that is a single arg
  290. // where as all the other instructions used a two arg comment string. This
  291. // function implements the single arg version.
  292. func withCmdCommentString(comment string, platform string) runConfigModifier {
  293. return func(runConfig *container.Config) {
  294. runConfig.Cmd = append(getShell(runConfig, platform), "#(nop) "+comment)
  295. }
  296. }
  297. func withEnv(env []string) runConfigModifier {
  298. return func(runConfig *container.Config) {
  299. runConfig.Env = env
  300. }
  301. }
  302. // withEntrypointOverride sets an entrypoint on runConfig if the command is
  303. // not empty. The entrypoint is left unmodified if command is empty.
  304. //
  305. // The dockerfile RUN instruction expect to run without an entrypoint
  306. // so the runConfig entrypoint needs to be modified accordingly. ContainerCreate
  307. // will change a []string{""} entrypoint to nil, so we probe the cache with the
  308. // nil entrypoint.
  309. func withEntrypointOverride(cmd []string, entrypoint []string) runConfigModifier {
  310. return func(runConfig *container.Config) {
  311. if len(cmd) > 0 {
  312. runConfig.Entrypoint = entrypoint
  313. }
  314. }
  315. }
  316. // withoutHealthcheck disables healthcheck.
  317. //
  318. // The dockerfile RUN instruction expect to run without healthcheck
  319. // so the runConfig Healthcheck needs to be disabled.
  320. func withoutHealthcheck() runConfigModifier {
  321. return func(runConfig *container.Config) {
  322. runConfig.Healthcheck = &container.HealthConfig{
  323. Test: []string{"NONE"},
  324. }
  325. }
  326. }
  327. func copyRunConfig(runConfig *container.Config, modifiers ...runConfigModifier) *container.Config {
  328. copy := *runConfig
  329. copy.Cmd = copyStringSlice(runConfig.Cmd)
  330. copy.Env = copyStringSlice(runConfig.Env)
  331. copy.Entrypoint = copyStringSlice(runConfig.Entrypoint)
  332. copy.OnBuild = copyStringSlice(runConfig.OnBuild)
  333. copy.Shell = copyStringSlice(runConfig.Shell)
  334. if copy.Volumes != nil {
  335. copy.Volumes = make(map[string]struct{}, len(runConfig.Volumes))
  336. for k, v := range runConfig.Volumes {
  337. copy.Volumes[k] = v
  338. }
  339. }
  340. if copy.ExposedPorts != nil {
  341. copy.ExposedPorts = make(nat.PortSet, len(runConfig.ExposedPorts))
  342. for k, v := range runConfig.ExposedPorts {
  343. copy.ExposedPorts[k] = v
  344. }
  345. }
  346. if copy.Labels != nil {
  347. copy.Labels = make(map[string]string, len(runConfig.Labels))
  348. for k, v := range runConfig.Labels {
  349. copy.Labels[k] = v
  350. }
  351. }
  352. for _, modifier := range modifiers {
  353. modifier(&copy)
  354. }
  355. return &copy
  356. }
  357. func copyStringSlice(orig []string) []string {
  358. if orig == nil {
  359. return nil
  360. }
  361. return append([]string{}, orig...)
  362. }
  363. // getShell is a helper function which gets the right shell for prefixing the
  364. // shell-form of RUN, ENTRYPOINT and CMD instructions
  365. func getShell(c *container.Config, os string) []string {
  366. if 0 == len(c.Shell) {
  367. return append([]string{}, defaultShellForOS(os)[:]...)
  368. }
  369. return append([]string{}, c.Shell[:]...)
  370. }
  371. func (b *Builder) probeCache(dispatchState *dispatchState, runConfig *container.Config) (bool, error) {
  372. cachedID, err := b.imageProber.Probe(dispatchState.imageID, runConfig)
  373. if cachedID == "" || err != nil {
  374. return false, err
  375. }
  376. fmt.Fprint(b.Stdout, " ---> Using cache\n")
  377. dispatchState.imageID = cachedID
  378. return true, nil
  379. }
  380. var defaultLogConfig = container.LogConfig{Type: "none"}
  381. func (b *Builder) probeAndCreate(dispatchState *dispatchState, runConfig *container.Config) (string, error) {
  382. if hit, err := b.probeCache(dispatchState, runConfig); err != nil || hit {
  383. return "", err
  384. }
  385. return b.create(runConfig)
  386. }
  387. func (b *Builder) create(runConfig *container.Config) (string, error) {
  388. logrus.Debugf("[BUILDER] Command to be executed: %v", runConfig.Cmd)
  389. isWCOW := runtime.GOOS == "windows" && b.platform != nil && b.platform.OS == "windows"
  390. hostConfig := hostConfigFromOptions(b.options, isWCOW)
  391. container, err := b.containerManager.Create(runConfig, hostConfig)
  392. if err != nil {
  393. return "", err
  394. }
  395. // TODO: could this be moved into containerManager.Create() ?
  396. for _, warning := range container.Warnings {
  397. fmt.Fprintf(b.Stdout, " ---> [Warning] %s\n", warning)
  398. }
  399. fmt.Fprintf(b.Stdout, " ---> Running in %s\n", stringid.TruncateID(container.ID))
  400. return container.ID, nil
  401. }
  402. func hostConfigFromOptions(options *types.ImageBuildOptions, isWCOW bool) *container.HostConfig {
  403. resources := container.Resources{
  404. CgroupParent: options.CgroupParent,
  405. CPUShares: options.CPUShares,
  406. CPUPeriod: options.CPUPeriod,
  407. CPUQuota: options.CPUQuota,
  408. CpusetCpus: options.CPUSetCPUs,
  409. CpusetMems: options.CPUSetMems,
  410. Memory: options.Memory,
  411. MemorySwap: options.MemorySwap,
  412. Ulimits: options.Ulimits,
  413. }
  414. hc := &container.HostConfig{
  415. SecurityOpt: options.SecurityOpt,
  416. Isolation: options.Isolation,
  417. ShmSize: options.ShmSize,
  418. Resources: resources,
  419. NetworkMode: container.NetworkMode(options.NetworkMode),
  420. // Set a log config to override any default value set on the daemon
  421. LogConfig: defaultLogConfig,
  422. ExtraHosts: options.ExtraHosts,
  423. }
  424. // For WCOW, the default of 20GB hard-coded in the platform
  425. // is too small for builder scenarios where many users are
  426. // using RUN statements to install large amounts of data.
  427. // Use 127GB as that's the default size of a VHD in Hyper-V.
  428. if isWCOW {
  429. hc.StorageOpt = make(map[string]string)
  430. hc.StorageOpt["size"] = "127GB"
  431. }
  432. return hc
  433. }
  434. // fromSlash works like filepath.FromSlash but with a given OS platform field
  435. func fromSlash(path, platform string) string {
  436. if platform == "windows" {
  437. return strings.Replace(path, "/", "\\", -1)
  438. }
  439. return path
  440. }
  441. // separator returns a OS path separator for the given OS platform
  442. func separator(platform string) byte {
  443. if platform == "windows" {
  444. return '\\'
  445. }
  446. return '/'
  447. }