builder.go 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420
  1. package dockerfile
  2. import (
  3. "bytes"
  4. "fmt"
  5. "io"
  6. "io/ioutil"
  7. "runtime"
  8. "strings"
  9. "time"
  10. "github.com/docker/docker/api/types"
  11. "github.com/docker/docker/api/types/backend"
  12. "github.com/docker/docker/api/types/container"
  13. "github.com/docker/docker/builder"
  14. "github.com/docker/docker/builder/dockerfile/instructions"
  15. "github.com/docker/docker/builder/dockerfile/parser"
  16. "github.com/docker/docker/builder/dockerfile/shell"
  17. "github.com/docker/docker/builder/fscache"
  18. "github.com/docker/docker/builder/remotecontext"
  19. "github.com/docker/docker/errdefs"
  20. "github.com/docker/docker/pkg/idtools"
  21. "github.com/docker/docker/pkg/streamformatter"
  22. "github.com/docker/docker/pkg/stringid"
  23. "github.com/docker/docker/pkg/system"
  24. "github.com/moby/buildkit/session"
  25. "github.com/pkg/errors"
  26. "github.com/sirupsen/logrus"
  27. "golang.org/x/net/context"
  28. "golang.org/x/sync/syncmap"
  29. )
  30. var validCommitCommands = map[string]bool{
  31. "cmd": true,
  32. "entrypoint": true,
  33. "healthcheck": true,
  34. "env": true,
  35. "expose": true,
  36. "label": true,
  37. "onbuild": true,
  38. "user": true,
  39. "volume": true,
  40. "workdir": true,
  41. }
  42. const (
  43. stepFormat = "Step %d/%d : %v"
  44. )
  45. // SessionGetter is object used to get access to a session by uuid
  46. type SessionGetter interface {
  47. Get(ctx context.Context, uuid string) (session.Caller, error)
  48. }
  49. // BuildManager is shared across all Builder objects
  50. type BuildManager struct {
  51. idMappings *idtools.IDMappings
  52. backend builder.Backend
  53. pathCache pathCache // TODO: make this persistent
  54. sg SessionGetter
  55. fsCache *fscache.FSCache
  56. }
  57. // NewBuildManager creates a BuildManager
  58. func NewBuildManager(b builder.Backend, sg SessionGetter, fsCache *fscache.FSCache, idMappings *idtools.IDMappings) (*BuildManager, error) {
  59. bm := &BuildManager{
  60. backend: b,
  61. pathCache: &syncmap.Map{},
  62. sg: sg,
  63. idMappings: idMappings,
  64. fsCache: fsCache,
  65. }
  66. if err := fsCache.RegisterTransport(remotecontext.ClientSessionRemote, NewClientSessionTransport()); err != nil {
  67. return nil, err
  68. }
  69. return bm, nil
  70. }
  71. // Build starts a new build from a BuildConfig
  72. func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (*builder.Result, error) {
  73. buildsTriggered.Inc()
  74. if config.Options.Dockerfile == "" {
  75. config.Options.Dockerfile = builder.DefaultDockerfileName
  76. }
  77. source, dockerfile, err := remotecontext.Detect(config)
  78. if err != nil {
  79. return nil, err
  80. }
  81. defer func() {
  82. if source != nil {
  83. if err := source.Close(); err != nil {
  84. logrus.Debugf("[BUILDER] failed to remove temporary context: %v", err)
  85. }
  86. }
  87. }()
  88. ctx, cancel := context.WithCancel(ctx)
  89. defer cancel()
  90. if src, err := bm.initializeClientSession(ctx, cancel, config.Options); err != nil {
  91. return nil, err
  92. } else if src != nil {
  93. source = src
  94. }
  95. os := runtime.GOOS
  96. optionsPlatform := system.ParsePlatform(config.Options.Platform)
  97. if dockerfile.OS != "" {
  98. if optionsPlatform.OS != "" && optionsPlatform.OS != dockerfile.OS {
  99. return nil, fmt.Errorf("invalid platform")
  100. }
  101. os = dockerfile.OS
  102. } else if optionsPlatform.OS != "" {
  103. os = optionsPlatform.OS
  104. }
  105. config.Options.Platform = os
  106. dockerfile.OS = os
  107. builderOptions := builderOptions{
  108. Options: config.Options,
  109. ProgressWriter: config.ProgressWriter,
  110. Backend: bm.backend,
  111. PathCache: bm.pathCache,
  112. IDMappings: bm.idMappings,
  113. }
  114. return newBuilder(ctx, builderOptions).build(source, dockerfile)
  115. }
  116. func (bm *BuildManager) initializeClientSession(ctx context.Context, cancel func(), options *types.ImageBuildOptions) (builder.Source, error) {
  117. if options.SessionID == "" || bm.sg == nil {
  118. return nil, nil
  119. }
  120. logrus.Debug("client is session enabled")
  121. connectCtx, cancelCtx := context.WithTimeout(ctx, sessionConnectTimeout)
  122. defer cancelCtx()
  123. c, err := bm.sg.Get(connectCtx, options.SessionID)
  124. if err != nil {
  125. return nil, err
  126. }
  127. go func() {
  128. <-c.Context().Done()
  129. cancel()
  130. }()
  131. if options.RemoteContext == remotecontext.ClientSessionRemote {
  132. st := time.Now()
  133. csi, err := NewClientSessionSourceIdentifier(ctx, bm.sg, options.SessionID)
  134. if err != nil {
  135. return nil, err
  136. }
  137. src, err := bm.fsCache.SyncFrom(ctx, csi)
  138. if err != nil {
  139. return nil, err
  140. }
  141. logrus.Debugf("sync-time: %v", time.Since(st))
  142. return src, nil
  143. }
  144. return nil, nil
  145. }
  146. // builderOptions are the dependencies required by the builder
  147. type builderOptions struct {
  148. Options *types.ImageBuildOptions
  149. Backend builder.Backend
  150. ProgressWriter backend.ProgressWriter
  151. PathCache pathCache
  152. IDMappings *idtools.IDMappings
  153. }
  154. // Builder is a Dockerfile builder
  155. // It implements the builder.Backend interface.
  156. type Builder struct {
  157. options *types.ImageBuildOptions
  158. Stdout io.Writer
  159. Stderr io.Writer
  160. Aux *streamformatter.AuxFormatter
  161. Output io.Writer
  162. docker builder.Backend
  163. clientCtx context.Context
  164. idMappings *idtools.IDMappings
  165. disableCommit bool
  166. imageSources *imageSources
  167. pathCache pathCache
  168. containerManager *containerManager
  169. imageProber ImageProber
  170. }
  171. // newBuilder creates a new Dockerfile builder from an optional dockerfile and a Options.
  172. func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
  173. config := options.Options
  174. if config == nil {
  175. config = new(types.ImageBuildOptions)
  176. }
  177. b := &Builder{
  178. clientCtx: clientCtx,
  179. options: config,
  180. Stdout: options.ProgressWriter.StdoutFormatter,
  181. Stderr: options.ProgressWriter.StderrFormatter,
  182. Aux: options.ProgressWriter.AuxFormatter,
  183. Output: options.ProgressWriter.Output,
  184. docker: options.Backend,
  185. idMappings: options.IDMappings,
  186. imageSources: newImageSources(clientCtx, options),
  187. pathCache: options.PathCache,
  188. imageProber: newImageProber(options.Backend, config.CacheFrom, config.NoCache),
  189. containerManager: newContainerManager(options.Backend),
  190. }
  191. return b
  192. }
  193. // Build runs the Dockerfile builder by parsing the Dockerfile and executing
  194. // the instructions from the file.
  195. func (b *Builder) build(source builder.Source, dockerfile *parser.Result) (*builder.Result, error) {
  196. defer b.imageSources.Unmount()
  197. addNodesForLabelOption(dockerfile.AST, b.options.Labels)
  198. stages, metaArgs, err := instructions.Parse(dockerfile.AST)
  199. if err != nil {
  200. if instructions.IsUnknownInstruction(err) {
  201. buildsFailed.WithValues(metricsUnknownInstructionError).Inc()
  202. }
  203. return nil, errdefs.InvalidParameter(err)
  204. }
  205. if b.options.Target != "" {
  206. targetIx, found := instructions.HasStage(stages, b.options.Target)
  207. if !found {
  208. buildsFailed.WithValues(metricsBuildTargetNotReachableError).Inc()
  209. return nil, errors.Errorf("failed to reach build target %s in Dockerfile", b.options.Target)
  210. }
  211. stages = stages[:targetIx+1]
  212. }
  213. dockerfile.PrintWarnings(b.Stderr)
  214. dispatchState, err := b.dispatchDockerfileWithCancellation(stages, metaArgs, dockerfile.EscapeToken, source)
  215. if err != nil {
  216. return nil, err
  217. }
  218. if dispatchState.imageID == "" {
  219. buildsFailed.WithValues(metricsDockerfileEmptyError).Inc()
  220. return nil, errors.New("No image was generated. Is your Dockerfile empty?")
  221. }
  222. return &builder.Result{ImageID: dispatchState.imageID, FromImage: dispatchState.baseImage}, nil
  223. }
  224. func emitImageID(aux *streamformatter.AuxFormatter, state *dispatchState) error {
  225. if aux == nil || state.imageID == "" {
  226. return nil
  227. }
  228. return aux.Emit(types.BuildResult{ID: state.imageID})
  229. }
  230. func processMetaArg(meta instructions.ArgCommand, shlex *shell.Lex, args *buildArgs) error {
  231. // shell.Lex currently only support the concatenated string format
  232. envs := convertMapToEnvList(args.GetAllAllowed())
  233. if err := meta.Expand(func(word string) (string, error) {
  234. return shlex.ProcessWord(word, envs)
  235. }); err != nil {
  236. return err
  237. }
  238. args.AddArg(meta.Key, meta.Value)
  239. args.AddMetaArg(meta.Key, meta.Value)
  240. return nil
  241. }
  242. func printCommand(out io.Writer, currentCommandIndex int, totalCommands int, cmd interface{}) int {
  243. fmt.Fprintf(out, stepFormat, currentCommandIndex, totalCommands, cmd)
  244. fmt.Fprintln(out)
  245. return currentCommandIndex + 1
  246. }
  247. func (b *Builder) dispatchDockerfileWithCancellation(parseResult []instructions.Stage, metaArgs []instructions.ArgCommand, escapeToken rune, source builder.Source) (*dispatchState, error) {
  248. dispatchRequest := dispatchRequest{}
  249. buildArgs := newBuildArgs(b.options.BuildArgs)
  250. totalCommands := len(metaArgs) + len(parseResult)
  251. currentCommandIndex := 1
  252. for _, stage := range parseResult {
  253. totalCommands += len(stage.Commands)
  254. }
  255. shlex := shell.NewLex(escapeToken)
  256. for _, meta := range metaArgs {
  257. currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, &meta)
  258. err := processMetaArg(meta, shlex, buildArgs)
  259. if err != nil {
  260. return nil, err
  261. }
  262. }
  263. stagesResults := newStagesBuildResults()
  264. for _, stage := range parseResult {
  265. if err := stagesResults.checkStageNameAvailable(stage.Name); err != nil {
  266. return nil, err
  267. }
  268. dispatchRequest = newDispatchRequest(b, escapeToken, source, buildArgs, stagesResults)
  269. currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, stage.SourceCode)
  270. if err := initializeStage(dispatchRequest, &stage); err != nil {
  271. return nil, err
  272. }
  273. dispatchRequest.state.updateRunConfig()
  274. fmt.Fprintf(b.Stdout, " ---> %s\n", stringid.TruncateID(dispatchRequest.state.imageID))
  275. for _, cmd := range stage.Commands {
  276. select {
  277. case <-b.clientCtx.Done():
  278. logrus.Debug("Builder: build cancelled!")
  279. fmt.Fprint(b.Stdout, "Build cancelled\n")
  280. buildsFailed.WithValues(metricsBuildCanceled).Inc()
  281. return nil, errors.New("Build cancelled")
  282. default:
  283. // Not cancelled yet, keep going...
  284. }
  285. currentCommandIndex = printCommand(b.Stdout, currentCommandIndex, totalCommands, cmd)
  286. if err := dispatch(dispatchRequest, cmd); err != nil {
  287. return nil, err
  288. }
  289. dispatchRequest.state.updateRunConfig()
  290. fmt.Fprintf(b.Stdout, " ---> %s\n", stringid.TruncateID(dispatchRequest.state.imageID))
  291. }
  292. if err := emitImageID(b.Aux, dispatchRequest.state); err != nil {
  293. return nil, err
  294. }
  295. buildArgs.MergeReferencedArgs(dispatchRequest.state.buildArgs)
  296. if err := commitStage(dispatchRequest.state, stagesResults); err != nil {
  297. return nil, err
  298. }
  299. }
  300. buildArgs.WarnOnUnusedBuildArgs(b.Stdout)
  301. return dispatchRequest.state, nil
  302. }
  303. func addNodesForLabelOption(dockerfile *parser.Node, labels map[string]string) {
  304. if len(labels) == 0 {
  305. return
  306. }
  307. node := parser.NodeFromLabels(labels)
  308. dockerfile.Children = append(dockerfile.Children, node)
  309. }
  310. // BuildFromConfig builds directly from `changes`, treating it as if it were the contents of a Dockerfile
  311. // It will:
  312. // - Call parse.Parse() to get an AST root for the concatenated Dockerfile entries.
  313. // - Do build by calling builder.dispatch() to call all entries' handling routines
  314. //
  315. // BuildFromConfig is used by the /commit endpoint, with the changes
  316. // coming from the query parameter of the same name.
  317. //
  318. // TODO: Remove?
  319. func BuildFromConfig(config *container.Config, changes []string, os string) (*container.Config, error) {
  320. if !system.IsOSSupported(os) {
  321. return nil, errdefs.InvalidParameter(system.ErrNotSupportedOperatingSystem)
  322. }
  323. if len(changes) == 0 {
  324. return config, nil
  325. }
  326. dockerfile, err := parser.Parse(bytes.NewBufferString(strings.Join(changes, "\n")))
  327. if err != nil {
  328. return nil, errdefs.InvalidParameter(err)
  329. }
  330. b := newBuilder(context.Background(), builderOptions{
  331. Options: &types.ImageBuildOptions{NoCache: true},
  332. })
  333. // ensure that the commands are valid
  334. for _, n := range dockerfile.AST.Children {
  335. if !validCommitCommands[n.Value] {
  336. return nil, errdefs.InvalidParameter(errors.Errorf("%s is not a valid change command", n.Value))
  337. }
  338. }
  339. b.Stdout = ioutil.Discard
  340. b.Stderr = ioutil.Discard
  341. b.disableCommit = true
  342. commands := []instructions.Command{}
  343. for _, n := range dockerfile.AST.Children {
  344. cmd, err := instructions.ParseCommand(n)
  345. if err != nil {
  346. return nil, errdefs.InvalidParameter(err)
  347. }
  348. commands = append(commands, cmd)
  349. }
  350. dispatchRequest := newDispatchRequest(b, dockerfile.EscapeToken, nil, newBuildArgs(b.options.BuildArgs), newStagesBuildResults())
  351. // We make mutations to the configuration, ensure we have a copy
  352. dispatchRequest.state.runConfig = copyRunConfig(config)
  353. dispatchRequest.state.imageID = config.Image
  354. dispatchRequest.state.operatingSystem = os
  355. for _, cmd := range commands {
  356. err := dispatch(dispatchRequest, cmd)
  357. if err != nil {
  358. return nil, errdefs.InvalidParameter(err)
  359. }
  360. dispatchRequest.state.updateRunConfig()
  361. }
  362. return dispatchRequest.state.runConfig, nil
  363. }
  364. func convertMapToEnvList(m map[string]string) []string {
  365. result := []string{}
  366. for k, v := range m {
  367. result = append(result, k+"="+v)
  368. }
  369. return result
  370. }