diff --git a/api/types/backend/build.go b/api/types/backend/build.go
index 01fcbd1a73..ad19c9a1d0 100644
--- a/api/types/backend/build.go
+++ b/api/types/backend/build.go
@@ -22,3 +22,10 @@ type BuildConfig struct {
 	ProgressWriter ProgressWriter
 	Options        *types.ImageBuildOptions
 }
+
+// GetImageAndLayerOptions are the options supported by GetImageAndLayer
+type GetImageAndLayerOptions struct {
+	ForcePull  bool
+	AuthConfig map[string]types.AuthConfig
+	Output     io.Writer
+}
diff --git a/builder/builder.go b/builder/builder.go
index ccc43f6cc2..359eb89b24 100644
--- a/builder/builder.go
+++ b/builder/builder.go
@@ -34,12 +34,8 @@ type Source interface {
 
 // Backend abstracts calls to a Docker Daemon.
 type Backend interface {
-	// TODO: use digest reference instead of name
+	GetImageAndLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (Image, ReleaseableLayer, error)
 
-	// GetImageOnBuild looks up a Docker image referenced by `name`.
-	GetImageOnBuild(name string) (Image, error)
-	// PullOnBuild tells Docker to pull image referenced by `name`.
-	PullOnBuild(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer) (Image, error)
 	// ContainerAttachRaw attaches to container.
 	ContainerAttachRaw(cID string, stdin io.ReadCloser, stdout, stderr io.Writer, stream bool, attached chan struct{}) error
 	// ContainerCreate creates a new Docker container and returns potential warnings
@@ -62,15 +58,6 @@ type Backend interface {
 	// TODO: extract in the builder instead of passing `decompress`
 	// TODO: use containerd/fs.changestream instead as a source
 	CopyOnBuild(containerID string, destPath string, srcRoot string, srcPath string, decompress bool) error
-
-	// MountImage returns mounted path with rootfs of an image.
-	MountImage(name string) (string, func() error, error)
-}
-
-// Image represents a Docker image used by the builder.
-type Image interface {
-	ImageID() string
-	RunConfig() *container.Config
 }
 
 // Result is the output produced by a Builder
@@ -92,3 +79,15 @@ type ImageCache interface {
 	// and runconfig equals `cfg`. A cache miss is expected to return an empty ID and a nil error.
 	GetCache(parentID string, cfg *container.Config) (imageID string, err error)
 }
+
+// Image represents a Docker image used by the builder.
+type Image interface {
+	ImageID() string
+	RunConfig() *container.Config
+}
+
+// ReleaseableLayer is an image layer that can be mounted and released
+type ReleaseableLayer interface {
+	Release() error
+	Mount() (string, error)
+}
diff --git a/builder/dockerfile/builder.go b/builder/dockerfile/builder.go
index ac346d5cc8..f7df0b71b9 100644
--- a/builder/dockerfile/builder.go
+++ b/builder/dockerfile/builder.go
@@ -45,7 +45,10 @@ type BuildManager struct {
 
 // NewBuildManager creates a BuildManager
 func NewBuildManager(b builder.Backend) *BuildManager {
-	return &BuildManager{backend: b, pathCache: &syncmap.Map{}}
+	return &BuildManager{
+		backend:   b,
+		pathCache: &syncmap.Map{},
+	}
 }
 
 // Build starts a new build from a BuildConfig
@@ -122,8 +125,8 @@ func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
 		docker:        options.Backend,
 		tmpContainers: map[string]struct{}{},
 		buildArgs:     newBuildArgs(config.BuildArgs),
+		imageContexts: &imageContexts{},
 	}
-	b.imageContexts = &imageContexts{b: b, cache: options.PathCache}
 	return b
 }
 
diff --git a/builder/dockerfile/dispatchers.go b/builder/dockerfile/dispatchers.go
index af9f67f3c9..da8b181637 100644
--- a/builder/dockerfile/dispatchers.go
+++ b/builder/dockerfile/dispatchers.go
@@ -20,9 +20,9 @@ import (
 	"github.com/Sirupsen/logrus"
 	"github.com/docker/docker/api"
 	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/api/types/strslice"
-	"github.com/docker/docker/builder"
 	"github.com/docker/docker/pkg/signal"
 	"github.com/docker/go-connections/nat"
 	"github.com/pkg/errors"
@@ -160,23 +160,29 @@ func dispatchCopy(req dispatchRequest) error {
 	}
 
 	flFrom := req.flags.AddString("from", "")
-
 	if err := req.flags.Parse(); err != nil {
 		return err
 	}
 
-	var im *imageMount
-	if flFrom.IsUsed() {
-		var err error
-		im, err = req.builder.imageContexts.get(flFrom.Value)
-		if err != nil {
-			return err
-		}
+	im, err := req.builder.getImageMount(flFrom)
+	if err != nil {
+		return errors.Wrapf(err, "invalid from flag value %s", flFrom.Value)
 	}
-
 	return req.builder.runContextCommand(req, false, false, "COPY", im)
 }
 
+func (b *Builder) getImageMount(fromFlag *Flag) (*imageMount, error) {
+	if !fromFlag.IsUsed() {
+		// TODO: this could return the mount in the default case as well
+		return nil, nil
+	}
+	im, err := b.imageContexts.getMount(fromFlag.Value)
+	if err != nil || im != nil {
+		return im, err
+	}
+	return b.getImage(fromFlag.Value)
+}
+
 // FROM imagename[:tag | @digest] [AS build-stage-name]
 //
 func from(req dispatchRequest) error {
@@ -192,20 +198,17 @@ func from(req dispatchRequest) error {
 	req.builder.resetImageCache()
 	req.state.noBaseImage = false
 	req.state.stageName = stageName
-	if _, err := req.builder.imageContexts.add(stageName); err != nil {
-		return err
-	}
-
-	image, err := req.builder.getFromImage(req.state, req.shlex, req.args[0])
+	image, err := req.builder.getFromImage(req.shlex, req.args[0])
 	if err != nil {
 		return err
 	}
-	switch image {
-	case nil:
+	if image == nil {
 		req.state.imageID = ""
 		req.state.noBaseImage = true
-	default:
-		req.builder.imageContexts.update(image.ImageID(), image.RunConfig())
+		image = newImageMount(nil, nil)
+	}
+	if err := req.builder.imageContexts.add(stageName, image); err != nil {
+		return err
 	}
 	req.state.baseImage = image
 
@@ -228,7 +231,7 @@ func parseBuildStageName(args []string) (string, error) {
 	return stageName, nil
 }
 
-func (b *Builder) getFromImage(dispatchState *dispatchState, shlex *ShellLex, name string) (builder.Image, error) {
+func (b *Builder) getFromImage(shlex *ShellLex, name string) (*imageMount, error) {
 	substitutionArgs := []string{}
 	for key, value := range b.buildArgs.GetAllMeta() {
 		substitutionArgs = append(substitutionArgs, key+"="+value)
@@ -254,7 +257,19 @@ func (b *Builder) getFromImage(dispatchState *dispatchState, shlex *ShellLex, na
 		}
 		return nil, nil
 	}
-	return pullOrGetImage(b, name)
+	return b.getImage(name)
+}
+
+func (b *Builder) getImage(name string) (*imageMount, error) {
+	image, layer, err := b.docker.GetImageAndLayer(b.clientCtx, name, backend.GetImageAndLayerOptions{
+		ForcePull:  b.options.PullParent,
+		AuthConfig: b.options.AuthConfigs,
+		Output:     b.Output,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return newImageMount(image, layer), nil
 }
 
 // ONBUILD RUN echo yo
@@ -801,29 +816,3 @@ func errBlankCommandNames(command string) error {
 func errTooManyArguments(command string) error {
 	return fmt.Errorf("Bad input to %s, too many arguments", command)
 }
-
-// mountByRef creates an imageMount from a reference. pulling the image if needed.
-func mountByRef(b *Builder, name string) (*imageMount, error) {
-	image, err := pullOrGetImage(b, name)
-	if err != nil {
-		return nil, err
-	}
-	im := b.imageContexts.newImageMount(image.ImageID())
-	return im, nil
-}
-
-func pullOrGetImage(b *Builder, name string) (builder.Image, error) {
-	var image builder.Image
-	if !b.options.PullParent {
-		image, _ = b.docker.GetImageOnBuild(name)
-		// TODO: shouldn't we error out if error is different from "not found" ?
-	}
-	if image == nil {
-		var err error
-		image, err = b.docker.PullOnBuild(b.clientCtx, name, b.options.AuthConfigs, b.Output)
-		if err != nil {
-			return nil, err
-		}
-	}
-	return image, nil
-}
diff --git a/builder/dockerfile/dispatchers_test.go b/builder/dockerfile/dispatchers_test.go
index 486b311ef8..5b8a23bb6c 100644
--- a/builder/dockerfile/dispatchers_test.go
+++ b/builder/dockerfile/dispatchers_test.go
@@ -47,16 +47,17 @@ func defaultDispatchReq(builder *Builder, args ...string) dispatchRequest {
 }
 
 func newBuilderWithMockBackend() *Builder {
+	mockBackend := &MockBackend{}
 	b := &Builder{
 		options:       &types.ImageBuildOptions{},
-		docker:        &MockBackend{},
+		docker:        mockBackend,
 		buildArgs:     newBuildArgs(make(map[string]*string)),
 		tmpContainers: make(map[string]struct{}),
 		Stdout:        new(bytes.Buffer),
 		clientCtx:     context.Background(),
 		disableCommit: true,
+		imageContexts: &imageContexts{},
 	}
-	b.imageContexts = &imageContexts{b: b}
 	return b
 }
 
@@ -198,12 +199,12 @@ func TestFromScratch(t *testing.T) {
 func TestFromWithArg(t *testing.T) {
 	tag, expected := ":sometag", "expectedthisid"
 
-	getImage := func(name string) (builder.Image, error) {
+	getImage := func(name string) (builder.Image, builder.ReleaseableLayer, error) {
 		assert.Equal(t, "alpine"+tag, name)
-		return &mockImage{id: "expectedthisid"}, nil
+		return &mockImage{id: "expectedthisid"}, nil, nil
 	}
 	b := newBuilderWithMockBackend()
-	b.docker.(*MockBackend).getImageOnBuildFunc = getImage
+	b.docker.(*MockBackend).getImageFunc = getImage
 
 	require.NoError(t, arg(defaultDispatchReq(b, "THETAG="+tag)))
 	req := defaultDispatchReq(b, "alpine${THETAG}")
@@ -219,12 +220,12 @@ func TestFromWithArg(t *testing.T) {
 func TestFromWithUndefinedArg(t *testing.T) {
 	tag, expected := "sometag", "expectedthisid"
 
-	getImage := func(name string) (builder.Image, error) {
+	getImage := func(name string) (builder.Image, builder.ReleaseableLayer, error) {
 		assert.Equal(t, "alpine", name)
-		return &mockImage{id: "expectedthisid"}, nil
+		return &mockImage{id: "expectedthisid"}, nil, nil
 	}
 	b := newBuilderWithMockBackend()
-	b.docker.(*MockBackend).getImageOnBuildFunc = getImage
+	b.docker.(*MockBackend).getImageFunc = getImage
 	b.options.BuildArgs = map[string]*string{"THETAG": &tag}
 
 	req := defaultDispatchReq(b, "alpine${THETAG}")
@@ -474,9 +475,11 @@ func TestRunWithBuildArgs(t *testing.T) {
 	b.imageCache = imageCache
 
 	mockBackend := b.docker.(*MockBackend)
-	mockBackend.getImageOnBuildImage = &mockImage{
-		id:     "abcdef",
-		config: &container.Config{Cmd: origCmd},
+	mockBackend.getImageFunc = func(_ string) (builder.Image, builder.ReleaseableLayer, error) {
+		return &mockImage{
+			id:     "abcdef",
+			config: &container.Config{Cmd: origCmd},
+		}, nil, nil
 	}
 	mockBackend.containerCreateFunc = func(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error) {
 		// Check the runConfig.Cmd sent to create()
diff --git a/builder/dockerfile/imagecontext.go b/builder/dockerfile/imagecontext.go
index b9393b7bc3..571a20116e 100644
--- a/builder/dockerfile/imagecontext.go
+++ b/builder/dockerfile/imagecontext.go
@@ -19,49 +19,40 @@ type pathCache interface {
 // imageContexts is a helper for stacking up built image rootfs and reusing
 // them as contexts
 type imageContexts struct {
-	b              *Builder
-	list           []*imageMount // indexed list of stages
-	implicitMounts []*imageMount // implicitly mounted images
-	byName         map[string]*imageMount
-	cache          pathCache
+	list   []*imageMount
+	byName map[string]*imageMount
+	cache  pathCache
 }
 
-func (ic *imageContexts) newImageMount(id string) *imageMount {
-	return &imageMount{ic: ic, id: id}
-}
-
-func (ic *imageContexts) add(name string) (*imageMount, error) {
-	im := &imageMount{ic: ic}
+func (ic *imageContexts) add(name string, im *imageMount) error {
 	if len(name) > 0 {
 		if ic.byName == nil {
 			ic.byName = make(map[string]*imageMount)
 		}
 		if _, ok := ic.byName[name]; ok {
-			return nil, errors.Errorf("duplicate name %s", name)
+			return errors.Errorf("duplicate name %s", name)
 		}
 		ic.byName[name] = im
 	}
 	ic.list = append(ic.list, im)
-	return im, nil
+	return nil
 }
 
 func (ic *imageContexts) update(imageID string, runConfig *container.Config) {
-	ic.list[len(ic.list)-1].id = imageID
-	ic.list[len(ic.list)-1].runConfig = runConfig
+	ic.list[len(ic.list)-1].update(imageID, runConfig)
 }
 
 func (ic *imageContexts) validate(i int) error {
 	if i < 0 || i >= len(ic.list)-1 {
-		var extraMsg string
 		if i == len(ic.list)-1 {
-			extraMsg = " refers current build block"
+			return errors.Errorf("%d refers to current build stage", i)
 		}
-		return errors.Errorf("invalid from flag value %d%s", i, extraMsg)
+		return errors.Errorf("index out of bounds")
 	}
 	return nil
 }
 
-func (ic *imageContexts) get(indexOrName string) (*imageMount, error) {
+func (ic *imageContexts) getMount(indexOrName string) (*imageMount, error) {
 	index, err := strconv.Atoi(indexOrName)
 	if err == nil {
 		if err := ic.validate(index); err != nil {
@@ -72,12 +63,7 @@ func (ic *imageContexts) get(indexOrName string) (*imageMount, error) {
 	if im, ok := ic.byName[strings.ToLower(indexOrName)]; ok {
 		return im, nil
 	}
-	im, err := mountByRef(ic.b, indexOrName)
-	if err != nil {
-		return nil, errors.Wrapf(err, "invalid from flag value %s", indexOrName)
-	}
-	ic.implicitMounts = append(ic.implicitMounts, im)
-	return im, nil
+	return nil, nil
 }
 
 func (ic *imageContexts) unmount() (retErr error) {
@@ -98,6 +84,7 @@ func (ic *imageContexts) unmount() (retErr error) {
 	return
 }
 
+// TODO: remove getCache/setCache from imageContexts
 func (ic *imageContexts) getCache(id, path string) (interface{}, bool) {
 	if ic.cache != nil {
 		if id == "" {
@@ -114,48 +101,56 @@ func (ic *imageContexts) setCache(id, path string, v interface{}) {
 	}
 }
 
-// imageMount is a reference for getting access to a buildcontext that is backed
-// by an existing image
+// imageMount is a reference to an image that can be used as a builder.Source
 type imageMount struct {
 	id        string
 	source    builder.Source
-	release   func() error
-	ic        *imageContexts
 	runConfig *container.Config
+	layer     builder.ReleaseableLayer
+}
+
+func newImageMount(image builder.Image, layer builder.ReleaseableLayer) *imageMount {
+	im := &imageMount{layer: layer}
+	if image != nil {
+		im.update(image.ImageID(), image.RunConfig())
+	}
+	return im
 }
 
 func (im *imageMount) context() (builder.Source, error) {
 	if im.source == nil {
-		if im.id == "" {
-			return nil, errors.Errorf("could not copy from empty context")
+		if im.id == "" || im.layer == nil {
+			return nil, errors.Errorf("empty context")
 		}
-		p, release, err := im.ic.b.docker.MountImage(im.id)
+		mountPath, err := im.layer.Mount()
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to mount %s", im.id)
 		}
-		source, err := remotecontext.NewLazyContext(p)
+		source, err := remotecontext.NewLazyContext(mountPath)
 		if err != nil {
-			return nil, errors.Wrapf(err, "failed to create lazycontext for %s", p)
+			return nil, errors.Wrapf(err, "failed to create lazycontext for %s", mountPath)
 		}
-		im.release = release
 		im.source = source
 	}
 	return im.source, nil
 }
 
 func (im *imageMount) unmount() error {
-	if im.release != nil {
-		if err := im.release(); err != nil {
-			return errors.Wrapf(err, "failed to unmount previous build image %s", im.id)
-		}
-		im.release = nil
+	if err := im.layer.Release(); err != nil {
+		return errors.Wrapf(err, "failed to unmount previous build image %s", im.id)
 	}
 	return nil
 }
 
+func (im *imageMount) update(imageID string, runConfig *container.Config) {
+	im.id = imageID
+	im.runConfig = runConfig
+}
+
 func (im *imageMount) ImageID() string {
 	return im.id
 }
+
 func (im *imageMount) RunConfig() *container.Config {
 	return im.runConfig
 }
diff --git a/builder/dockerfile/internals.go b/builder/dockerfile/internals.go
index 963f0509f0..b70a61b8e6 100644
--- a/builder/dockerfile/internals.go
+++ b/builder/dockerfile/internals.go
@@ -372,7 +372,7 @@ func (b *Builder) calcCopyInfo(cmdName, origPath string, allowLocalDecompression
 	if imageSource != nil {
 		source, err = imageSource.context()
 		if err != nil {
-			return nil, err
+			return nil, errors.Wrapf(err, "failed to copy")
 		}
 	}
 
diff --git a/builder/dockerfile/mockbackend_test.go b/builder/dockerfile/mockbackend_test.go
index bdd198ccfe..a874dfb550 100644
--- a/builder/dockerfile/mockbackend_test.go
+++ b/builder/dockerfile/mockbackend_test.go
@@ -15,30 +15,15 @@ import (
 
 // MockBackend implements the builder.Backend interface for unit testing
 type MockBackend struct {
-	getImageOnBuildFunc  func(string) (builder.Image, error)
-	getImageOnBuildImage *mockImage
-	containerCreateFunc  func(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error)
-	commitFunc           func(string, *backend.ContainerCommitConfig) (string, error)
-}
-
-func (m *MockBackend) GetImageOnBuild(name string) (builder.Image, error) {
-	if m.getImageOnBuildFunc != nil {
-		return m.getImageOnBuildFunc(name)
-	}
-	if m.getImageOnBuildImage != nil {
-		return m.getImageOnBuildImage, nil
-	}
-	return &mockImage{id: "theid"}, nil
+	containerCreateFunc func(config types.ContainerCreateConfig) (container.ContainerCreateCreatedBody, error)
+	commitFunc          func(string, *backend.ContainerCommitConfig) (string, error)
+	getImageFunc        func(string) (builder.Image, builder.ReleaseableLayer, error)
 }
 
 func (m *MockBackend) TagImageWithReference(image.ID, reference.Named) error {
 	return nil
 }
 
-func (m *MockBackend) PullOnBuild(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer) (builder.Image, error) {
-	return nil, nil
-}
-
 func (m *MockBackend) ContainerAttachRaw(cID string, stdin io.ReadCloser, stdout, stderr io.Writer, stream bool, attached chan struct{}) error {
 	return nil
 }
@@ -89,8 +74,12 @@ func (m *MockBackend) SquashImage(from string, to string) (string, error) {
 	return "", nil
 }
 
-func (m *MockBackend) MountImage(name string) (string, func() error, error) {
-	return "", func() error { return nil }, nil
+func (m *MockBackend) GetImageAndLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ReleaseableLayer, error) {
+	if m.getImageFunc != nil {
+		return m.getImageFunc(refOrID)
+	}
+
+	return &mockImage{id: "theid"}, &mockLayer{}, nil
 }
 
 type mockImage struct {
@@ -116,3 +105,13 @@ func (mic *mockImageCache) GetCache(parentID string, cfg *container.Config) (str
 	}
 	return "", nil
 }
+
+type mockLayer struct{}
+
+func (l *mockLayer) Release() error {
+	return nil
+}
+
+func (l *mockLayer) Mount() (string, error) {
+	return "mountPath", nil
+}
diff --git a/daemon/archive.go b/daemon/archive.go
index 9b0a2f4a02..dbef0b4d48 100644
--- a/daemon/archive.go
+++ b/daemon/archive.go
@@ -8,12 +8,10 @@ import (
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/container"
-	"github.com/docker/docker/layer"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/chrootarchive"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/ioutils"
-	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/docker/pkg/symlink"
 	"github.com/docker/docker/pkg/system"
 	"github.com/pkg/errors"
@@ -470,34 +468,3 @@ func (daemon *Daemon) CopyOnBuild(cID, destPath, srcRoot, srcPath string, decomp
 
 	return fixPermissions(fullSrcPath, destPath, rootUID, rootGID, destExists)
 }
-
-// MountImage returns mounted path with rootfs of an image.
-func (daemon *Daemon) MountImage(name string) (string, func() error, error) {
-	img, err := daemon.GetImage(name)
-	if err != nil {
-		return "", nil, errors.Wrapf(err, "no such image: %s", name)
-	}
-
-	mountID := stringid.GenerateRandomID()
-	rwLayer, err := daemon.layerStore.CreateRWLayer(mountID, img.RootFS.ChainID(), nil)
-	if err != nil {
-		return "", nil, errors.Wrap(err, "failed to create rwlayer")
-	}
-
-	mountPath, err := rwLayer.Mount("")
-	if err != nil {
-		metadata, releaseErr := daemon.layerStore.ReleaseRWLayer(rwLayer)
-		if releaseErr != nil {
-			err = errors.Wrapf(err, "failed to release rwlayer: %s", releaseErr.Error())
-		}
-		layer.LogReleaseMetadata(metadata)
-		return "", nil, errors.Wrap(err, "failed to mount rwlayer")
-	}
-
-	return mountPath, func() error {
-		rwLayer.Unmount()
-		metadata, err := daemon.layerStore.ReleaseRWLayer(rwLayer)
-		layer.LogReleaseMetadata(metadata)
-		return err
-	}, nil
-}
diff --git a/daemon/build.go b/daemon/build.go
new file mode 100644
index 0000000000..44640efee5
--- /dev/null
+++ b/daemon/build.go
@@ -0,0 +1,108 @@
+package daemon
+
+import (
+	"github.com/docker/distribution/reference"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/backend"
+	"github.com/docker/docker/builder"
+	"github.com/docker/docker/image"
+	"github.com/docker/docker/layer"
+	"github.com/docker/docker/pkg/stringid"
+	"github.com/docker/docker/registry"
+	"github.com/pkg/errors"
+	"golang.org/x/net/context"
+	"io"
+)
+
+type releaseableLayer struct {
+	rwLayer layer.RWLayer
+	release func(layer.RWLayer) error
+	mount   func() (layer.RWLayer, error)
+}
+
+func (rl *releaseableLayer) Release() error {
+	if rl.rwLayer == nil {
+		return nil
+	}
+	rl.rwLayer.Unmount()
+	return rl.release(rl.rwLayer)
+}
+
+func (rl *releaseableLayer) Mount() (string, error) {
+	var err error
+	// daemon.layerStore.CreateRWLayer(mountID, img.RootFS.ChainID(), nil)
+	rl.rwLayer, err = rl.mount()
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create rwlayer")
+	}
+
+	mountPath, err := rl.rwLayer.Mount("")
+	if err != nil {
+		releaseErr := rl.release(rl.rwLayer)
+		if releaseErr != nil {
+			err = errors.Wrapf(err, "failed to release rwlayer: %s", releaseErr.Error())
+		}
+		return "", errors.Wrap(err, "failed to mount rwlayer")
+	}
+	return mountPath, err
+}
+
+func (daemon *Daemon) getReleasableLayerForImage(img *image.Image) (*releaseableLayer, error) {
+	mountFunc := func() (layer.RWLayer, error) {
+		mountID := stringid.GenerateRandomID()
+		return daemon.layerStore.CreateRWLayer(mountID, img.RootFS.ChainID(), nil)
+	}
+
+	releaseFunc := func(rwLayer layer.RWLayer) error {
+		metadata, err := daemon.layerStore.ReleaseRWLayer(rwLayer)
+		layer.LogReleaseMetadata(metadata)
+		return err
+	}
+
+	return &releaseableLayer{mount: mountFunc, release: releaseFunc}, nil
+}
+
+// TODO: could this use the regular daemon PullImage ?
+func (daemon *Daemon) pullForBuilder(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer) (*image.Image, error) {
+	ref, err := reference.ParseNormalizedNamed(name)
+	if err != nil {
+		return nil, err
+	}
+	ref = reference.TagNameOnly(ref)
+
+	pullRegistryAuth := &types.AuthConfig{}
+	if len(authConfigs) > 0 {
+		// The request came with a full auth config file, we prefer to use that
+		repoInfo, err := daemon.RegistryService.ResolveRepository(ref)
+		if err != nil {
+			return nil, err
+		}
+
+		resolvedConfig := registry.ResolveAuthConfig(authConfigs, repoInfo.Index)
+		pullRegistryAuth = &resolvedConfig
+	}
+
+	if err := daemon.pullImageWithReference(ctx, ref, nil, pullRegistryAuth, output); err != nil {
+		return nil, err
+	}
+	return daemon.GetImage(name)
+}
+
+// GetImageAndLayer returns an image and releaseable layer for a reference or ID
+func (daemon *Daemon) GetImageAndLayer(ctx context.Context, refOrID string, opts backend.GetImageAndLayerOptions) (builder.Image, builder.ReleaseableLayer, error) {
+	if !opts.ForcePull {
+		image, _ := daemon.GetImage(refOrID)
+		// TODO: shouldn't we error out if error is different from "not found" ?
+		if image != nil {
+			layer, err := daemon.getReleasableLayerForImage(image)
+			return image, layer, err
+		}
+	}
+
+	image, err := daemon.pullForBuilder(ctx, refOrID, opts.AuthConfig, opts.Output)
+	if err != nil {
+		return nil, nil, err
+	}
+	layer, err := daemon.getReleasableLayerForImage(image)
+	return image, layer, err
+}
diff --git a/daemon/image.go b/daemon/image.go
index 43ee483ff0..d10457118f 100644
--- a/daemon/image.go
+++ b/daemon/image.go
@@ -4,7 +4,6 @@ import (
 	"fmt"
 
 	"github.com/docker/distribution/reference"
-	"github.com/docker/docker/builder"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/pkg/stringid"
 )
@@ -75,12 +74,3 @@ func (daemon *Daemon) GetImage(refOrID string) (*image.Image, error) {
 	}
 	return daemon.imageStore.Get(imgID)
 }
-
-// GetImageOnBuild looks up a Docker image referenced by `name`.
-func (daemon *Daemon) GetImageOnBuild(name string) (builder.Image, error) {
-	img, err := daemon.GetImage(name)
-	if err != nil {
-		return nil, err
-	}
-	return img, nil
-}
diff --git a/daemon/image_pull.go b/daemon/image_pull.go
index e0ac92ce44..304fd9f024 100644
--- a/daemon/image_pull.go
+++ b/daemon/image_pull.go
@@ -7,7 +7,6 @@ import (
 	dist "github.com/docker/distribution"
 	"github.com/docker/distribution/reference"
 	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/builder"
 	"github.com/docker/docker/distribution"
 	progressutils "github.com/docker/docker/distribution/utils"
 	"github.com/docker/docker/pkg/progress"
@@ -46,35 +45,6 @@ func (daemon *Daemon) PullImage(ctx context.Context, image, tag string, metaHead
 	return daemon.pullImageWithReference(ctx, ref, metaHeaders, authConfig, outStream)
 }
 
-// PullOnBuild tells Docker to pull image referenced by `name`.
-func (daemon *Daemon) PullOnBuild(ctx context.Context, name string, authConfigs map[string]types.AuthConfig, output io.Writer) (builder.Image, error) {
-	ref, err := reference.ParseNormalizedNamed(name)
-	if err != nil {
-		return nil, err
-	}
-	ref = reference.TagNameOnly(ref)
-
-	pullRegistryAuth := &types.AuthConfig{}
-	if len(authConfigs) > 0 {
-		// The request came with a full auth config file, we prefer to use that
-		repoInfo, err := daemon.RegistryService.ResolveRepository(ref)
-		if err != nil {
-			return nil, err
-		}
-
-		resolvedConfig := registry.ResolveAuthConfig(
-			authConfigs,
-			repoInfo.Index,
-		)
-		pullRegistryAuth = &resolvedConfig
-	}
-
-	if err := daemon.pullImageWithReference(ctx, ref, nil, pullRegistryAuth, output); err != nil {
-		return nil, err
-	}
-	return daemon.GetImage(name)
-}
-
 func (daemon *Daemon) pullImageWithReference(ctx context.Context, ref reference.Named, metaHeaders map[string][]string, authConfig *types.AuthConfig, outStream io.Writer) error {
 	// Include a buffer so that slow client connections don't affect
 	// transfer performance.