LCOW: Implemented support for docker cp + build

This enables docker cp and ADD/COPY docker build support for LCOW.
Originally, the graphdriver.Get() interface returned a local path
to the container root filesystem. This does not work for LCOW, so
the Get() method now returns an interface that LCOW implements to
support copying to and from the container.

Signed-off-by: Akash Gupta <akagup@microsoft.com>

api/server/httputils/form.go

@@ -2,7 +2,6 @@ package httputils
 
 import (
 	"net/http"
-	"path/filepath"
 	"strconv"
 	"strings"
 )
@@ -69,8 +68,7 @@ func ArchiveFormValues(r *http.Request, vars map[string]string) (ArchiveOptions,
 	if name == "" {
 		return ArchiveOptions{}, badParameterError{"name"}
 	}
-
-	path := filepath.FromSlash(r.Form.Get("path"))
+	path := r.Form.Get("path")
 	if path == "" {
 		return ArchiveOptions{}, badParameterError{"path"}
 	}

builder/builder.go

@@ -12,6 +12,7 @@ import (
 	"github.com/docker/docker/api/types/container"
 	containerpkg "github.com/docker/docker/container"
 	"github.com/docker/docker/layer"
+	"github.com/docker/docker/pkg/containerfs"
 	"golang.org/x/net/context"
 )
 
@@ -24,7 +25,7 @@ const (
 // instructions in the builder.
 type Source interface {
 	// Root returns root path for accessing source
-	Root() string
+	Root() containerfs.ContainerFS
 	// Close allows to signal that the filesystem tree won't be used anymore.
 	// For Context implementations using a temporary directory, it is recommended to
 	// delete the temporary directory in Close().
@@ -99,7 +100,7 @@ type Image interface {
 // ReleaseableLayer is an image layer that can be mounted and released
 type ReleaseableLayer interface {
 	Release() error
-	Mount() (string, error)
+	Mount() (containerfs.ContainerFS, error)
 	Commit(platform string) (ReleaseableLayer, error)
 	DiffID() layer.DiffID
 }

builder/dockerfile/builder.go

@@ -17,8 +17,6 @@ import (
 	"github.com/docker/docker/builder/dockerfile/parser"
 	"github.com/docker/docker/builder/fscache"
 	"github.com/docker/docker/builder/remotecontext"
-	"github.com/docker/docker/pkg/archive"
-	"github.com/docker/docker/pkg/chrootarchive"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/streamformatter"
 	"github.com/docker/docker/pkg/stringid"
@@ -50,21 +48,21 @@ type SessionGetter interface {
 
 // BuildManager is shared across all Builder objects
 type BuildManager struct {
-	archiver  *archive.Archiver
-	backend   builder.Backend
-	pathCache pathCache // TODO: make this persistent
-	sg        SessionGetter
-	fsCache   *fscache.FSCache
+	idMappings *idtools.IDMappings
+	backend    builder.Backend
+	pathCache  pathCache // TODO: make this persistent
+	sg         SessionGetter
+	fsCache    *fscache.FSCache
 }
 
 // NewBuildManager creates a BuildManager
 func NewBuildManager(b builder.Backend, sg SessionGetter, fsCache *fscache.FSCache, idMappings *idtools.IDMappings) (*BuildManager, error) {
 	bm := &BuildManager{
-		backend:   b,
-		pathCache: &syncmap.Map{},
-		sg:        sg,
-		archiver:  chrootarchive.NewArchiver(idMappings),
-		fsCache:   fsCache,
+		backend:    b,
+		pathCache:  &syncmap.Map{},
+		sg:         sg,
+		idMappings: idMappings,
+		fsCache:    fsCache,
 	}
 	if err := fsCache.RegisterTransport(remotecontext.ClientSessionRemote, NewClientSessionTransport()); err != nil {
 		return nil, err
@@ -114,7 +112,7 @@ func (bm *BuildManager) Build(ctx context.Context, config backend.BuildConfig) (
 		ProgressWriter: config.ProgressWriter,
 		Backend:        bm.backend,
 		PathCache:      bm.pathCache,
-		Archiver:       bm.archiver,
+		IDMappings:     bm.idMappings,
 		Platform:       dockerfile.Platform,
 	}
 
@@ -160,7 +158,7 @@ type builderOptions struct {
 	Backend        builder.Backend
 	ProgressWriter backend.ProgressWriter
 	PathCache      pathCache
-	Archiver       *archive.Archiver
+	IDMappings     *idtools.IDMappings
 	Platform       string
 }
 
@@ -177,7 +175,7 @@ type Builder struct {
 	docker    builder.Backend
 	clientCtx context.Context
 
-	archiver         *archive.Archiver
+	idMappings       *idtools.IDMappings
 	buildStages      *buildStages
 	disableCommit    bool
 	buildArgs        *buildArgs
@@ -219,7 +217,7 @@ func newBuilder(clientCtx context.Context, options builderOptions) *Builder {
 		Aux:              options.ProgressWriter.AuxFormatter,
 		Output:           options.ProgressWriter.Output,
 		docker:           options.Backend,
-		archiver:         options.Archiver,
+		idMappings:       options.IDMappings,
 		buildArgs:        newBuildArgs(config.BuildArgs),
 		buildStages:      newBuildStages(),
 		imageSources:     newImageSources(clientCtx, options),

builder/dockerfile/copy.go

@@ -1,6 +1,7 @@
 package dockerfile
 
 import (
+	"archive/tar"
 	"fmt"
 	"io"
 	"mime"
@@ -8,6 +9,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strings"
 	"time"
@@ -15,11 +17,11 @@ import (
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/builder/remotecontext"
 	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/docker/pkg/progress"
 	"github.com/docker/docker/pkg/streamformatter"
-	"github.com/docker/docker/pkg/symlink"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/docker/pkg/urlutil"
 	"github.com/pkg/errors"
@@ -35,14 +37,14 @@ type pathCache interface {
 // copyInfo is a data object which stores the metadata about each source file in
 // a copyInstruction
 type copyInfo struct {
-	root         string
+	root         containerfs.ContainerFS
 	path         string
 	hash         string
 	noDecompress bool
 }
 
 func (c copyInfo) fullPath() (string, error) {
-	return symlink.FollowSymlinkInScope(filepath.Join(c.root, c.path), c.root)
+	return c.root.ResolveScopedPath(c.path, true)
 }
 
 func newCopyInfoFromSource(source builder.Source, path string, hash string) copyInfo {
@@ -71,6 +73,7 @@ type copier struct {
 	pathCache   pathCache
 	download    sourceDownloader
 	tmpPaths    []string
+	platform    string
 }
 
 func copierFromDispatchRequest(req dispatchRequest, download sourceDownloader, imageSource *imageMount) copier {
@@ -79,6 +82,7 @@ func copierFromDispatchRequest(req dispatchRequest, download sourceDownloader, i
 		pathCache:   req.builder.pathCache,
 		download:    download,
 		imageSource: imageSource,
+		platform:    req.builder.platform,
 	}
 }
 
@@ -86,13 +90,14 @@ func (o *copier) createCopyInstruction(args []string, cmdName string) (copyInstr
 	inst := copyInstruction{cmdName: cmdName}
 	last := len(args) - 1
 
-	// Work in daemon-specific filepath semantics
-	inst.dest = filepath.FromSlash(args[last])
+	// Work in platform-specific filepath semantics
+	inst.dest = fromSlash(args[last], o.platform)
+	separator := string(separator(o.platform))
 	infos, err := o.getCopyInfosForSourcePaths(args[0:last], inst.dest)
 	if err != nil {
 		return inst, errors.Wrapf(err, "%s failed", cmdName)
 	}
-	if len(infos) > 1 && !strings.HasSuffix(inst.dest, string(os.PathSeparator)) {
+	if len(infos) > 1 && !strings.HasSuffix(inst.dest, separator) {
 		return inst, errors.Errorf("When using %s with more than one source file, the destination must be a directory and end with a /", cmdName)
 	}
 	inst.infos = infos
@@ -122,6 +127,7 @@ func (o *copier) getCopyInfoForSourcePath(orig, dest string) ([]copyInfo, error)
 	if !urlutil.IsURL(orig) {
 		return o.calcCopyInfo(orig, true)
 	}
+
 	remote, path, err := o.download(orig)
 	if err != nil {
 		return nil, err
@@ -134,7 +140,7 @@ func (o *copier) getCopyInfoForSourcePath(orig, dest string) ([]copyInfo, error)
 		}
 		path = unnamedFilename
 	}
-	o.tmpPaths = append(o.tmpPaths, remote.Root())
+	o.tmpPaths = append(o.tmpPaths, remote.Root().Path())
 
 	hash, err := remote.Hash(path)
 	ci := newCopyInfoFromSource(remote, path, hash)
@@ -154,14 +160,6 @@ func (o *copier) Cleanup() {
 // TODO: allowWildcards can probably be removed by refactoring this function further.
 func (o *copier) calcCopyInfo(origPath string, allowWildcards bool) ([]copyInfo, error) {
 	imageSource := o.imageSource
-	if err := validateCopySourcePath(imageSource, origPath); err != nil {
-		return nil, err
-	}
-
-	// Work in daemon-specific OS filepath semantics
-	origPath = filepath.FromSlash(origPath)
-	origPath = strings.TrimPrefix(origPath, string(os.PathSeparator))
-	origPath = strings.TrimPrefix(origPath, "."+string(os.PathSeparator))
 
 	// TODO: do this when creating copier. Requires validateCopySourcePath
 	// (and other below) to be aware of the difference sources. Why is it only
@@ -178,8 +176,20 @@ func (o *copier) calcCopyInfo(origPath string, allowWildcards bool) ([]copyInfo,
 		return nil, errors.Errorf("missing build context")
 	}
 
+	root := o.source.Root()
+
+	if err := validateCopySourcePath(imageSource, origPath, root.OS()); err != nil {
+		return nil, err
+	}
+
+	// Work in source OS specific filepath semantics
+	// For LCOW, this is NOT the daemon OS.
+	origPath = root.FromSlash(origPath)
+	origPath = strings.TrimPrefix(origPath, string(root.Separator()))
+	origPath = strings.TrimPrefix(origPath, "."+string(root.Separator()))
+
 	// Deal with wildcards
-	if allowWildcards && containsWildcards(origPath) {
+	if allowWildcards && containsWildcards(origPath, root.OS()) {
 		return o.copyWithWildcards(origPath)
 	}
 
@@ -211,6 +221,19 @@ func (o *copier) calcCopyInfo(origPath string, allowWildcards bool) ([]copyInfo,
 	return newCopyInfos(newCopyInfoFromSource(o.source, origPath, hash)), nil
 }
 
+func containsWildcards(name, platform string) bool {
+	isWindows := platform == "windows"
+	for i := 0; i < len(name); i++ {
+		ch := name[i]
+		if ch == '\\' && !isWindows {
+			i++
+		} else if ch == '*' || ch == '?' || ch == '[' {
+			return true
+		}
+	}
+	return false
+}
+
 func (o *copier) storeInPathCache(im *imageMount, path string, hash string) {
 	if im != nil {
 		o.pathCache.Store(im.ImageID()+path, hash)
@@ -218,12 +241,13 @@ func (o *copier) storeInPathCache(im *imageMount, path string, hash string) {
 }
 
 func (o *copier) copyWithWildcards(origPath string) ([]copyInfo, error) {
+	root := o.source.Root()
 	var copyInfos []copyInfo
-	if err := filepath.Walk(o.source.Root(), func(path string, info os.FileInfo, err error) error {
+	if err := root.Walk(root.Path(), func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
-		rel, err := remotecontext.Rel(o.source.Root(), path)
+		rel, err := remotecontext.Rel(root, path)
 		if err != nil {
 			return err
 		}
@@ -231,7 +255,7 @@ func (o *copier) copyWithWildcards(origPath string) ([]copyInfo, error) {
 		if rel == "." {
 			return nil
 		}
-		if match, _ := filepath.Match(origPath, rel); !match {
+		if match, _ := root.Match(origPath, rel); !match {
 			return nil
 		}
 
@@ -273,7 +297,7 @@ func walkSource(source builder.Source, origPath string) ([]string, error) {
 	}
 	// Must be a dir
 	var subfiles []string
-	err = filepath.Walk(fp, func(path string, info os.FileInfo, err error) error {
+	err = source.Root().Walk(fp, func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -398,14 +422,19 @@ func downloadSource(output io.Writer, stdout io.Writer, srcURL string) (remote b
 		return
 	}
 
-	lc, err := remotecontext.NewLazySource(tmpDir)
+	lc, err := remotecontext.NewLazySource(containerfs.NewLocalContainerFS(tmpDir))
 	return lc, filename, err
 }
 
 type copyFileOptions struct {
 	decompress bool
-	archiver   *archive.Archiver
 	chownPair  idtools.IDPair
+	archiver   Archiver
+}
+
+type copyEndpoint struct {
+	driver containerfs.Driver
+	path   string
 }
 
 func performCopyForInfo(dest copyInfo, source copyInfo, options copyFileOptions) error {
@@ -413,6 +442,7 @@ func performCopyForInfo(dest copyInfo, source copyInfo, options copyFileOptions)
 	if err != nil {
 		return err
 	}
+
 	destPath, err := dest.fullPath()
 	if err != nil {
 		return err
@@ -420,59 +450,90 @@ func performCopyForInfo(dest copyInfo, source copyInfo, options copyFileOptions)
 
 	archiver := options.archiver
 
-	src, err := os.Stat(srcPath)
+	srcEndpoint := &copyEndpoint{driver: source.root, path: srcPath}
+	destEndpoint := &copyEndpoint{driver: dest.root, path: destPath}
+
+	src, err := source.root.Stat(srcPath)
 	if err != nil {
 		return errors.Wrapf(err, "source path not found")
 	}
 	if src.IsDir() {
-		return copyDirectory(archiver, srcPath, destPath, options.chownPair)
+		return copyDirectory(archiver, srcEndpoint, destEndpoint, options.chownPair)
 	}
-	if options.decompress && archive.IsArchivePath(srcPath) && !source.noDecompress {
+	if options.decompress && isArchivePath(source.root, srcPath) && !source.noDecompress {
 		return archiver.UntarPath(srcPath, destPath)
 	}
 
-	destExistsAsDir, err := isExistingDirectory(destPath)
+	destExistsAsDir, err := isExistingDirectory(destEndpoint)
 	if err != nil {
 		return err
 	}
 	// dest.path must be used because destPath has already been cleaned of any
 	// trailing slash
-	if endsInSlash(dest.path) || destExistsAsDir {
+	if endsInSlash(dest.root, dest.path) || destExistsAsDir {
 		// source.path must be used to get the correct filename when the source
 		// is a symlink
-		destPath = filepath.Join(destPath, filepath.Base(source.path))
+		destPath = dest.root.Join(destPath, source.root.Base(source.path))
+		destEndpoint = &copyEndpoint{driver: dest.root, path: destPath}
 	}
-	return copyFile(archiver, srcPath, destPath, options.chownPair)
+	return copyFile(archiver, srcEndpoint, destEndpoint, options.chownPair)
 }
 
-func copyDirectory(archiver *archive.Archiver, source, dest string, chownPair idtools.IDPair) error {
+func isArchivePath(driver containerfs.ContainerFS, path string) bool {
+	file, err := driver.Open(path)
+	if err != nil {
+		return false
+	}
+	defer file.Close()
+	rdr, err := archive.DecompressStream(file)
+	if err != nil {
+		return false
+	}
+	r := tar.NewReader(rdr)
+	_, err = r.Next()
+	return err == nil
+}
+
+func copyDirectory(archiver Archiver, source, dest *copyEndpoint, chownPair idtools.IDPair) error {
 	destExists, err := isExistingDirectory(dest)
 	if err != nil {
 		return errors.Wrapf(err, "failed to query destination path")
 	}
-	if err := archiver.CopyWithTar(source, dest); err != nil {
+
+	if err := archiver.CopyWithTar(source.path, dest.path); err != nil {
 		return errors.Wrapf(err, "failed to copy directory")
 	}
-	return fixPermissions(source, dest, chownPair, !destExists)
+	// TODO: @gupta-ak. Investigate how LCOW permission mappings will work.
+	return fixPermissions(source.path, dest.path, chownPair, !destExists)
 }
 
-func copyFile(archiver *archive.Archiver, source, dest string, chownPair idtools.IDPair) error {
-	if err := idtools.MkdirAllAndChownNew(filepath.Dir(dest), 0755, chownPair); err != nil {
-		return errors.Wrapf(err, "failed to create new directory")
+func copyFile(archiver Archiver, source, dest *copyEndpoint, chownPair idtools.IDPair) error {
+	if runtime.GOOS == "windows" && dest.driver.OS() == "linux" {
+		// LCOW
+		if err := dest.driver.MkdirAll(dest.driver.Dir(dest.path), 0755); err != nil {
+			return errors.Wrapf(err, "failed to create new directory")
+		}
+	} else {
+		if err := idtools.MkdirAllAndChownNew(filepath.Dir(dest.path), 0755, chownPair); err != nil {
+			// Normal containers
+			return errors.Wrapf(err, "failed to create new directory")
+		}
 	}
-	if err := archiver.CopyFileWithTar(source, dest); err != nil {
+
+	if err := archiver.CopyFileWithTar(source.path, dest.path); err != nil {
 		return errors.Wrapf(err, "failed to copy file")
 	}
-	return fixPermissions(source, dest, chownPair, false)
+	// TODO: @gupta-ak. Investigate how LCOW permission mappings will work.
+	return fixPermissions(source.path, dest.path, chownPair, false)
 }
 
-func endsInSlash(path string) bool {
-	return strings.HasSuffix(path, string(os.PathSeparator))
+func endsInSlash(driver containerfs.Driver, path string) bool {
+	return strings.HasSuffix(path, string(driver.Separator()))
 }
 
 // isExistingDirectory returns true if the path exists and is a directory
-func isExistingDirectory(path string) (bool, error) {
-	destStat, err := os.Stat(path)
+func isExistingDirectory(point *copyEndpoint) (bool, error) {
+	destStat, err := point.driver.Stat(point.path)
 	switch {
 	case os.IsNotExist(err):
 		return false, nil

builder/dockerfile/copy_test.go

@@ -4,6 +4,7 @@ import (
 	"net/http"
 	"testing"
 
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/gotestyourself/gotestyourself/fs"
 	"github.com/stretchr/testify/assert"
 )
@@ -37,7 +38,7 @@ func TestIsExistingDirectory(t *testing.T) {
 	}
 
 	for _, testcase := range testcases {
-		result, err := isExistingDirectory(testcase.path)
+		result, err := isExistingDirectory(&copyEndpoint{driver: containerfs.NewLocalDriver(), path: testcase.path})
 		if !assert.NoError(t, err) {
 			continue
 		}

builder/dockerfile/copy_unix.go

@@ -6,6 +6,7 @@ import (
 	"os"
 	"path/filepath"
 
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 )
 
@@ -15,7 +16,8 @@ func fixPermissions(source, destination string, rootIDs idtools.IDPair, override
 		err           error
 	)
 	if !overrideSkip {
-		skipChownRoot, err = isExistingDirectory(destination)
+		destEndpoint := &copyEndpoint{driver: containerfs.NewLocalDriver(), path: destination}
+		skipChownRoot, err = isExistingDirectory(destEndpoint)
 		if err != nil {
 			return err
 		}
@@ -40,3 +42,7 @@ func fixPermissions(source, destination string, rootIDs idtools.IDPair, override
 		return os.Lchown(fullpath, rootIDs.UID, rootIDs.GID)
 	})
 }
+
+func validateCopySourcePath(imageSource *imageMount, origPath, platform string) error {
+	return nil
+}

builder/dockerfile/copy_windows.go

@@ -1,8 +1,38 @@
 package dockerfile
 
-import "github.com/docker/docker/pkg/idtools"
+import (
+	"errors"
+	"path/filepath"
+	"strings"
+
+	"github.com/docker/docker/pkg/idtools"
+)
 
 func fixPermissions(source, destination string, rootIDs idtools.IDPair, overrideSkip bool) error {
 	// chown is not supported on Windows
 	return nil
 }
+
+func validateCopySourcePath(imageSource *imageMount, origPath, platform string) error {
+	// validate windows paths from other images + LCOW
+	if imageSource == nil || platform != "windows" {
+		return nil
+	}
+
+	origPath = filepath.FromSlash(origPath)
+	p := strings.ToLower(filepath.Clean(origPath))
+	if !filepath.IsAbs(p) {
+		if filepath.VolumeName(p) != "" {
+			if p[len(p)-2:] == ":." { // case where clean returns weird c:. paths
+				p = p[:len(p)-1]
+			}
+			p += "\\"
+		} else {
+			p = filepath.Join("c:\\", p)
+		}
+	}
+	if _, blacklisted := pathBlacklist[p]; blacklisted {
+		return errors.New("copy from c:\\ or c:\\windows is not allowed on windows")
+	}
+	return nil
+}

builder/dockerfile/internals.go

@@ -7,6 +7,9 @@ import (
 	"crypto/sha256"
 	"encoding/hex"
 	"fmt"
+	"io"
+	"os"
+	"path"
 	"path/filepath"
 	"strconv"
 	"strings"
@@ -15,13 +18,69 @@ import (
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/api/types/container"
 	"github.com/docker/docker/image"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/chrootarchive"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/docker/pkg/symlink"
+	"github.com/docker/docker/pkg/system"
 	lcUser "github.com/opencontainers/runc/libcontainer/user"
 	"github.com/pkg/errors"
 )
 
+// For Windows only
+var pathBlacklist = map[string]bool{
+	"c:\\":        true,
+	"c:\\windows": true,
+}
+
+// Archiver defines an interface for copying files from one destination to
+// another using Tar/Untar.
+type Archiver interface {
+	TarUntar(src, dst string) error
+	UntarPath(src, dst string) error
+	CopyWithTar(src, dst string) error
+	CopyFileWithTar(src, dst string) error
+	IDMappings() *idtools.IDMappings
+}
+
+// The builder will use the following interfaces if the container fs implements
+// these for optimized copies to and from the container.
+type extractor interface {
+	ExtractArchive(src io.Reader, dst string, opts *archive.TarOptions) error
+}
+
+type archiver interface {
+	ArchivePath(src string, opts *archive.TarOptions) (io.ReadCloser, error)
+}
+
+// helper functions to get tar/untar func
+func untarFunc(i interface{}) containerfs.UntarFunc {
+	if ea, ok := i.(extractor); ok {
+		return ea.ExtractArchive
+	}
+	return chrootarchive.Untar
+}
+
+func tarFunc(i interface{}) containerfs.TarFunc {
+	if ap, ok := i.(archiver); ok {
+		return ap.ArchivePath
+	}
+	return archive.TarWithOptions
+}
+
+func (b *Builder) getArchiver(src, dst containerfs.Driver) Archiver {
+	t, u := tarFunc(src), untarFunc(dst)
+	return &containerfs.Archiver{
+		SrcDriver:     src,
+		DstDriver:     dst,
+		Tar:           t,
+		Untar:         u,
+		IDMappingsVar: b.idMappings,
+	}
+}
+
 func (b *Builder) commit(dispatchState *dispatchState, comment string) error {
 	if b.disableCommit {
 		return nil
@@ -131,28 +190,29 @@ func (b *Builder) performCopy(state *dispatchState, inst copyInstruction) error
 	if err != nil {
 		return errors.Wrapf(err, "failed to get destination image %q", state.imageID)
 	}
-	destInfo, err := createDestInfo(state.runConfig.WorkingDir, inst, imageMount)
+
+	destInfo, err := createDestInfo(state.runConfig.WorkingDir, inst, imageMount, b.platform)
 	if err != nil {
 		return err
 	}
 
-	chownPair := b.archiver.IDMappings.RootPair()
+	chownPair := b.idMappings.RootPair()
 	// if a chown was requested, perform the steps to get the uid, gid
 	// translated (if necessary because of user namespaces), and replace
 	// the root pair with the chown pair for copy operations
 	if inst.chownStr != "" {
-		chownPair, err = parseChownFlag(inst.chownStr, destInfo.root, b.archiver.IDMappings)
+		chownPair, err = parseChownFlag(inst.chownStr, destInfo.root.Path(), b.idMappings)
 		if err != nil {
 			return errors.Wrapf(err, "unable to convert uid/gid chown string to host mapping")
 		}
 	}
 
-	opts := copyFileOptions{
-		decompress: inst.allowLocalDecompression,
-		archiver:   b.archiver,
-		chownPair:  chownPair,
-	}
 	for _, info := range inst.infos {
+		opts := copyFileOptions{
+			decompress: inst.allowLocalDecompression,
+			archiver:   b.getArchiver(info.root, destInfo.root),
+			chownPair:  chownPair,
+		}
 		if err := performCopyForInfo(destInfo, info, opts); err != nil {
 			return errors.Wrapf(err, "failed to copy files")
 		}
@@ -236,10 +296,10 @@ func lookupGroup(groupStr, filepath string) (int, error) {
 	return groups[0].Gid, nil
 }
 
-func createDestInfo(workingDir string, inst copyInstruction, imageMount *imageMount) (copyInfo, error) {
+func createDestInfo(workingDir string, inst copyInstruction, imageMount *imageMount, platform string) (copyInfo, error) {
 	// Twiddle the destination when it's a relative path - meaning, make it
 	// relative to the WORKINGDIR
-	dest, err := normalizeDest(workingDir, inst.dest)
+	dest, err := normalizeDest(workingDir, inst.dest, platform)
 	if err != nil {
 		return copyInfo{}, errors.Wrapf(err, "invalid %s", inst.cmdName)
 	}
@@ -252,6 +312,63 @@ func createDestInfo(workingDir string, inst copyInstruction, imageMount *imageMo
 	return newCopyInfoFromSource(destMount, dest, ""), nil
 }
 
+// normalizeDest normalises the destination of a COPY/ADD command in a
+// platform semantically consistent way.
+func normalizeDest(workingDir, requested string, platform string) (string, error) {
+	dest := fromSlash(requested, platform)
+	endsInSlash := strings.HasSuffix(dest, string(separator(platform)))
+
+	if platform != "windows" {
+		if !path.IsAbs(requested) {
+			dest = path.Join("/", filepath.ToSlash(workingDir), dest)
+			// Make sure we preserve any trailing slash
+			if endsInSlash {
+				dest += "/"
+			}
+		}
+		return dest, nil
+	}
+
+	// We are guaranteed that the working directory is already consistent,
+	// However, Windows also has, for now, the limitation that ADD/COPY can
+	// only be done to the system drive, not any drives that might be present
+	// as a result of a bind mount.
+	//
+	// So... if the path requested is Linux-style absolute (/foo or \\foo),
+	// we assume it is the system drive. If it is a Windows-style absolute
+	// (DRIVE:\\foo), error if DRIVE is not C. And finally, ensure we
+	// strip any configured working directories drive letter so that it
+	// can be subsequently legitimately converted to a Windows volume-style
+	// pathname.
+
+	// Not a typo - filepath.IsAbs, not system.IsAbs on this next check as
+	// we only want to validate where the DriveColon part has been supplied.
+	if filepath.IsAbs(dest) {
+		if strings.ToUpper(string(dest[0])) != "C" {
+			return "", fmt.Errorf("Windows does not support destinations not on the system drive (C:)")
+		}
+		dest = dest[2:] // Strip the drive letter
+	}
+
+	// Cannot handle relative where WorkingDir is not the system drive.
+	if len(workingDir) > 0 {
+		if ((len(workingDir) > 1) && !system.IsAbs(workingDir[2:])) || (len(workingDir) == 1) {
+			return "", fmt.Errorf("Current WorkingDir %s is not platform consistent", workingDir)
+		}
+		if !system.IsAbs(dest) {
+			if string(workingDir[0]) != "C" {
+				return "", fmt.Errorf("Windows does not support relative paths when WORKDIR is not the system drive")
+			}
+			dest = filepath.Join(string(os.PathSeparator), workingDir[2:], dest)
+			// Make sure we preserve any trailing slash
+			if endsInSlash {
+				dest += string(os.PathSeparator)
+			}
+		}
+	}
+	return dest, nil
+}
+
 // For backwards compat, if there's just one info then use it as the
 // cache look-up string, otherwise hash 'em all into one
 func getSourceHashFromInfos(infos []copyInfo) string {
@@ -397,3 +514,19 @@ func hostConfigFromOptions(options *types.ImageBuildOptions) *container.HostConf
 		ExtraHosts: options.ExtraHosts,
 	}
 }
+
+// fromSlash works like filepath.FromSlash but with a given OS platform field
+func fromSlash(path, platform string) string {
+	if platform == "windows" {
+		return strings.Replace(path, "/", "\\", -1)
+	}
+	return path
+}
+
+// separator returns a OS path separator for the given OS platform
+func separator(platform string) byte {
+	if platform == "windows" {
+		return '\\'
+	}
+	return '/'
+}

builder/dockerfile/internals_unix.go

@@ -1,42 +0,0 @@
-// +build !windows
-
-package dockerfile
-
-import (
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/docker/docker/pkg/system"
-)
-
-// normalizeDest normalizes the destination of a COPY/ADD command in a
-// platform semantically consistent way.
-func normalizeDest(workingDir, requested string) (string, error) {
-	dest := filepath.FromSlash(requested)
-	endsInSlash := strings.HasSuffix(requested, string(os.PathSeparator))
-	if !system.IsAbs(requested) {
-		dest = filepath.Join(string(os.PathSeparator), filepath.FromSlash(workingDir), dest)
-		// Make sure we preserve any trailing slash
-		if endsInSlash {
-			dest += string(os.PathSeparator)
-		}
-	}
-	return dest, nil
-}
-
-func containsWildcards(name string) bool {
-	for i := 0; i < len(name); i++ {
-		ch := name[i]
-		if ch == '\\' {
-			i++
-		} else if ch == '*' || ch == '?' || ch == '[' {
-			return true
-		}
-	}
-	return false
-}
-
-func validateCopySourcePath(imageSource *imageMount, origPath string) error {
-	return nil
-}

builder/dockerfile/internals_windows.go

@@ -1,95 +0,0 @@
-package dockerfile
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/docker/docker/pkg/system"
-	"github.com/pkg/errors"
-)
-
-// normalizeDest normalizes the destination of a COPY/ADD command in a
-// platform semantically consistent way.
-func normalizeDest(workingDir, requested string) (string, error) {
-	dest := filepath.FromSlash(requested)
-	endsInSlash := strings.HasSuffix(dest, string(os.PathSeparator))
-
-	// We are guaranteed that the working directory is already consistent,
-	// However, Windows also has, for now, the limitation that ADD/COPY can
-	// only be done to the system drive, not any drives that might be present
-	// as a result of a bind mount.
-	//
-	// So... if the path requested is Linux-style absolute (/foo or \\foo),
-	// we assume it is the system drive. If it is a Windows-style absolute
-	// (DRIVE:\\foo), error if DRIVE is not C. And finally, ensure we
-	// strip any configured working directories drive letter so that it
-	// can be subsequently legitimately converted to a Windows volume-style
-	// pathname.
-
-	// Not a typo - filepath.IsAbs, not system.IsAbs on this next check as
-	// we only want to validate where the DriveColon part has been supplied.
-	if filepath.IsAbs(dest) {
-		if strings.ToUpper(string(dest[0])) != "C" {
-			return "", fmt.Errorf("Windows does not support destinations not on the system drive (C:)")
-		}
-		dest = dest[2:] // Strip the drive letter
-	}
-
-	// Cannot handle relative where WorkingDir is not the system drive.
-	if len(workingDir) > 0 {
-		if ((len(workingDir) > 1) && !system.IsAbs(workingDir[2:])) || (len(workingDir) == 1) {
-			return "", fmt.Errorf("Current WorkingDir %s is not platform consistent", workingDir)
-		}
-		if !system.IsAbs(dest) {
-			if string(workingDir[0]) != "C" {
-				return "", fmt.Errorf("Windows does not support relative paths when WORKDIR is not the system drive")
-			}
-			dest = filepath.Join(string(os.PathSeparator), workingDir[2:], dest)
-			// Make sure we preserve any trailing slash
-			if endsInSlash {
-				dest += string(os.PathSeparator)
-			}
-		}
-	}
-	return dest, nil
-}
-
-func containsWildcards(name string) bool {
-	for i := 0; i < len(name); i++ {
-		ch := name[i]
-		if ch == '*' || ch == '?' || ch == '[' {
-			return true
-		}
-	}
-	return false
-}
-
-var pathBlacklist = map[string]bool{
-	"c:\\":        true,
-	"c:\\windows": true,
-}
-
-func validateCopySourcePath(imageSource *imageMount, origPath string) error {
-	// validate windows paths from other images
-	if imageSource == nil {
-		return nil
-	}
-	origPath = filepath.FromSlash(origPath)
-	p := strings.ToLower(filepath.Clean(origPath))
-	if !filepath.IsAbs(p) {
-		if filepath.VolumeName(p) != "" {
-			if p[len(p)-2:] == ":." { // case where clean returns weird c:. paths
-				p = p[:len(p)-1]
-			}
-			p += "\\"
-		} else {
-			p = filepath.Join("c:\\", p)
-		}
-	}
-	if _, blacklisted := pathBlacklist[p]; blacklisted {
-		return errors.New("copy from c:\\ or c:\\windows is not allowed on windows")
-	}
-	return nil
-}

builder/dockerfile/internals_windows_test.go

@@ -40,7 +40,7 @@ func TestNormalizeDest(t *testing.T) {
 	}
 	for _, testcase := range tests {
 		msg := fmt.Sprintf("Input: %s, %s", testcase.current, testcase.requested)
-		actual, err := normalizeDest(testcase.current, testcase.requested)
+		actual, err := normalizeDest(testcase.current, testcase.requested, "windows")
 		if testcase.etext == "" {
 			if !assert.NoError(t, err, msg) {
 				continue

builder/dockerfile/mockbackend_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/docker/docker/builder"
 	containerpkg "github.com/docker/docker/container"
 	"github.com/docker/docker/layer"
+	"github.com/docker/docker/pkg/containerfs"
 	"golang.org/x/net/context"
 )
 
@@ -117,8 +118,8 @@ func (l *mockLayer) Release() error {
 	return nil
 }
 
-func (l *mockLayer) Mount() (string, error) {
-	return "mountPath", nil
+func (l *mockLayer) Mount() (containerfs.ContainerFS, error) {
+	return containerfs.NewLocalContainerFS("mountPath"), nil
 }
 
 func (l *mockLayer) Commit(string) (builder.ReleaseableLayer, error) {

builder/fscache/fscache_test.go

@@ -36,25 +36,25 @@ func TestFSCache(t *testing.T) {
 	src1, err := fscache.SyncFrom(context.TODO(), &testIdentifier{"foo", "data", "bar"})
 	assert.Nil(t, err)
 
-	dt, err := ioutil.ReadFile(filepath.Join(src1.Root(), "foo"))
+	dt, err := ioutil.ReadFile(filepath.Join(src1.Root().Path(), "foo"))
 	assert.Nil(t, err)
 	assert.Equal(t, string(dt), "data")
 
 	// same id doesn't recalculate anything
 	src2, err := fscache.SyncFrom(context.TODO(), &testIdentifier{"foo", "data2", "bar"})
 	assert.Nil(t, err)
-	assert.Equal(t, src1.Root(), src2.Root())
+	assert.Equal(t, src1.Root().Path(), src2.Root().Path())
 
-	dt, err = ioutil.ReadFile(filepath.Join(src1.Root(), "foo"))
+	dt, err = ioutil.ReadFile(filepath.Join(src1.Root().Path(), "foo"))
 	assert.Nil(t, err)
 	assert.Equal(t, string(dt), "data")
 	assert.Nil(t, src2.Close())
 
 	src3, err := fscache.SyncFrom(context.TODO(), &testIdentifier{"foo2", "data2", "bar"})
 	assert.Nil(t, err)
-	assert.NotEqual(t, src1.Root(), src3.Root())
+	assert.NotEqual(t, src1.Root().Path(), src3.Root().Path())
 
-	dt, err = ioutil.ReadFile(filepath.Join(src3.Root(), "foo2"))
+	dt, err = ioutil.ReadFile(filepath.Join(src3.Root().Path(), "foo2"))
 	assert.Nil(t, err)
 	assert.Equal(t, string(dt), "data2")
 
@@ -71,12 +71,12 @@ func TestFSCache(t *testing.T) {
 	// new upload with the same shared key shoutl overwrite
 	src4, err := fscache.SyncFrom(context.TODO(), &testIdentifier{"foo3", "data3", "bar"})
 	assert.Nil(t, err)
-	assert.NotEqual(t, src1.Root(), src3.Root())
+	assert.NotEqual(t, src1.Root().Path(), src3.Root().Path())
 
-	dt, err = ioutil.ReadFile(filepath.Join(src3.Root(), "foo3"))
+	dt, err = ioutil.ReadFile(filepath.Join(src3.Root().Path(), "foo3"))
 	assert.Nil(t, err)
 	assert.Equal(t, string(dt), "data3")
-	assert.Equal(t, src4.Root(), src3.Root())
+	assert.Equal(t, src4.Root().Path(), src3.Root().Path())
 	assert.Nil(t, src4.Close())
 
 	s, err = fscache.DiskUsage()

builder/remotecontext/archive.go

@@ -8,19 +8,19 @@ import (
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/chrootarchive"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/ioutils"
-	"github.com/docker/docker/pkg/symlink"
 	"github.com/docker/docker/pkg/tarsum"
 	"github.com/pkg/errors"
 )
 
 type archiveContext struct {
-	root string
+	root containerfs.ContainerFS
 	sums tarsum.FileInfoSums
 }
 
 func (c *archiveContext) Close() error {
-	return os.RemoveAll(c.root)
+	return c.root.RemoveAll(c.root.Path())
 }
 
 func convertPathError(err error, cleanpath string) error {
@@ -52,7 +52,8 @@ func FromArchive(tarStream io.Reader) (builder.Source, error) {
 		return nil, err
 	}
 
-	tsc := &archiveContext{root: root}
+	// Assume local file system. Since it's coming from a tar file.
+	tsc := &archiveContext{root: containerfs.NewLocalContainerFS(root)}
 
 	// Make sure we clean-up upon error.  In the happy case the caller
 	// is expected to manage the clean-up
@@ -82,7 +83,7 @@ func FromArchive(tarStream io.Reader) (builder.Source, error) {
 	return tsc, nil
 }
 
-func (c *archiveContext) Root() string {
+func (c *archiveContext) Root() containerfs.ContainerFS {
 	return c.root
 }
 
@@ -91,7 +92,7 @@ func (c *archiveContext) Remove(path string) error {
 	if err != nil {
 		return err
 	}
-	return os.RemoveAll(fullpath)
+	return c.root.RemoveAll(fullpath)
 }
 
 func (c *archiveContext) Hash(path string) (string, error) {
@@ -100,7 +101,7 @@ func (c *archiveContext) Hash(path string) (string, error) {
 		return "", err
 	}
 
-	rel, err := filepath.Rel(c.root, fullpath)
+	rel, err := c.root.Rel(c.root.Path(), fullpath)
 	if err != nil {
 		return "", convertPathError(err, cleanpath)
 	}
@@ -115,13 +116,13 @@ func (c *archiveContext) Hash(path string) (string, error) {
 	return path, nil // backwards compat TODO: see if really needed
 }
 
-func normalize(path, root string) (cleanPath, fullPath string, err error) {
-	cleanPath = filepath.Clean(string(os.PathSeparator) + path)[1:]
-	fullPath, err = symlink.FollowSymlinkInScope(filepath.Join(root, path), root)
+func normalize(path string, root containerfs.ContainerFS) (cleanPath, fullPath string, err error) {
+	cleanPath = root.Clean(string(root.Separator()) + path)[1:]
+	fullPath, err = root.ResolveScopedPath(path, true)
 	if err != nil {
 		return "", "", errors.Wrapf(err, "forbidden path outside the build context: %s (%s)", path, cleanPath)
 	}
-	if _, err := os.Lstat(fullPath); err != nil {
+	if _, err := root.Lstat(fullPath); err != nil {
 		return "", "", errors.WithStack(convertPathError(err, path))
 	}
 	return

builder/remotecontext/detect.go

@@ -5,15 +5,14 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"path/filepath"
 	"strings"
 
+	"github.com/containerd/continuity/driver"
 	"github.com/docker/docker/api/types/backend"
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/builder/dockerfile/parser"
 	"github.com/docker/docker/builder/dockerignore"
 	"github.com/docker/docker/pkg/fileutils"
-	"github.com/docker/docker/pkg/symlink"
 	"github.com/docker/docker/pkg/urlutil"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -157,12 +156,12 @@ func readAndParseDockerfile(name string, rc io.Reader) (*parser.Result, error) {
 	return parser.Parse(br)
 }
 
-func openAt(remote builder.Source, path string) (*os.File, error) {
+func openAt(remote builder.Source, path string) (driver.File, error) {
 	fullPath, err := FullPath(remote, path)
 	if err != nil {
 		return nil, err
 	}
-	return os.Open(fullPath)
+	return remote.Root().Open(fullPath)
 }
 
 // StatAt is a helper for calling Stat on a path from a source
@@ -171,12 +170,12 @@ func StatAt(remote builder.Source, path string) (os.FileInfo, error) {
 	if err != nil {
 		return nil, err
 	}
-	return os.Stat(fullPath)
+	return remote.Root().Stat(fullPath)
 }
 
 // FullPath is a helper for getting a full path for a path from a source
 func FullPath(remote builder.Source, path string) (string, error) {
-	fullPath, err := symlink.FollowSymlinkInScope(filepath.Join(remote.Root(), path), remote.Root())
+	fullPath, err := remote.Root().ResolveScopedPath(path, true)
 	if err != nil {
 		return "", fmt.Errorf("Forbidden path outside the build context: %s (%s)", path, fullPath) // backwards compat with old error
 	}

builder/remotecontext/detect_test.go

@@ -5,11 +5,11 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
-	"path/filepath"
 	"sort"
 	"testing"
 
 	"github.com/docker/docker/builder"
+	"github.com/docker/docker/pkg/containerfs"
 )
 
 const (
@@ -53,7 +53,7 @@ func checkDirectory(t *testing.T, dir string, expectedFiles []string) {
 }
 
 func executeProcess(t *testing.T, contextDir string) {
-	modifiableCtx := &stubRemote{root: contextDir}
+	modifiableCtx := &stubRemote{root: containerfs.NewLocalContainerFS(contextDir)}
 
 	err := removeDockerfile(modifiableCtx, builder.DefaultDockerfileName)
 
@@ -105,19 +105,19 @@ func TestProcessShouldLeaveAllFiles(t *testing.T) {
 
 // TODO: remove after moving to a separate pkg
 type stubRemote struct {
-	root string
+	root containerfs.ContainerFS
 }
 
 func (r *stubRemote) Hash(path string) (string, error) {
 	return "", errors.New("not implemented")
 }
 
-func (r *stubRemote) Root() string {
+func (r *stubRemote) Root() containerfs.ContainerFS {
 	return r.root
 }
 func (r *stubRemote) Close() error {
 	return errors.New("not implemented")
 }
 func (r *stubRemote) Remove(p string) error {
-	return os.Remove(filepath.Join(r.root, p))
+	return r.root.Remove(r.root.Join(r.root.Path(), p))
 }

builder/remotecontext/lazycontext.go

@@ -3,11 +3,10 @@ package remotecontext
 import (
 	"encoding/hex"
 	"os"
-	"path/filepath"
-	"runtime"
 	"strings"
 
 	"github.com/docker/docker/builder"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/pools"
 	"github.com/pkg/errors"
 )
@@ -15,7 +14,7 @@ import (
 // NewLazySource creates a new LazyContext. LazyContext defines a hashed build
 // context based on a root directory. Individual files are hashed first time
 // they are asked. It is not safe to call methods of LazyContext concurrently.
-func NewLazySource(root string) (builder.Source, error) {
+func NewLazySource(root containerfs.ContainerFS) (builder.Source, error) {
 	return &lazySource{
 		root: root,
 		sums: make(map[string]string),
@@ -23,11 +22,11 @@ func NewLazySource(root string) (builder.Source, error) {
 }
 
 type lazySource struct {
-	root string
+	root containerfs.ContainerFS
 	sums map[string]string
 }
 
-func (c *lazySource) Root() string {
+func (c *lazySource) Root() containerfs.ContainerFS {
 	return c.root
 }
 
@@ -41,7 +40,7 @@ func (c *lazySource) Hash(path string) (string, error) {
 		return "", err
 	}
 
-	fi, err := os.Lstat(fullPath)
+	fi, err := c.root.Lstat(fullPath)
 	if err != nil {
 		return "", errors.WithStack(err)
 	}
@@ -63,13 +62,13 @@ func (c *lazySource) Hash(path string) (string, error) {
 }
 
 func (c *lazySource) prepareHash(relPath string, fi os.FileInfo) (string, error) {
-	p := filepath.Join(c.root, relPath)
+	p := c.root.Join(c.root.Path(), relPath)
 	h, err := NewFileHash(p, relPath, fi)
 	if err != nil {
 		return "", errors.Wrapf(err, "failed to create hash for %s", relPath)
 	}
 	if fi.Mode().IsRegular() && fi.Size() > 0 {
-		f, err := os.Open(p)
+		f, err := c.root.Open(p)
 		if err != nil {
 			return "", errors.Wrapf(err, "failed to open %s", relPath)
 		}
@@ -85,10 +84,10 @@ func (c *lazySource) prepareHash(relPath string, fi os.FileInfo) (string, error)
 
 // Rel makes a path relative to base path. Same as `filepath.Rel` but can also
 // handle UUID paths in windows.
-func Rel(basepath, targpath string) (string, error) {
+func Rel(basepath containerfs.ContainerFS, targpath string) (string, error) {
 	// filepath.Rel can't handle UUID paths in windows
-	if runtime.GOOS == "windows" {
-		pfx := basepath + `\`
+	if basepath.OS() == "windows" {
+		pfx := basepath.Path() + `\`
 		if strings.HasPrefix(targpath, pfx) {
 			p := strings.TrimPrefix(targpath, pfx)
 			if p == "" {
@@ -97,5 +96,5 @@ func Rel(basepath, targpath string) (string, error) {
 			return p, nil
 		}
 	}
-	return filepath.Rel(basepath, targpath)
+	return basepath.Rel(basepath.Path(), targpath)
 }

builder/remotecontext/tarsum.go

@@ -3,11 +3,11 @@ package remotecontext
 import (
 	"fmt"
 	"os"
-	"path/filepath"
 	"sync"
 
-	"github.com/docker/docker/pkg/symlink"
 	iradix "github.com/hashicorp/go-immutable-radix"
+
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/pkg/errors"
 	"github.com/tonistiigi/fsutil"
 )
@@ -19,7 +19,7 @@ type hashed interface {
 // CachableSource is a source that contains cache records for its contents
 type CachableSource struct {
 	mu   sync.Mutex
-	root string
+	root containerfs.ContainerFS
 	tree *iradix.Tree
 	txn  *iradix.Txn
 }
@@ -28,7 +28,7 @@ type CachableSource struct {
 func NewCachableSource(root string) *CachableSource {
 	ts := &CachableSource{
 		tree: iradix.New(),
-		root: root,
+		root: containerfs.NewLocalContainerFS(root),
 	}
 	return ts
 }
@@ -67,7 +67,7 @@ func (cs *CachableSource) Scan() error {
 		return err
 	}
 	txn := iradix.New().Txn()
-	err = filepath.Walk(cs.root, func(path string, info os.FileInfo, err error) error {
+	err = cs.root.Walk(cs.root.Path(), func(path string, info os.FileInfo, err error) error {
 		if err != nil {
 			return errors.Wrapf(err, "failed to walk %s", path)
 		}
@@ -134,12 +134,12 @@ func (cs *CachableSource) Close() error {
 }
 
 func (cs *CachableSource) normalize(path string) (cleanpath, fullpath string, err error) {
-	cleanpath = filepath.Clean(string(os.PathSeparator) + path)[1:]
-	fullpath, err = symlink.FollowSymlinkInScope(filepath.Join(cs.root, path), cs.root)
+	cleanpath = cs.root.Clean(string(cs.root.Separator()) + path)[1:]
+	fullpath, err = cs.root.ResolveScopedPath(path, true)
 	if err != nil {
 		return "", "", fmt.Errorf("Forbidden path outside the context: %s (%s)", path, fullpath)
 	}
-	_, err = os.Lstat(fullpath)
+	_, err = cs.root.Lstat(fullpath)
 	if err != nil {
 		return "", "", convertPathError(err, path)
 	}
@@ -158,7 +158,7 @@ func (cs *CachableSource) Hash(path string) (string, error) {
 }
 
 // Root returns a root directory for the source
-func (cs *CachableSource) Root() string {
+func (cs *CachableSource) Root() containerfs.ContainerFS {
 	return cs.root
 }
 

builder/remotecontext/tarsum.pb.go

@@ -94,7 +94,7 @@ func (this *TarsumBackup) GoString() string {
 	s := make([]string, 0, 5)
 	s = append(s, "&remotecontext.TarsumBackup{")
 	keysForHashes := make([]string, 0, len(this.Hashes))
-	for k, _ := range this.Hashes {
+	for k := range this.Hashes {
 		keysForHashes = append(keysForHashes, k)
 	}
 	github_com_gogo_protobuf_sortkeys.Strings(keysForHashes)
@@ -133,7 +133,7 @@ func (m *TarsumBackup) MarshalTo(dAtA []byte) (int, error) {
 	var l int
 	_ = l
 	if len(m.Hashes) > 0 {
-		for k, _ := range m.Hashes {
+		for k := range m.Hashes {
 			dAtA[i] = 0xa
 			i++
 			v := m.Hashes[k]
@@ -211,7 +211,7 @@ func (this *TarsumBackup) String() string {
 		return "nil"
 	}
 	keysForHashes := make([]string, 0, len(this.Hashes))
-	for k, _ := range this.Hashes {
+	for k := range this.Hashes {
 		keysForHashes = append(keysForHashes, k)
 	}
 	github_com_gogo_protobuf_sortkeys.Strings(keysForHashes)

builder/remotecontext/tarsum_test.go

@@ -35,7 +35,7 @@ func TestCloseRootDirectory(t *testing.T) {
 		t.Fatalf("Error while executing Close: %s", err)
 	}
 
-	_, err = os.Stat(src.Root())
+	_, err = os.Stat(src.Root().Path())
 
 	if !os.IsNotExist(err) {
 		t.Fatal("Directory should not exist at this point")

container/archive.go

@@ -2,7 +2,6 @@ package container
 
 import (
 	"os"
-	"path/filepath"
 
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/pkg/archive"
@@ -15,17 +14,20 @@ import (
 // an error if the path points to outside the container's rootfs.
 func (container *Container) ResolvePath(path string) (resolvedPath, absPath string, err error) {
 	// Check if a drive letter supplied, it must be the system drive. No-op except on Windows
-	path, err = system.CheckSystemDriveAndRemoveDriveLetter(path)
+	path, err = system.CheckSystemDriveAndRemoveDriveLetter(path, container.BaseFS)
 	if err != nil {
 		return "", "", err
 	}
 
 	// Consider the given path as an absolute path in the container.
-	absPath = archive.PreserveTrailingDotOrSeparator(filepath.Join(string(filepath.Separator), path), path)
+	absPath = archive.PreserveTrailingDotOrSeparator(
+		container.BaseFS.Join(string(container.BaseFS.Separator()), path),
+		path,
+		container.BaseFS.Separator())
 
 	// Split the absPath into its Directory and Base components. We will
 	// resolve the dir in the scope of the container then append the base.
-	dirPath, basePath := filepath.Split(absPath)
+	dirPath, basePath := container.BaseFS.Split(absPath)
 
 	resolvedDirPath, err := container.GetResourcePath(dirPath)
 	if err != nil {
@@ -34,8 +36,7 @@ func (container *Container) ResolvePath(path string) (resolvedPath, absPath stri
 
 	// resolvedDirPath will have been cleaned (no trailing path separators) so
 	// we can manually join it with the base path element.
-	resolvedPath = resolvedDirPath + string(filepath.Separator) + basePath
-
+	resolvedPath = resolvedDirPath + string(container.BaseFS.Separator()) + basePath
 	return resolvedPath, absPath, nil
 }
 
@@ -44,7 +45,9 @@ func (container *Container) ResolvePath(path string) (resolvedPath, absPath stri
 // resolved to a path on the host corresponding to the given absolute path
 // inside the container.
 func (container *Container) StatPath(resolvedPath, absPath string) (stat *types.ContainerPathStat, err error) {
-	lstat, err := os.Lstat(resolvedPath)
+	driver := container.BaseFS
+
+	lstat, err := driver.Lstat(resolvedPath)
 	if err != nil {
 		return nil, err
 	}
@@ -57,17 +60,17 @@ func (container *Container) StatPath(resolvedPath, absPath string) (stat *types.
 			return nil, err
 		}
 
-		linkTarget, err = filepath.Rel(container.BaseFS, hostPath)
+		linkTarget, err = driver.Rel(driver.Path(), hostPath)
 		if err != nil {
 			return nil, err
 		}
 
 		// Make it an absolute path.
-		linkTarget = filepath.Join(string(filepath.Separator), linkTarget)
+		linkTarget = driver.Join(string(driver.Separator()), linkTarget)
 	}
 
 	return &types.ContainerPathStat{
-		Name:       filepath.Base(absPath),
+		Name:       driver.Base(absPath),
 		Size:       lstat.Size(),
 		Mode:       lstat.Mode(),
 		Mtime:      lstat.ModTime(),

container/container.go

@@ -28,6 +28,7 @@ import (
 	"github.com/docker/docker/layer"
 	"github.com/docker/docker/libcontainerd"
 	"github.com/docker/docker/opts"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/docker/pkg/signal"
@@ -64,10 +65,10 @@ var (
 type Container struct {
 	StreamConfig *stream.Config
 	// embed for Container to support states directly.
-	*State          `json:"State"` // Needed for Engine API version <= 1.11
-	Root            string         `json:"-"` // Path to the "home" of the container, including metadata.
-	BaseFS          string         `json:"-"` // Path to the graphdriver mountpoint
-	RWLayer         layer.RWLayer  `json:"-"`
+	*State          `json:"State"`          // Needed for Engine API version <= 1.11
+	Root            string                  `json:"-"` // Path to the "home" of the container, including metadata.
+	BaseFS          containerfs.ContainerFS `json:"-"` // interface containing graphdriver mount
+	RWLayer         layer.RWLayer           `json:"-"`
 	ID              string
 	Created         time.Time
 	Managed         bool
@@ -305,15 +306,13 @@ func (container *Container) SetupWorkingDirectory(rootIDs idtools.IDPair) error
 func (container *Container) GetResourcePath(path string) (string, error) {
 	// IMPORTANT - These are paths on the OS where the daemon is running, hence
 	// any filepath operations must be done in an OS agnostic way.
-
-	cleanPath := cleanResourcePath(path)
-	r, e := symlink.FollowSymlinkInScope(filepath.Join(container.BaseFS, cleanPath), container.BaseFS)
+	r, e := container.BaseFS.ResolveScopedPath(path, false)
 
 	// Log this here on the daemon side as there's otherwise no indication apart
 	// from the error being propagated all the way back to the client. This makes
 	// debugging significantly easier and clearly indicates the error comes from the daemon.
 	if e != nil {
-		logrus.Errorf("Failed to FollowSymlinkInScope BaseFS %s cleanPath %s path %s %s\n", container.BaseFS, cleanPath, path, e)
+		logrus.Errorf("Failed to ResolveScopedPath BaseFS %s path %s %s\n", container.BaseFS.Path(), path, e)
 	}
 	return r, e
 }

container/container_unix.go

@@ -5,7 +5,6 @@ package container
 import (
 	"io/ioutil"
 	"os"
-	"path/filepath"
 
 	"github.com/docker/docker/api/types"
 	containertypes "github.com/docker/docker/api/types/container"
@@ -13,7 +12,6 @@ import (
 	"github.com/docker/docker/pkg/chrootarchive"
 	"github.com/docker/docker/pkg/mount"
 	"github.com/docker/docker/pkg/stringid"
-	"github.com/docker/docker/pkg/symlink"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/docker/volume"
 	"github.com/opencontainers/selinux/go-selinux/label"
@@ -130,7 +128,7 @@ func (container *Container) NetworkMounts() []Mount {
 
 // CopyImagePathContent copies files in destination to the volume.
 func (container *Container) CopyImagePathContent(v volume.Volume, destination string) error {
-	rootfs, err := symlink.FollowSymlinkInScope(filepath.Join(container.BaseFS, destination), container.BaseFS)
+	rootfs, err := container.GetResourcePath(destination)
 	if err != nil {
 		return err
 	}
@@ -453,11 +451,6 @@ func (container *Container) TmpfsMounts() ([]Mount, error) {
 	return mounts, nil
 }
 
-// cleanResourcePath cleans a resource path and prepares to combine with mnt path
-func cleanResourcePath(path string) string {
-	return filepath.Join(string(os.PathSeparator), path)
-}
-
 // EnableServiceDiscoveryOnDefaultNetwork Enable service discovery on default network
 func (container *Container) EnableServiceDiscoveryOnDefaultNetwork() bool {
 	return false

container/container_windows.go

@@ -172,18 +172,6 @@ func (container *Container) UpdateContainer(hostConfig *containertypes.HostConfi
 	return nil
 }
 
-// cleanResourcePath cleans a resource path by removing C:\ syntax, and prepares
-// to combine with a volume path
-func cleanResourcePath(path string) string {
-	if len(path) >= 2 {
-		c := path[0]
-		if path[1] == ':' && ('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') {
-			path = path[2:]
-		}
-	}
-	return filepath.Join(string(os.PathSeparator), path)
-}
-
 // BuildHostnameFile writes the container's hostname file.
 func (container *Container) BuildHostnameFile() error {
 	return nil

daemon/archive.go

@@ -3,7 +3,6 @@ package daemon
 import (
 	"io"
 	"os"
-	"path/filepath"
 	"strings"
 
 	"github.com/docker/docker/api/types"
@@ -20,6 +19,31 @@ import (
 // path does not refer to a directory.
 var ErrExtractPointNotDirectory = errors.New("extraction point is not a directory")
 
+// The daemon will use the following interfaces if the container fs implements
+// these for optimized copies to and from the container.
+type extractor interface {
+	ExtractArchive(src io.Reader, dst string, opts *archive.TarOptions) error
+}
+
+type archiver interface {
+	ArchivePath(src string, opts *archive.TarOptions) (io.ReadCloser, error)
+}
+
+// helper functions to extract or archive
+func extractArchive(i interface{}, src io.Reader, dst string, opts *archive.TarOptions) error {
+	if ea, ok := i.(extractor); ok {
+		return ea.ExtractArchive(src, dst, opts)
+	}
+	return chrootarchive.Untar(src, dst, opts)
+}
+
+func archivePath(i interface{}, src string, opts *archive.TarOptions) (io.ReadCloser, error) {
+	if ap, ok := i.(archiver); ok {
+		return ap.ArchivePath(src, opts)
+	}
+	return archive.TarWithOptions(src, opts)
+}
+
 // ContainerCopy performs a deprecated operation of archiving the resource at
 // the specified path in the container identified by the given name.
 func (daemon *Daemon) ContainerCopy(name string, res string) (io.ReadCloser, error) {
@@ -138,6 +162,9 @@ func (daemon *Daemon) containerStatPath(container *container.Container, path str
 		return nil, err
 	}
 
+	// Normalize path before sending to rootfs
+	path = container.BaseFS.FromSlash(path)
+
 	resolvedPath, absPath, err := container.ResolvePath(path)
 	if err != nil {
 		return nil, err
@@ -178,6 +205,9 @@ func (daemon *Daemon) containerArchivePath(container *container.Container, path
 		return nil, nil, err
 	}
 
+	// Normalize path before sending to rootfs
+	path = container.BaseFS.FromSlash(path)
+
 	resolvedPath, absPath, err := container.ResolvePath(path)
 	if err != nil {
 		return nil, nil, err
@@ -196,7 +226,18 @@ func (daemon *Daemon) containerArchivePath(container *container.Container, path
 	// also catches the case when the root directory of the container is
 	// requested: we want the archive entries to start with "/" and not the
 	// container ID.
-	data, err := archive.TarResourceRebase(resolvedPath, filepath.Base(absPath))
+	driver := container.BaseFS
+
+	// Get the source and the base paths of the container resolved path in order
+	// to get the proper tar options for the rebase tar.
+	resolvedPath = driver.Clean(resolvedPath)
+	if driver.Base(resolvedPath) == "." {
+		resolvedPath += string(driver.Separator()) + "."
+	}
+	sourceDir, sourceBase := driver.Dir(resolvedPath), driver.Base(resolvedPath)
+	opts := archive.TarResourceRebaseOpts(sourceBase, driver.Base(absPath))
+
+	data, err := archivePath(driver, sourceDir, opts)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -235,8 +276,12 @@ func (daemon *Daemon) containerExtractToDir(container *container.Container, path
 		return err
 	}
 
+	// Normalize path before sending to rootfs'
+	path = container.BaseFS.FromSlash(path)
+	driver := container.BaseFS
+
 	// Check if a drive letter supplied, it must be the system drive. No-op except on Windows
-	path, err = system.CheckSystemDriveAndRemoveDriveLetter(path)
+	path, err = system.CheckSystemDriveAndRemoveDriveLetter(path, driver)
 	if err != nil {
 		return err
 	}
@@ -248,7 +293,10 @@ func (daemon *Daemon) containerExtractToDir(container *container.Container, path
 	// that you can extract an archive to a symlink that points to a directory.
 
 	// Consider the given path as an absolute path in the container.
-	absPath := archive.PreserveTrailingDotOrSeparator(filepath.Join(string(filepath.Separator), path), path)
+	absPath := archive.PreserveTrailingDotOrSeparator(
+		driver.Join(string(driver.Separator()), path),
+		path,
+		driver.Separator())
 
 	// This will evaluate the last path element if it is a symlink.
 	resolvedPath, err := container.GetResourcePath(absPath)
@@ -256,7 +304,7 @@ func (daemon *Daemon) containerExtractToDir(container *container.Container, path
 		return err
 	}
 
-	stat, err := os.Lstat(resolvedPath)
+	stat, err := driver.Lstat(resolvedPath)
 	if err != nil {
 		return err
 	}
@@ -279,21 +327,24 @@ func (daemon *Daemon) containerExtractToDir(container *container.Container, path
 	// a volume file path.
 	var baseRel string
 	if strings.HasPrefix(resolvedPath, `\\?\Volume{`) {
-		if strings.HasPrefix(resolvedPath, container.BaseFS) {
-			baseRel = resolvedPath[len(container.BaseFS):]
+		if strings.HasPrefix(resolvedPath, driver.Path()) {
+			baseRel = resolvedPath[len(driver.Path()):]
 			if baseRel[:1] == `\` {
 				baseRel = baseRel[1:]
 			}
 		}
 	} else {
-		baseRel, err = filepath.Rel(container.BaseFS, resolvedPath)
+		baseRel, err = driver.Rel(driver.Path(), resolvedPath)
 	}
 	if err != nil {
 		return err
 	}
 	// Make it an absolute path.
-	absPath = filepath.Join(string(filepath.Separator), baseRel)
+	absPath = driver.Join(string(driver.Separator()), baseRel)
 
+	// @ TODO: gupta-ak: Technically, this works since it no-ops
+	// on Windows and the file system is local anyway on linux.
+	// But eventually, it should be made driver aware.
 	toVolume, err := checkIfPathIsInAVolume(container, absPath)
 	if err != nil {
 		return err
@@ -315,7 +366,7 @@ func (daemon *Daemon) containerExtractToDir(container *container.Container, path
 		}
 	}
 
-	if err := chrootarchive.Untar(content, resolvedPath, options); err != nil {
+	if err := extractArchive(driver, content, resolvedPath, options); err != nil {
 		return err
 	}
 
@@ -356,24 +407,28 @@ func (daemon *Daemon) containerCopy(container *container.Container, resource str
 		return nil, err
 	}
 
+	// Normalize path before sending to rootfs
+	resource = container.BaseFS.FromSlash(resource)
+	driver := container.BaseFS
+
 	basePath, err := container.GetResourcePath(resource)
 	if err != nil {
 		return nil, err
 	}
-	stat, err := os.Stat(basePath)
+	stat, err := driver.Stat(basePath)
 	if err != nil {
 		return nil, err
 	}
 	var filter []string
 	if !stat.IsDir() {
-		d, f := filepath.Split(basePath)
+		d, f := driver.Split(basePath)
 		basePath = d
 		filter = []string{f}
 	} else {
-		filter = []string{filepath.Base(basePath)}
-		basePath = filepath.Dir(basePath)
+		filter = []string{driver.Base(basePath)}
+		basePath = driver.Dir(basePath)
 	}
-	archive, err := archive.TarWithOptions(basePath, &archive.TarOptions{
+	archive, err := archivePath(driver, basePath, &archive.TarOptions{
 		Compression:  archive.Uncompressed,
 		IncludeFiles: filter,
 	})

daemon/build.go

@@ -10,6 +10,7 @@ import (
 	"github.com/docker/docker/builder"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/layer"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/docker/registry"
@@ -25,9 +26,9 @@ type releaseableLayer struct {
 	rwLayer    layer.RWLayer
 }
 
-func (rl *releaseableLayer) Mount() (string, error) {
+func (rl *releaseableLayer) Mount() (containerfs.ContainerFS, error) {
 	var err error
-	var mountPath string
+	var mountPath containerfs.ContainerFS
 	var chainID layer.ChainID
 	if rl.roLayer != nil {
 		chainID = rl.roLayer.ChainID()
@@ -36,7 +37,7 @@ func (rl *releaseableLayer) Mount() (string, error) {
 	mountID := stringid.GenerateRandomID()
 	rl.rwLayer, err = rl.layerStore.CreateRWLayer(mountID, chainID, nil)
 	if err != nil {
-		return "", errors.Wrap(err, "failed to create rwlayer")
+		return nil, errors.Wrap(err, "failed to create rwlayer")
 	}
 
 	mountPath, err = rl.rwLayer.Mount("")
@@ -48,7 +49,7 @@ func (rl *releaseableLayer) Mount() (string, error) {
 			logrus.Errorf("Failed to release RWLayer: %s", err)
 		}
 		rl.rwLayer = nil
-		return "", err
+		return nil, err
 	}
 
 	return mountPath, nil

daemon/daemon.go

@@ -40,6 +40,7 @@ import (
 	"github.com/docker/docker/layer"
 	"github.com/docker/docker/libcontainerd"
 	"github.com/docker/docker/migrate/v1"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/plugingetter"
 	"github.com/docker/docker/pkg/sysinfo"
@@ -966,11 +967,11 @@ func (daemon *Daemon) Mount(container *container.Container) error {
 	}
 	logrus.Debugf("container mounted via layerStore: %v", dir)
 
-	if container.BaseFS != dir {
+	if container.BaseFS != nil && container.BaseFS.Path() != dir.Path() {
 		// The mount path reported by the graph driver should always be trusted on Windows, since the
 		// volume path for a given mounted layer may change over time.  This should only be an error
 		// on non-Windows operating systems.
-		if container.BaseFS != "" && runtime.GOOS != "windows" {
+		if runtime.GOOS != "windows" {
 			daemon.Unmount(container)
 			return fmt.Errorf("Error: driver %s is returning inconsistent paths for container %s ('%s' then '%s')",
 				daemon.GraphDriverName(container.Platform), container.ID, container.BaseFS, dir)
@@ -1045,7 +1046,7 @@ func prepareTempDir(rootDir string, rootIDs idtools.IDPair) (string, error) {
 	return tmpDir, idtools.MkdirAllAndChown(tmpDir, 0700, rootIDs)
 }
 
-func (daemon *Daemon) setupInitLayer(initPath string) error {
+func (daemon *Daemon) setupInitLayer(initPath containerfs.ContainerFS) error {
 	rootIDs := daemon.idMappings.RootPair()
 	return initlayer.Setup(initPath, rootIDs)
 }

daemon/daemon_solaris.go

@@ -12,6 +12,7 @@ import (
 	"github.com/docker/docker/container"
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/image"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/fileutils"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/parsers/kernel"
@@ -97,7 +98,7 @@ func setupDaemonRoot(config *config.Config, rootDir string, rootIDs idtools.IDPa
 	return nil
 }
 
-func (daemon *Daemon) getLayerInit() func(string) error {
+func (daemon *Daemon) getLayerInit() func(containerfs.ContainerFS) error {
 	return nil
 }
 

daemon/daemon_unix.go

@@ -24,6 +24,7 @@ import (
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/image"
 	"github.com/docker/docker/opts"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/parsers"
 	"github.com/docker/docker/pkg/parsers/kernel"
@@ -987,7 +988,7 @@ func removeDefaultBridgeInterface() {
 	}
 }
 
-func (daemon *Daemon) getLayerInit() func(string) error {
+func (daemon *Daemon) getLayerInit() func(containerfs.ContainerFS) error {
 	return daemon.setupInitLayer
 }
 

daemon/daemon_windows.go

@@ -12,6 +12,7 @@ import (
 	"github.com/docker/docker/container"
 	"github.com/docker/docker/daemon/config"
 	"github.com/docker/docker/image"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/fileutils"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/parsers"
@@ -56,7 +57,7 @@ func parseSecurityOpt(container *container.Container, config *containertypes.Hos
 	return nil
 }
 
-func (daemon *Daemon) getLayerInit() func(string) error {
+func (daemon *Daemon) getLayerInit() func(containerfs.ContainerFS) error {
 	return nil
 }
 

daemon/export.go

@@ -40,7 +40,7 @@ func (daemon *Daemon) containerExport(container *container.Container) (io.ReadCl
 		return nil, err
 	}
 
-	archive, err := archive.TarWithOptions(container.BaseFS, &archive.TarOptions{
+	archive, err := archivePath(container.BaseFS, container.BaseFS.Path(), &archive.TarOptions{
 		Compression: archive.Uncompressed,
 		UIDMaps:     daemon.idMappings.UIDs(),
 		GIDMaps:     daemon.idMappings.GIDs(),

daemon/graphdriver/aufs/aufs.go

@@ -38,6 +38,7 @@ import (
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/chrootarchive"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/directory"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/locker"
@@ -388,12 +389,12 @@ func atomicRemove(source string) error {
 
 // Get returns the rootfs path for the id.
 // This will mount the dir at its given path
-func (a *Driver) Get(id, mountLabel string) (string, error) {
+func (a *Driver) Get(id, mountLabel string) (containerfs.ContainerFS, error) {
 	a.locker.Lock(id)
 	defer a.locker.Unlock(id)
 	parents, err := a.getParentLayerPaths(id)
 	if err != nil && !os.IsNotExist(err) {
-		return "", err
+		return nil, err
 	}
 
 	a.pathCacheLock.Lock()
@@ -407,21 +408,21 @@ func (a *Driver) Get(id, mountLabel string) (string, error) {
 		}
 	}
 	if count := a.ctr.Increment(m); count > 1 {
-		return m, nil
+		return containerfs.NewLocalContainerFS(m), nil
 	}
 
 	// If a dir does not have a parent ( no layers )do not try to mount
 	// just return the diff path to the data
 	if len(parents) > 0 {
 		if err := a.mount(id, m, mountLabel, parents); err != nil {
-			return "", err
+			return nil, err
 		}
 	}
 
 	a.pathCacheLock.Lock()
 	a.pathCache[id] = m
 	a.pathCacheLock.Unlock()
-	return m, nil
+	return containerfs.NewLocalContainerFS(m), nil
 }
 
 // Put unmounts and updates list of active mounts.

daemon/graphdriver/aufs/aufs_test.go

@@ -9,11 +9,10 @@ import (
 	"io/ioutil"
 	"os"
 	"path"
+	"path/filepath"
 	"sync"
 	"testing"
 
-	"path/filepath"
-
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/reexec"
@@ -43,6 +42,14 @@ func testInit(dir string, t testing.TB) graphdriver.Driver {
 	return d
 }
 
+func driverGet(d *Driver, id string, mntLabel string) (string, error) {
+	mnt, err := d.Get(id, mntLabel)
+	if err != nil {
+		return "", err
+	}
+	return mnt.Path(), nil
+}
+
 func newDriver(t testing.TB) *Driver {
 	if err := os.MkdirAll(tmp, 0755); err != nil {
 		t.Fatal(err)
@@ -172,7 +179,7 @@ func TestGetWithoutParent(t *testing.T) {
 		t.Fatal(err)
 	}
 	expected := path.Join(tmp, "diff", "1")
-	if diffPath != expected {
+	if diffPath.Path() != expected {
 		t.Fatalf("Expected path %s got %s", expected, diffPath)
 	}
 }
@@ -249,13 +256,13 @@ func TestMountWithParent(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if mntPath == "" {
-		t.Fatal("mntPath should not be empty string")
+	if mntPath == nil {
+		t.Fatal("mntPath should not be nil")
 	}
 
 	expected := path.Join(tmp, "mnt", "2")
-	if mntPath != expected {
-		t.Fatalf("Expected %s got %s", expected, mntPath)
+	if mntPath.Path() != expected {
+		t.Fatalf("Expected %s got %s", expected, mntPath.Path())
 	}
 }
 
@@ -280,8 +287,8 @@ func TestRemoveMountedDir(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	if mntPath == "" {
-		t.Fatal("mntPath should not be empty string")
+	if mntPath == nil {
+		t.Fatal("mntPath should not be nil")
 	}
 
 	mounted, err := d.mounted(d.pathCache["2"])
@@ -315,7 +322,7 @@ func TestGetDiff(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	diffPath, err := d.Get("1", "")
+	diffPath, err := driverGet(d, "1", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -359,7 +366,7 @@ func TestChanges(t *testing.T) {
 		}
 	}()
 
-	mntPoint, err := d.Get("2", "")
+	mntPoint, err := driverGet(d, "2", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -398,7 +405,7 @@ func TestChanges(t *testing.T) {
 	if err := d.CreateReadWrite("3", "2", nil); err != nil {
 		t.Fatal(err)
 	}
-	mntPoint, err = d.Get("3", "")
+	mntPoint, err = driverGet(d, "3", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -444,7 +451,7 @@ func TestDiffSize(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	diffPath, err := d.Get("1", "")
+	diffPath, err := driverGet(d, "1", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -486,7 +493,7 @@ func TestChildDiffSize(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	diffPath, err := d.Get("1", "")
+	diffPath, err := driverGet(d, "1", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -587,7 +594,7 @@ func TestApplyDiff(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	diffPath, err := d.Get("1", "")
+	diffPath, err := driverGet(d, "1", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -622,7 +629,7 @@ func TestApplyDiff(t *testing.T) {
 
 	// Ensure that the file is in the mount point for id 3
 
-	mountPoint, err := d.Get("3", "")
+	mountPoint, err := driverGet(d, "3", "")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -665,7 +672,7 @@ func testMountMoreThan42Layers(t *testing.T, mountPath string) {
 		err := d.CreateReadWrite(current, parent, nil)
 		require.NoError(t, err, "current layer %d", i)
 
-		point, err := d.Get(current, "")
+		point, err := driverGet(d, current, "")
 		require.NoError(t, err, "current layer %d", i)
 
 		f, err := os.Create(path.Join(point, current))
@@ -681,7 +688,7 @@ func testMountMoreThan42Layers(t *testing.T, mountPath string) {
 	}
 
 	// Perform the actual mount for the top most image
-	point, err := d.Get(last, "")
+	point, err := driverGet(d, last, "")
 	require.NoError(t, err)
 	files, err := ioutil.ReadDir(point)
 	require.NoError(t, err)

daemon/graphdriver/btrfs/btrfs.go

@@ -27,6 +27,7 @@ import (
 	"unsafe"
 
 	"github.com/docker/docker/daemon/graphdriver"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/mount"
 	"github.com/docker/docker/pkg/parsers"
@@ -631,29 +632,29 @@ func (d *Driver) Remove(id string) error {
 }
 
 // Get the requested filesystem id.
-func (d *Driver) Get(id, mountLabel string) (string, error) {
+func (d *Driver) Get(id, mountLabel string) (containerfs.ContainerFS, error) {
 	dir := d.subvolumesDirID(id)
 	st, err := os.Stat(dir)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
 	if !st.IsDir() {
-		return "", fmt.Errorf("%s: not a directory", dir)
+		return nil, fmt.Errorf("%s: not a directory", dir)
 	}
 
 	if quota, err := ioutil.ReadFile(d.quotasDirID(id)); err == nil {
 		if size, err := strconv.ParseUint(string(quota), 10, 64); err == nil && size >= d.options.minSpace {
 			if err := d.subvolEnableQuota(); err != nil {
-				return "", err
+				return nil, err
 			}
 			if err := subvolLimitQgroup(dir, size); err != nil {
-				return "", err
+				return nil, err
 			}
 		}
 	}
 
-	return dir, nil
+	return containerfs.NewLocalContainerFS(dir), nil
 }
 
 // Put is not implemented for BTRFS as there is no cleanup required for the id.

daemon/graphdriver/btrfs/btrfs_test.go

@@ -35,12 +35,14 @@ func TestBtrfsSubvolDelete(t *testing.T) {
 	}
 	defer graphtest.PutDriver(t)
 
-	dir, err := d.Get("test", "")
+	dirFS, err := d.Get("test", "")
 	if err != nil {
 		t.Fatal(err)
 	}
 	defer d.Put("test")
 
+	dir := dirFS.Path()
+
 	if err := subvolCreate(dir, "subvoltest"); err != nil {
 		t.Fatal(err)
 	}

daemon/graphdriver/devmapper/driver.go

@@ -12,6 +12,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/docker/docker/daemon/graphdriver"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/devicemapper"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/locker"
@@ -163,41 +164,41 @@ func (d *Driver) Remove(id string) error {
 }
 
 // Get mounts a device with given id into the root filesystem
-func (d *Driver) Get(id, mountLabel string) (string, error) {
+func (d *Driver) Get(id, mountLabel string) (containerfs.ContainerFS, error) {
 	d.locker.Lock(id)
 	defer d.locker.Unlock(id)
 	mp := path.Join(d.home, "mnt", id)
 	rootFs := path.Join(mp, "rootfs")
 	if count := d.ctr.Increment(mp); count > 1 {
-		return rootFs, nil
+		return containerfs.NewLocalContainerFS(rootFs), nil
 	}
 
 	uid, gid, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
 	if err != nil {
 		d.ctr.Decrement(mp)
-		return "", err
+		return nil, err
 	}
 
 	// Create the target directories if they don't exist
 	if err := idtools.MkdirAllAs(path.Join(d.home, "mnt"), 0755, uid, gid); err != nil && !os.IsExist(err) {
 		d.ctr.Decrement(mp)
-		return "", err
+		return nil, err
 	}
 	if err := idtools.MkdirAs(mp, 0755, uid, gid); err != nil && !os.IsExist(err) {
 		d.ctr.Decrement(mp)
-		return "", err
+		return nil, err
 	}
 
 	// Mount the device
 	if err := d.DeviceSet.MountDevice(id, mp, mountLabel); err != nil {
 		d.ctr.Decrement(mp)
-		return "", err
+		return nil, err
 	}
 
 	if err := idtools.MkdirAllAs(rootFs, 0755, uid, gid); err != nil && !os.IsExist(err) {
 		d.ctr.Decrement(mp)
 		d.DeviceSet.UnmountDevice(id, mp)
-		return "", err
+		return nil, err
 	}
 
 	idFile := path.Join(mp, "id")
@@ -207,11 +208,11 @@ func (d *Driver) Get(id, mountLabel string) (string, error) {
 		if err := ioutil.WriteFile(idFile, []byte(id), 0600); err != nil {
 			d.ctr.Decrement(mp)
 			d.DeviceSet.UnmountDevice(id, mp)
-			return "", err
+			return nil, err
 		}
 	}
 
-	return rootFs, nil
+	return containerfs.NewLocalContainerFS(rootFs), nil
 }
 
 // Put unmounts a device and removes it.

daemon/graphdriver/driver.go

@@ -12,6 +12,7 @@ import (
 	"github.com/vbatts/tar-split/tar/storage"
 
 	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/plugingetter"
 )
@@ -68,7 +69,7 @@ type ProtoDriver interface {
 	// Get returns the mountpoint for the layered filesystem referred
 	// to by this id. You can optionally specify a mountLabel or "".
 	// Returns the absolute path to the mounted layered filesystem.
-	Get(id, mountLabel string) (dir string, err error)
+	Get(id, mountLabel string) (fs containerfs.ContainerFS, err error)
 	// Put releases the system resources for the specified id,
 	// e.g, unmounting layered filesystem.
 	Put(id string) error

daemon/graphdriver/fsdiff.go

@@ -18,9 +18,9 @@ var (
 )
 
 // NaiveDiffDriver takes a ProtoDriver and adds the
-// capability of the Diffing methods which it may or may not
-// support on its own. See the comment on the exported
-// NewNaiveDiffDriver function below.
+// capability of the Diffing methods on the local file system,
+// which it may or may not support on its own. See the comment
+// on the exported NewNaiveDiffDriver function below.
 // Notably, the AUFS driver doesn't need to be wrapped like this.
 type NaiveDiffDriver struct {
 	ProtoDriver
@@ -47,10 +47,11 @@ func (gdw *NaiveDiffDriver) Diff(id, parent string) (arch io.ReadCloser, err err
 	startTime := time.Now()
 	driver := gdw.ProtoDriver
 
-	layerFs, err := driver.Get(id, "")
+	layerRootFs, err := driver.Get(id, "")
 	if err != nil {
 		return nil, err
 	}
+	layerFs := layerRootFs.Path()
 
 	defer func() {
 		if err != nil {
@@ -70,12 +71,14 @@ func (gdw *NaiveDiffDriver) Diff(id, parent string) (arch io.ReadCloser, err err
 		}), nil
 	}
 
-	parentFs, err := driver.Get(parent, "")
+	parentRootFs, err := driver.Get(parent, "")
 	if err != nil {
 		return nil, err
 	}
 	defer driver.Put(parent)
 
+	parentFs := parentRootFs.Path()
+
 	changes, err := archive.ChangesDirs(layerFs, parentFs)
 	if err != nil {
 		return nil, err
@@ -104,20 +107,22 @@ func (gdw *NaiveDiffDriver) Diff(id, parent string) (arch io.ReadCloser, err err
 func (gdw *NaiveDiffDriver) Changes(id, parent string) ([]archive.Change, error) {
 	driver := gdw.ProtoDriver
 
-	layerFs, err := driver.Get(id, "")
+	layerRootFs, err := driver.Get(id, "")
 	if err != nil {
 		return nil, err
 	}
 	defer driver.Put(id)
 
+	layerFs := layerRootFs.Path()
 	parentFs := ""
 
 	if parent != "" {
-		parentFs, err = driver.Get(parent, "")
+		parentRootFs, err := driver.Get(parent, "")
 		if err != nil {
 			return nil, err
 		}
 		defer driver.Put(parent)
+		parentFs = parentRootFs.Path()
 	}
 
 	return archive.ChangesDirs(layerFs, parentFs)
@@ -130,12 +135,13 @@ func (gdw *NaiveDiffDriver) ApplyDiff(id, parent string, diff io.Reader) (size i
 	driver := gdw.ProtoDriver
 
 	// Mount the root filesystem so we can apply the diff/layer.
-	layerFs, err := driver.Get(id, "")
+	layerRootFs, err := driver.Get(id, "")
 	if err != nil {
 		return
 	}
 	defer driver.Put(id)
 
+	layerFs := layerRootFs.Path()
 	options := &archive.TarOptions{UIDMaps: gdw.uidMaps,
 		GIDMaps: gdw.gidMaps}
 	start := time.Now().UTC()
@@ -165,5 +171,5 @@ func (gdw *NaiveDiffDriver) DiffSize(id, parent string) (size int64, err error)
 	}
 	defer driver.Put(id)
 
-	return archive.ChangesSize(layerFs, changes), nil
+	return archive.ChangesSize(layerFs.Path(), changes), nil
 }

daemon/graphdriver/graphtest/graphbench_unix.go

@@ -5,9 +5,9 @@ package graphtest
 import (
 	"io"
 	"io/ioutil"
-	"path/filepath"
 	"testing"
 
+	contdriver "github.com/containerd/continuity/driver"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/stretchr/testify/require"
 )
@@ -245,7 +245,7 @@ func DriverBenchDeepLayerRead(b *testing.B, layerCount int, drivername string, d
 	for i := 0; i < b.N; i++ {
 
 		// Read content
-		c, err := ioutil.ReadFile(filepath.Join(root, "testfile.txt"))
+		c, err := contdriver.ReadFile(root, root.Join(root.Path(), "testfile.txt"))
 		if err != nil {
 			b.Fatal(err)
 		}

daemon/graphdriver/graphtest/graphtest_unix.go

@@ -97,10 +97,10 @@ func DriverTestCreateEmpty(t testing.TB, drivername string, driverOptions ...str
 	dir, err := driver.Get("empty", "")
 	require.NoError(t, err)
 
-	verifyFile(t, dir, 0755|os.ModeDir, 0, 0)
+	verifyFile(t, dir.Path(), 0755|os.ModeDir, 0, 0)
 
 	// Verify that the directory is empty
-	fis, err := readDir(dir)
+	fis, err := readDir(dir, dir.Path())
 	require.NoError(t, err)
 	assert.Len(t, fis, 0)
 
@@ -328,9 +328,9 @@ func DriverTestSetQuota(t *testing.T, drivername string) {
 	}
 
 	quota := uint64(50 * units.MiB)
-	err = writeRandomFile(path.Join(mountPath, "file"), quota*2)
+
+	err = writeRandomFile(path.Join(mountPath.Path(), "file"), quota*2)
 	if pathError, ok := err.(*os.PathError); ok && pathError.Err != unix.EDQUOT {
 		t.Fatalf("expect write() to fail with %v, got %v", unix.EDQUOT, err)
 	}
-
 }

daemon/graphdriver/graphtest/testutil.go

@@ -3,12 +3,11 @@ package graphtest
 import (
 	"bytes"
 	"fmt"
-	"io/ioutil"
 	"math/rand"
 	"os"
-	"path"
 	"sort"
 
+	"github.com/containerd/continuity/driver"
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/stringid"
@@ -36,17 +35,17 @@ func addFiles(drv graphdriver.Driver, layer string, seed int64) error {
 	}
 	defer drv.Put(layer)
 
-	if err := ioutil.WriteFile(path.Join(root, "file-a"), randomContent(64, seed), 0755); err != nil {
+	if err := driver.WriteFile(root, root.Join(root.Path(), "file-a"), randomContent(64, seed), 0755); err != nil {
 		return err
 	}
-	if err := os.MkdirAll(path.Join(root, "dir-b"), 0755); err != nil {
+	if err := root.MkdirAll(root.Join(root.Path(), "dir-b"), 0755); err != nil {
 		return err
 	}
-	if err := ioutil.WriteFile(path.Join(root, "dir-b", "file-b"), randomContent(128, seed+1), 0755); err != nil {
+	if err := driver.WriteFile(root, root.Join(root.Path(), "dir-b", "file-b"), randomContent(128, seed+1), 0755); err != nil {
 		return err
 	}
 
-	return ioutil.WriteFile(path.Join(root, "file-c"), randomContent(128*128, seed+2), 0755)
+	return driver.WriteFile(root, root.Join(root.Path(), "file-c"), randomContent(128*128, seed+2), 0755)
 }
 
 func checkFile(drv graphdriver.Driver, layer, filename string, content []byte) error {
@@ -56,7 +55,7 @@ func checkFile(drv graphdriver.Driver, layer, filename string, content []byte) e
 	}
 	defer drv.Put(layer)
 
-	fileContent, err := ioutil.ReadFile(path.Join(root, filename))
+	fileContent, err := driver.ReadFile(root, root.Join(root.Path(), filename))
 	if err != nil {
 		return err
 	}
@@ -75,7 +74,7 @@ func addFile(drv graphdriver.Driver, layer, filename string, content []byte) err
 	}
 	defer drv.Put(layer)
 
-	return ioutil.WriteFile(path.Join(root, filename), content, 0755)
+	return driver.WriteFile(root, root.Join(root.Path(), filename), content, 0755)
 }
 
 func addDirectory(drv graphdriver.Driver, layer, dir string) error {
@@ -85,7 +84,7 @@ func addDirectory(drv graphdriver.Driver, layer, dir string) error {
 	}
 	defer drv.Put(layer)
 
-	return os.MkdirAll(path.Join(root, dir), 0755)
+	return root.MkdirAll(root.Join(root.Path(), dir), 0755)
 }
 
 func removeAll(drv graphdriver.Driver, layer string, names ...string) error {
@@ -96,7 +95,7 @@ func removeAll(drv graphdriver.Driver, layer string, names ...string) error {
 	defer drv.Put(layer)
 
 	for _, filename := range names {
-		if err := os.RemoveAll(path.Join(root, filename)); err != nil {
+		if err := root.RemoveAll(root.Join(root.Path(), filename)); err != nil {
 			return err
 		}
 	}
@@ -110,8 +109,8 @@ func checkFileRemoved(drv graphdriver.Driver, layer, filename string) error {
 	}
 	defer drv.Put(layer)
 
-	if _, err := os.Stat(path.Join(root, filename)); err == nil {
-		return fmt.Errorf("file still exists: %s", path.Join(root, filename))
+	if _, err := root.Stat(root.Join(root.Path(), filename)); err == nil {
+		return fmt.Errorf("file still exists: %s", root.Join(root.Path(), filename))
 	} else if !os.IsNotExist(err) {
 		return err
 	}
@@ -127,13 +126,13 @@ func addManyFiles(drv graphdriver.Driver, layer string, count int, seed int64) e
 	defer drv.Put(layer)
 
 	for i := 0; i < count; i += 100 {
-		dir := path.Join(root, fmt.Sprintf("directory-%d", i))
-		if err := os.MkdirAll(dir, 0755); err != nil {
+		dir := root.Join(root.Path(), fmt.Sprintf("directory-%d", i))
+		if err := root.MkdirAll(dir, 0755); err != nil {
 			return err
 		}
 		for j := 0; i+j < count && j < 100; j++ {
-			file := path.Join(dir, fmt.Sprintf("file-%d", i+j))
-			if err := ioutil.WriteFile(file, randomContent(64, seed+int64(i+j)), 0755); err != nil {
+			file := root.Join(dir, fmt.Sprintf("file-%d", i+j))
+			if err := driver.WriteFile(root, file, randomContent(64, seed+int64(i+j)), 0755); err != nil {
 				return err
 			}
 		}
@@ -152,7 +151,7 @@ func changeManyFiles(drv graphdriver.Driver, layer string, count int, seed int64
 	changes := []archive.Change{}
 	for i := 0; i < count; i += 100 {
 		archiveRoot := fmt.Sprintf("/directory-%d", i)
-		if err := os.MkdirAll(path.Join(root, archiveRoot), 0755); err != nil {
+		if err := root.MkdirAll(root.Join(root.Path(), archiveRoot), 0755); err != nil {
 			return nil, err
 		}
 		for j := 0; i+j < count && j < 100; j++ {
@@ -166,23 +165,23 @@ func changeManyFiles(drv graphdriver.Driver, layer string, count int, seed int64
 			switch j % 3 {
 			// Update file
 			case 0:
-				change.Path = path.Join(archiveRoot, fmt.Sprintf("file-%d", i+j))
+				change.Path = root.Join(archiveRoot, fmt.Sprintf("file-%d", i+j))
 				change.Kind = archive.ChangeModify
-				if err := ioutil.WriteFile(path.Join(root, change.Path), randomContent(64, seed+int64(i+j)), 0755); err != nil {
+				if err := driver.WriteFile(root, root.Join(root.Path(), change.Path), randomContent(64, seed+int64(i+j)), 0755); err != nil {
 					return nil, err
 				}
 			// Add file
 			case 1:
-				change.Path = path.Join(archiveRoot, fmt.Sprintf("file-%d-%d", seed, i+j))
+				change.Path = root.Join(archiveRoot, fmt.Sprintf("file-%d-%d", seed, i+j))
 				change.Kind = archive.ChangeAdd
-				if err := ioutil.WriteFile(path.Join(root, change.Path), randomContent(64, seed+int64(i+j)), 0755); err != nil {
+				if err := driver.WriteFile(root, root.Join(root.Path(), change.Path), randomContent(64, seed+int64(i+j)), 0755); err != nil {
 					return nil, err
 				}
 			// Remove file
 			case 2:
-				change.Path = path.Join(archiveRoot, fmt.Sprintf("file-%d", i+j))
+				change.Path = root.Join(archiveRoot, fmt.Sprintf("file-%d", i+j))
 				change.Kind = archive.ChangeDelete
-				if err := os.Remove(path.Join(root, change.Path)); err != nil {
+				if err := root.Remove(root.Join(root.Path(), change.Path)); err != nil {
 					return nil, err
 				}
 			}
@@ -201,10 +200,10 @@ func checkManyFiles(drv graphdriver.Driver, layer string, count int, seed int64)
 	defer drv.Put(layer)
 
 	for i := 0; i < count; i += 100 {
-		dir := path.Join(root, fmt.Sprintf("directory-%d", i))
+		dir := root.Join(root.Path(), fmt.Sprintf("directory-%d", i))
 		for j := 0; i+j < count && j < 100; j++ {
-			file := path.Join(dir, fmt.Sprintf("file-%d", i+j))
-			fileContent, err := ioutil.ReadFile(file)
+			file := root.Join(dir, fmt.Sprintf("file-%d", i+j))
+			fileContent, err := driver.ReadFile(root, file)
 			if err != nil {
 				return err
 			}
@@ -254,17 +253,17 @@ func addLayerFiles(drv graphdriver.Driver, layer, parent string, i int) error {
 	}
 	defer drv.Put(layer)
 
-	if err := ioutil.WriteFile(path.Join(root, "top-id"), []byte(layer), 0755); err != nil {
+	if err := driver.WriteFile(root, root.Join(root.Path(), "top-id"), []byte(layer), 0755); err != nil {
 		return err
 	}
-	layerDir := path.Join(root, fmt.Sprintf("layer-%d", i))
-	if err := os.MkdirAll(layerDir, 0755); err != nil {
+	layerDir := root.Join(root.Path(), fmt.Sprintf("layer-%d", i))
+	if err := root.MkdirAll(layerDir, 0755); err != nil {
 		return err
 	}
-	if err := ioutil.WriteFile(path.Join(layerDir, "layer-id"), []byte(layer), 0755); err != nil {
+	if err := driver.WriteFile(root, root.Join(layerDir, "layer-id"), []byte(layer), 0755); err != nil {
 		return err
 	}
-	if err := ioutil.WriteFile(path.Join(layerDir, "parent-id"), []byte(parent), 0755); err != nil {
+	if err := driver.WriteFile(root, root.Join(layerDir, "parent-id"), []byte(parent), 0755); err != nil {
 		return err
 	}
 
@@ -295,7 +294,7 @@ func checkManyLayers(drv graphdriver.Driver, layer string, count int) error {
 	}
 	defer drv.Put(layer)
 
-	layerIDBytes, err := ioutil.ReadFile(path.Join(root, "top-id"))
+	layerIDBytes, err := driver.ReadFile(root, root.Join(root.Path(), "top-id"))
 	if err != nil {
 		return err
 	}
@@ -305,16 +304,16 @@ func checkManyLayers(drv graphdriver.Driver, layer string, count int) error {
 	}
 
 	for i := count; i > 0; i-- {
-		layerDir := path.Join(root, fmt.Sprintf("layer-%d", i))
+		layerDir := root.Join(root.Path(), fmt.Sprintf("layer-%d", i))
 
-		thisLayerIDBytes, err := ioutil.ReadFile(path.Join(layerDir, "layer-id"))
+		thisLayerIDBytes, err := driver.ReadFile(root, root.Join(layerDir, "layer-id"))
 		if err != nil {
 			return err
 		}
 		if !bytes.Equal(thisLayerIDBytes, layerIDBytes) {
 			return fmt.Errorf("mismatched file content %v, expecting %v", thisLayerIDBytes, layerIDBytes)
 		}
-		layerIDBytes, err = ioutil.ReadFile(path.Join(layerDir, "parent-id"))
+		layerIDBytes, err = driver.ReadFile(root, root.Join(layerDir, "parent-id"))
 		if err != nil {
 			return err
 		}
@@ -322,11 +321,11 @@ func checkManyLayers(drv graphdriver.Driver, layer string, count int) error {
 	return nil
 }
 
-// readDir reads a directory just like ioutil.ReadDir()
+// readDir reads a directory just like driver.ReadDir()
 // then hides specific files (currently "lost+found")
 // so the tests don't "see" it
-func readDir(dir string) ([]os.FileInfo, error) {
-	a, err := ioutil.ReadDir(dir)
+func readDir(r driver.Driver, dir string) ([]os.FileInfo, error) {
+	a, err := driver.ReadDir(r, dir)
 	if err != nil {
 		return nil, err
 	}

daemon/graphdriver/graphtest/testutil_unix.go

@@ -3,12 +3,11 @@
 package graphtest
 
 import (
-	"io/ioutil"
 	"os"
-	"path"
 	"syscall"
 	"testing"
 
+	contdriver "github.com/containerd/continuity/driver"
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -40,31 +39,31 @@ func createBase(t testing.TB, driver graphdriver.Driver, name string) {
 	err := driver.CreateReadWrite(name, "", nil)
 	require.NoError(t, err)
 
-	dir, err := driver.Get(name, "")
+	dirFS, err := driver.Get(name, "")
 	require.NoError(t, err)
 	defer driver.Put(name)
 
-	subdir := path.Join(dir, "a subdir")
-	require.NoError(t, os.Mkdir(subdir, 0705|os.ModeSticky))
-	require.NoError(t, os.Chown(subdir, 1, 2))
+	subdir := dirFS.Join(dirFS.Path(), "a subdir")
+	require.NoError(t, dirFS.Mkdir(subdir, 0705|os.ModeSticky))
+	require.NoError(t, dirFS.Lchown(subdir, 1, 2))
 
-	file := path.Join(dir, "a file")
-	err = ioutil.WriteFile(file, []byte("Some data"), 0222|os.ModeSetuid)
+	file := dirFS.Join(dirFS.Path(), "a file")
+	err = contdriver.WriteFile(dirFS, file, []byte("Some data"), 0222|os.ModeSetuid)
 	require.NoError(t, err)
 }
 
 func verifyBase(t testing.TB, driver graphdriver.Driver, name string) {
-	dir, err := driver.Get(name, "")
+	dirFS, err := driver.Get(name, "")
 	require.NoError(t, err)
 	defer driver.Put(name)
 
-	subdir := path.Join(dir, "a subdir")
+	subdir := dirFS.Join(dirFS.Path(), "a subdir")
 	verifyFile(t, subdir, 0705|os.ModeDir|os.ModeSticky, 1, 2)
 
-	file := path.Join(dir, "a file")
+	file := dirFS.Join(dirFS.Path(), "a file")
 	verifyFile(t, file, 0222|os.ModeSetuid, 0, 0)
 
-	files, err := readDir(dir)
+	files, err := readDir(dirFS, dirFS.Path())
 	require.NoError(t, err)
 	assert.Len(t, files, 2)
 }

daemon/graphdriver/lcow/lcow.go

@@ -65,12 +65,14 @@ import (
 	"strconv"
 	"strings"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/Microsoft/hcsshim"
 	"github.com/Microsoft/opengcs/client"
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/docker/pkg/system"
@@ -106,72 +108,24 @@ const (
 
 	// scratchDirectory is the sub-folder under the driver's data-root used for scratch VHDs in service VMs
 	scratchDirectory = "scratch"
-)
-
-// cacheItem is our internal structure representing an item in our local cache
-// of things that have been mounted.
-type cacheItem struct {
-	sync.Mutex        // Protects operations performed on this item
-	uvmPath    string // Path in utility VM
-	hostPath   string // Path on host
-	refCount   int    // How many times its been mounted
-	isSandbox  bool   // True if a sandbox
-	isMounted  bool   // True when mounted in a service VM
-}
-
-// setIsMounted is a helper function for a cacheItem which does exactly what it says
-func (ci *cacheItem) setIsMounted() {
-	logrus.Debugf("locking cache item for set isMounted")
-	ci.Lock()
-	defer ci.Unlock()
-	ci.isMounted = true
-	logrus.Debugf("set isMounted on cache item")
-}
 
-// incrementRefCount is a helper function for a cacheItem which does exactly what it says
-func (ci *cacheItem) incrementRefCount() {
-	logrus.Debugf("locking cache item for increment")
-	ci.Lock()
-	defer ci.Unlock()
-	ci.refCount++
-	logrus.Debugf("incremented refcount on cache item %+v", ci)
-}
-
-// decrementRefCount is a helper function for a cacheItem which does exactly what it says
-func (ci *cacheItem) decrementRefCount() int {
-	logrus.Debugf("locking cache item for decrement")
-	ci.Lock()
-	defer ci.Unlock()
-	ci.refCount--
-	logrus.Debugf("decremented refcount on cache item %+v", ci)
-	return ci.refCount
-}
-
-// serviceVMItem is our internal structure representing an item in our
-// map of service VMs we are maintaining.
-type serviceVMItem struct {
-	sync.Mutex                     // Serialises operations being performed in this service VM.
-	scratchAttached bool           // Has a scratch been attached?
-	config          *client.Config // Represents the service VM item.
-}
+	// errOperationPending is the HRESULT returned by the HCS when the VM termination operation is still pending.
+	errOperationPending syscall.Errno = 0xc0370103
+)
 
 // Driver represents an LCOW graph driver.
 type Driver struct {
-	dataRoot           string                    // Root path on the host where we are storing everything.
-	cachedSandboxFile  string                    // Location of the local default-sized cached sandbox.
-	cachedSandboxMutex sync.Mutex                // Protects race conditions from multiple threads creating the cached sandbox.
-	cachedScratchFile  string                    // Location of the local cached empty scratch space.
-	cachedScratchMutex sync.Mutex                // Protects race conditions from multiple threads creating the cached scratch.
-	options            []string                  // Graphdriver options we are initialised with.
-	serviceVmsMutex    sync.Mutex                // Protects add/updates/delete to the serviceVMs map.
-	serviceVms         map[string]*serviceVMItem // Map of the configs representing the service VM(s) we are running.
-	globalMode         bool                      // Indicates if running in an unsafe/global service VM mode.
+	dataRoot           string     // Root path on the host where we are storing everything.
+	cachedSandboxFile  string     // Location of the local default-sized cached sandbox.
+	cachedSandboxMutex sync.Mutex // Protects race conditions from multiple threads creating the cached sandbox.
+	cachedScratchFile  string     // Location of the local cached empty scratch space.
+	cachedScratchMutex sync.Mutex // Protects race conditions from multiple threads creating the cached scratch.
+	options            []string   // Graphdriver options we are initialised with.
+	globalMode         bool       // Indicates if running in an unsafe/global service VM mode.
 
 	// NOTE: It is OK to use a cache here because Windows does not support
 	// restoring containers when the daemon dies.
-
-	cacheMutex sync.Mutex            // Protects add/update/deletes to cache.
-	cache      map[string]*cacheItem // Map holding a cache of all the IDs we've mounted/unmounted.
+	serviceVms *serviceVMMap // Map of the configs representing the service VM(s) we are running.
 }
 
 // layerDetails is the structure returned by a helper function `getLayerDetails`
@@ -204,9 +158,10 @@ func InitDriver(dataRoot string, options []string, _, _ []idtools.IDMap) (graphd
 		options:           options,
 		cachedSandboxFile: filepath.Join(cd, sandboxFilename),
 		cachedScratchFile: filepath.Join(cd, scratchFilename),
-		cache:             make(map[string]*cacheItem),
-		serviceVms:        make(map[string]*serviceVMItem),
-		globalMode:        false,
+		serviceVms: &serviceVMMap{
+			svms: make(map[string]*serviceVMMapItem),
+		},
+		globalMode: false,
 	}
 
 	// Looks for relevant options
@@ -248,53 +203,59 @@ func InitDriver(dataRoot string, options []string, _, _ []idtools.IDMap) (graphd
 	return d, nil
 }
 
+func (d *Driver) getVMID(id string) string {
+	if d.globalMode {
+		return svmGlobalID
+	}
+	return id
+}
+
 // startServiceVMIfNotRunning starts a service utility VM if it is not currently running.
 // It can optionally be started with a mapped virtual disk. Returns a opengcs config structure
 // representing the VM.
-func (d *Driver) startServiceVMIfNotRunning(id string, mvdToAdd *hcsshim.MappedVirtualDisk, context string) (*serviceVMItem, error) {
+func (d *Driver) startServiceVMIfNotRunning(id string, mvdToAdd []hcsshim.MappedVirtualDisk, context string) (_ *serviceVM, err error) {
 	// Use the global ID if in global mode
-	if d.globalMode {
-		id = svmGlobalID
-	}
+	id = d.getVMID(id)
 
 	title := fmt.Sprintf("lcowdriver: startservicevmifnotrunning %s:", id)
 
-	// Make sure thread-safe when interrogating the map
-	logrus.Debugf("%s taking serviceVmsMutex", title)
-	d.serviceVmsMutex.Lock()
-
-	// Nothing to do if it's already running except add the mapped drive if supplied.
-	if svm, ok := d.serviceVms[id]; ok {
-		logrus.Debugf("%s exists, releasing serviceVmsMutex", title)
-		d.serviceVmsMutex.Unlock()
-
-		if mvdToAdd != nil {
-			logrus.Debugf("hot-adding %s to %s", mvdToAdd.HostPath, mvdToAdd.ContainerPath)
-
-			// Ensure the item is locked while doing this
-			logrus.Debugf("%s locking serviceVmItem %s", title, svm.config.Name)
-			svm.Lock()
-
-			if err := svm.config.HotAddVhd(mvdToAdd.HostPath, mvdToAdd.ContainerPath, false, true); err != nil {
-				logrus.Debugf("%s releasing serviceVmItem %s on hot-add failure %s", title, svm.config.Name, err)
-				svm.Unlock()
-				return nil, fmt.Errorf("%s hot add %s to %s failed: %s", title, mvdToAdd.HostPath, mvdToAdd.ContainerPath, err)
-			}
+	// Attempt to add ID to the service vm map
+	logrus.Debugf("%s: Adding entry to service vm map", title)
+	svm, exists, err := d.serviceVms.add(id)
+	if err != nil && err == errVMisTerminating {
+		// VM is in the process of terminating. Wait until it's done and and then try again
+		logrus.Debugf("%s: VM with current ID still in the process of terminating: %s", title, id)
+		if err := svm.getStopError(); err != nil {
+			logrus.Debugf("%s: VM %s did not stop succesfully: %s", title, id, err)
+			return nil, err
+		}
+		return d.startServiceVMIfNotRunning(id, mvdToAdd, context)
+	} else if err != nil {
+		logrus.Debugf("%s: failed to add service vm to map: %s", err)
+		return nil, fmt.Errorf("%s: failed to add to service vm map: %s", title, err)
+	}
 
-			logrus.Debugf("%s releasing serviceVmItem %s", title, svm.config.Name)
-			svm.Unlock()
+	if exists {
+		// Service VM is already up and running. In this case, just hot add the vhds.
+		logrus.Debugf("%s: service vm already exists. Just hot adding: %+v", title, mvdToAdd)
+		if err := svm.hotAddVHDs(mvdToAdd...); err != nil {
+			logrus.Debugf("%s: failed to hot add vhds on service vm creation: %s", title, err)
+			return nil, fmt.Errorf("%s: failed to hot add vhds on service vm: %s", title, err)
 		}
 		return svm, nil
 	}
 
-	// Release the lock early
-	logrus.Debugf("%s releasing serviceVmsMutex", title)
-	d.serviceVmsMutex.Unlock()
+	// We are the first service for this id, so we need to start it
+	logrus.Debugf("%s: service vm doesn't exist. Now starting it up: %s", title, id)
 
-	// So we are starting one. First need an enpty structure.
-	svm := &serviceVMItem{
-		config: &client.Config{},
-	}
+	defer func() {
+		// Signal that start has finished, passing in the error if any.
+		svm.signalStartFinished(err)
+		if err != nil {
+			// We added a ref to the VM, since we failed, we should delete the ref.
+			d.terminateServiceVM(id, "error path on startServiceVMIfNotRunning", false)
+		}
+	}()
 
 	// Generate a default configuration
 	if err := svm.config.GenerateDefault(d.options); err != nil {
@@ -335,12 +296,14 @@ func (d *Driver) startServiceVMIfNotRunning(id string, mvdToAdd *hcsshim.MappedV
 		svm.config.MappedVirtualDisks = append(svm.config.MappedVirtualDisks, mvd)
 		svm.scratchAttached = true
 	}
+
 	logrus.Debugf("%s releasing cachedScratchMutex", title)
 	d.cachedScratchMutex.Unlock()
 
 	// If requested to start it with a mapped virtual disk, add it now.
-	if mvdToAdd != nil {
-		svm.config.MappedVirtualDisks = append(svm.config.MappedVirtualDisks, *mvdToAdd)
+	svm.config.MappedVirtualDisks = append(svm.config.MappedVirtualDisks, mvdToAdd...)
+	for _, mvd := range svm.config.MappedVirtualDisks {
+		svm.attachedVHDs[mvd.HostPath] = 1
 	}
 
 	// Start it.
@@ -349,108 +312,80 @@ func (d *Driver) startServiceVMIfNotRunning(id string, mvdToAdd *hcsshim.MappedV
 		return nil, fmt.Errorf("failed to start service utility VM (%s): %s", context, err)
 	}
 
-	// As it's now running, add it to the map, checking for a race where another
-	// thread has simultaneously tried to start it.
-	logrus.Debugf("%s locking serviceVmsMutex for insertion", title)
-	d.serviceVmsMutex.Lock()
-	if svm, ok := d.serviceVms[id]; ok {
-		logrus.Debugf("%s releasing serviceVmsMutex after insertion but exists", title)
-		d.serviceVmsMutex.Unlock()
-		return svm, nil
-	}
-	d.serviceVms[id] = svm
-	logrus.Debugf("%s releasing serviceVmsMutex after insertion", title)
-	d.serviceVmsMutex.Unlock()
+	// defer function to terminate the VM if the next steps fail
+	defer func() {
+		if err != nil {
+			waitTerminate(svm, fmt.Sprintf("startServiceVmIfNotRunning: %s (%s)", id, context))
+		}
+	}()
 
 	// Now we have a running service VM, we can create the cached scratch file if it doesn't exist.
 	logrus.Debugf("%s locking cachedScratchMutex", title)
 	d.cachedScratchMutex.Lock()
 	if _, err := os.Stat(d.cachedScratchFile); err != nil {
-		logrus.Debugf("%s (%s): creating an SVM scratch - locking serviceVM", title, context)
-		svm.Lock()
+		logrus.Debugf("%s (%s): creating an SVM scratch", title, context)
+
+		// Don't use svm.CreateExt4Vhdx since that only works when the service vm is setup,
+		// but we're still in that process right now.
 		if err := svm.config.CreateExt4Vhdx(scratchTargetFile, client.DefaultVhdxSizeGB, d.cachedScratchFile); err != nil {
-			logrus.Debugf("%s (%s): releasing serviceVM on error path from CreateExt4Vhdx: %s", title, context, err)
-			svm.Unlock()
 			logrus.Debugf("%s (%s): releasing cachedScratchMutex on error path", title, context)
 			d.cachedScratchMutex.Unlock()
-
-			// Do a force terminate and remove it from the map on failure, ignoring any errors
-			if err2 := d.terminateServiceVM(id, "error path from CreateExt4Vhdx", true); err2 != nil {
-				logrus.Warnf("failed to terminate service VM on error path from CreateExt4Vhdx: %s", err2)
-			}
-
+			logrus.Debugf("%s: failed to create vm scratch %s: %s", title, scratchTargetFile, err)
 			return nil, fmt.Errorf("failed to create SVM scratch VHDX (%s): %s", context, err)
 		}
-		logrus.Debugf("%s (%s): releasing serviceVM after %s created and cached to %s", title, context, scratchTargetFile, d.cachedScratchFile)
-		svm.Unlock()
 	}
 	logrus.Debugf("%s (%s): releasing cachedScratchMutex", title, context)
 	d.cachedScratchMutex.Unlock()
 
 	// Hot-add the scratch-space if not already attached
 	if !svm.scratchAttached {
-		logrus.Debugf("lcowdriver: startServiceVmIfNotRunning: (%s) hot-adding scratch %s - locking serviceVM", context, scratchTargetFile)
-		svm.Lock()
-		if err := svm.config.HotAddVhd(scratchTargetFile, toolsScratchPath, false, true); err != nil {
-			logrus.Debugf("%s (%s): releasing serviceVM on error path of HotAddVhd: %s", title, context, err)
-			svm.Unlock()
-
-			// Do a force terminate and remove it from the map on failure, ignoring any errors
-			if err2 := d.terminateServiceVM(id, "error path from HotAddVhd", true); err2 != nil {
-				logrus.Warnf("failed to terminate service VM on error path from HotAddVhd: %s", err2)
-			}
-
+		logrus.Debugf("lcowdriver: startServiceVmIfNotRunning: (%s) hot-adding scratch %s", context, scratchTargetFile)
+		if err := svm.hotAddVHDsAtStart(hcsshim.MappedVirtualDisk{
+			HostPath:          scratchTargetFile,
+			ContainerPath:     toolsScratchPath,
+			CreateInUtilityVM: true,
+		}); err != nil {
+			logrus.Debugf("%s: failed to hot-add scratch %s: %s", title, scratchTargetFile, err)
 			return nil, fmt.Errorf("failed to hot-add %s failed: %s", scratchTargetFile, err)
 		}
-		logrus.Debugf("%s (%s): releasing serviceVM", title, context)
-		svm.Unlock()
+		svm.scratchAttached = true
 	}
 
 	logrus.Debugf("lcowdriver: startServiceVmIfNotRunning: (%s) success", context)
 	return svm, nil
 }
 
-// getServiceVM returns the appropriate service utility VM instance, optionally
-// deleting it from the map (but not the global one)
-func (d *Driver) getServiceVM(id string, deleteFromMap bool) (*serviceVMItem, error) {
-	logrus.Debugf("lcowdriver: getservicevm:locking serviceVmsMutex")
-	d.serviceVmsMutex.Lock()
-	defer func() {
-		logrus.Debugf("lcowdriver: getservicevm:releasing serviceVmsMutex")
-		d.serviceVmsMutex.Unlock()
-	}()
-	if d.globalMode {
-		id = svmGlobalID
-	}
-	if _, ok := d.serviceVms[id]; !ok {
-		return nil, fmt.Errorf("getservicevm for %s failed as not found", id)
-	}
-	svm := d.serviceVms[id]
-	if deleteFromMap && id != svmGlobalID {
-		logrus.Debugf("lcowdriver: getservicevm: removing %s from map", id)
-		delete(d.serviceVms, id)
+// terminateServiceVM terminates a service utility VM if its running if it's,
+// not being used by any goroutine, but does nothing when in global mode as it's
+// lifetime is limited to that of the daemon. If the force flag is set, then
+// the VM will be killed regardless of the ref count or if it's global.
+func (d *Driver) terminateServiceVM(id, context string, force bool) (err error) {
+	// We don't do anything in safe mode unless the force flag has been passed, which
+	// is only the case for cleanup at driver termination.
+	if d.globalMode && !force {
+		logrus.Debugf("lcowdriver: terminateservicevm: %s (%s) - doing nothing as in global mode", id, context)
+		return nil
 	}
-	return svm, nil
-}
 
-// terminateServiceVM terminates a service utility VM if its running, but does nothing
-// when in global mode as it's lifetime is limited to that of the daemon.
-func (d *Driver) terminateServiceVM(id, context string, force bool) error {
+	id = d.getVMID(id)
 
-	// We don't do anything in safe mode unless the force flag has been passed, which
-	// is only the case for cleanup at driver termination.
-	if d.globalMode {
-		if !force {
-			logrus.Debugf("lcowdriver: terminateservicevm: %s (%s) - doing nothing as in global mode", id, context)
-			return nil
-		}
-		id = svmGlobalID
+	var svm *serviceVM
+	var lastRef bool
+	if !force {
+		// In the not force case, we ref count
+		svm, lastRef, err = d.serviceVms.decrementRefCount(id)
+	} else {
+		// In the force case, we ignore the ref count and just set it to 0
+		svm, err = d.serviceVms.setRefCountZero(id)
+		lastRef = true
 	}
 
-	// Get the service VM and delete it from the map
-	svm, err := d.getServiceVM(id, true)
-	if err != nil {
-		return err
+	if err == errVMUnknown {
+		return nil
+	} else if err == errVMisTerminating {
+		return svm.getStopError()
+	} else if !lastRef {
+		return nil
 	}
 
 	// We run the deletion of the scratch as a deferred function to at least attempt
@@ -459,26 +394,64 @@ func (d *Driver) terminateServiceVM(id, context string, force bool) error {
 		if svm.scratchAttached {
 			scratchTargetFile := filepath.Join(d.dataRoot, scratchDirectory, fmt.Sprintf("%s.vhdx", id))
 			logrus.Debugf("lcowdriver: terminateservicevm: %s (%s) - deleting scratch %s", id, context, scratchTargetFile)
-			if err := os.Remove(scratchTargetFile); err != nil {
-				logrus.Warnf("failed to remove scratch file %s (%s): %s", scratchTargetFile, context, err)
+			if errRemove := os.Remove(scratchTargetFile); errRemove != nil {
+				logrus.Warnf("failed to remove scratch file %s (%s): %s", scratchTargetFile, context, errRemove)
+				err = errRemove
 			}
 		}
+
+		// This function shouldn't actually return error unless there is a bug
+		if errDelete := d.serviceVms.deleteID(id); errDelete != nil {
+			logrus.Warnf("failed to service vm from svm map %s (%s): %s", id, context, errDelete)
+		}
+
+		// Signal that this VM has stopped
+		svm.signalStopFinished(err)
 	}()
 
-	// Nothing to do if it's not running
-	if svm.config.Uvm != nil {
-		logrus.Debugf("lcowdriver: terminateservicevm: %s (%s) - calling terminate", id, context)
-		if err := svm.config.Uvm.Terminate(); err != nil {
-			return fmt.Errorf("failed to terminate utility VM (%s): %s", context, err)
+	// Now it's possible that the serivce VM failed to start and now we are trying to termiante it.
+	// In this case, we will relay the error to the goroutines waiting for this vm to stop.
+	if err := svm.getStartError(); err != nil {
+		logrus.Debugf("lcowdriver: terminateservicevm: %s had failed to start up: %s", id, err)
+		return err
+	}
+
+	if err := waitTerminate(svm, fmt.Sprintf("terminateservicevm: %s (%s)", id, context)); err != nil {
+		return err
+	}
+
+	logrus.Debugf("lcowdriver: terminateservicevm: %s (%s) - success", id, context)
+	return nil
+}
+
+func waitTerminate(svm *serviceVM, context string) error {
+	if svm.config == nil {
+		return fmt.Errorf("lcowdriver: waitTermiante: Nil utility VM. %s", context)
+	}
+
+	logrus.Debugf("lcowdriver: waitTerminate: Calling terminate: %s", context)
+	if err := svm.config.Uvm.Terminate(); err != nil {
+		// We might get operation still pending from the HCS. In that case, we shouldn't return
+		// an error since we call wait right after.
+		underlyingError := err
+		if conterr, ok := err.(*hcsshim.ContainerError); ok {
+			underlyingError = conterr.Err
 		}
 
-		logrus.Debugf("lcowdriver: terminateservicevm: %s (%s) - waiting for utility VM to terminate", id, context)
-		if err := svm.config.Uvm.WaitTimeout(time.Duration(svm.config.UvmTimeoutSeconds) * time.Second); err != nil {
-			return fmt.Errorf("failed waiting for utility VM to terminate (%s): %s", context, err)
+		if syscallErr, ok := underlyingError.(syscall.Errno); ok {
+			underlyingError = syscallErr
+		}
+
+		if underlyingError != errOperationPending {
+			return fmt.Errorf("failed to terminate utility VM (%s): %s", context, err)
 		}
+		logrus.Debugf("lcowdriver: waitTerminate: uvm.Terminate() returned operation pending (%s)", context)
 	}
 
-	logrus.Debugf("lcowdriver: terminateservicevm: %s (%s) - success", id, context)
+	logrus.Debugf("lcowdriver: waitTerminate: (%s) - waiting for utility VM to terminate", context)
+	if err := svm.config.Uvm.WaitTimeout(time.Duration(svm.config.UvmTimeoutSeconds) * time.Second); err != nil {
+		return fmt.Errorf("failed waiting for utility VM to terminate (%s): %s", context, err)
+	}
 	return nil
 }
 
@@ -571,25 +544,18 @@ func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts
 		}()
 	}
 
-	// Synchronise the operation in the service VM.
-	logrus.Debugf("%s: locking svm for sandbox creation", title)
-	svm.Lock()
-	defer func() {
-		logrus.Debugf("%s: releasing svm for sandbox creation", title)
-		svm.Unlock()
-	}()
-
 	// Make sure we don't write to our local cached copy if this is for a non-default size request.
 	targetCacheFile := d.cachedSandboxFile
 	if sandboxSize != client.DefaultVhdxSizeGB {
 		targetCacheFile = ""
 	}
 
-	// Actually do the creation.
-	if err := svm.config.CreateExt4Vhdx(filepath.Join(d.dir(id), sandboxFilename), uint32(sandboxSize), targetCacheFile); err != nil {
+	// Create the ext4 vhdx
+	logrus.Debugf("%s: creating sandbox ext4 vhdx", title)
+	if err := svm.createExt4VHDX(filepath.Join(d.dir(id), sandboxFilename), uint32(sandboxSize), targetCacheFile); err != nil {
+		logrus.Debugf("%s: failed to create sandbox vhdx for %s: %s", title, id, err)
 		return err
 	}
-
 	return nil
 }
 
@@ -638,6 +604,21 @@ func (d *Driver) Remove(id string) error {
 	layerPath := d.dir(id)
 
 	logrus.Debugf("lcowdriver: remove: id %s: layerPath %s", id, layerPath)
+
+	// Unmount all the layers
+	err := d.Put(id)
+	if err != nil {
+		logrus.Debugf("lcowdriver: remove id %s: failed to unmount: %s", id, err)
+		return err
+	}
+
+	// for non-global case just kill the vm
+	if !d.globalMode {
+		if err := d.terminateServiceVM(id, fmt.Sprintf("Remove %s", id), true); err != nil {
+			return err
+		}
+	}
+
 	if err := os.Rename(layerPath, tmpLayerPath); err != nil && !os.IsNotExist(err) {
 		return err
 	}
@@ -659,43 +640,24 @@ func (d *Driver) Remove(id string) error {
 // For optimisation, we don't actually mount the filesystem (which in our
 // case means [hot-]adding it to a service VM. But we track that and defer
 // the actual adding to the point we need to access it.
-func (d *Driver) Get(id, mountLabel string) (string, error) {
+func (d *Driver) Get(id, mountLabel string) (containerfs.ContainerFS, error) {
 	title := fmt.Sprintf("lcowdriver: get: %s", id)
 	logrus.Debugf(title)
 
-	// Work out what we are working on
-	ld, err := getLayerDetails(d.dir(id))
+	// Generate the mounts needed for the defered operation.
+	disks, err := d.getAllMounts(id)
 	if err != nil {
-		logrus.Debugf("%s failed to get layer details from %s: %s", title, d.dir(id), err)
-		return "", fmt.Errorf("%s failed to open layer or sandbox VHD to open in %s: %s", title, d.dir(id), err)
-	}
-	logrus.Debugf("%s %s, size %d, isSandbox %t", title, ld.filename, ld.size, ld.isSandbox)
-
-	// Add item to cache, or update existing item, but ensure we have the
-	// lock while updating items.
-	logrus.Debugf("%s: locking cacheMutex", title)
-	d.cacheMutex.Lock()
-	var ci *cacheItem
-	if item, ok := d.cache[id]; !ok {
-		// The item is not currently in the cache.
-		ci = &cacheItem{
-			refCount:  1,
-			isSandbox: ld.isSandbox,
-			hostPath:  ld.filename,
-			uvmPath:   fmt.Sprintf("/mnt/%s", id),
-			isMounted: false, // we defer this as an optimisation
-		}
-		d.cache[id] = ci
-		logrus.Debugf("%s: added cache item %+v", title, ci)
-	} else {
-		// Increment the reference counter in the cache.
-		item.incrementRefCount()
+		logrus.Debugf("%s failed to get all layer details for %s: %s", title, d.dir(id), err)
+		return nil, fmt.Errorf("%s failed to get layer details for %s: %s", title, d.dir(id), err)
 	}
-	logrus.Debugf("%s: releasing cacheMutex", title)
-	d.cacheMutex.Unlock()
 
-	logrus.Debugf("%s %s success. %s: %+v: size %d", title, id, d.dir(id), ci, ld.size)
-	return d.dir(id), nil
+	logrus.Debugf("%s: got layer mounts: %+v", title, disks)
+	return &lcowfs{
+		root:        unionMountName(disks),
+		d:           d,
+		mappedDisks: disks,
+		vmID:        d.getVMID(id),
+	}, nil
 }
 
 // Put does the reverse of get. If there are no more references to
@@ -703,56 +665,45 @@ func (d *Driver) Get(id, mountLabel string) (string, error) {
 func (d *Driver) Put(id string) error {
 	title := fmt.Sprintf("lcowdriver: put: %s", id)
 
-	logrus.Debugf("%s: locking cacheMutex", title)
-	d.cacheMutex.Lock()
-	item, ok := d.cache[id]
-	if !ok {
-		logrus.Debugf("%s: releasing cacheMutex on error path", title)
-		d.cacheMutex.Unlock()
-		return fmt.Errorf("%s possible ref-count error, or invalid id was passed to the graphdriver. Cannot handle id %s as it's not in the cache", title, id)
-	}
-
-	// Decrement the ref-count, and nothing more to do if still in use.
-	if item.decrementRefCount() > 0 {
-		logrus.Debugf("%s: releasing cacheMutex. Cache item is still in use", title)
-		d.cacheMutex.Unlock()
+	// Get the service VM that we need to remove from
+	svm, err := d.serviceVms.get(d.getVMID(id))
+	if err == errVMUnknown {
 		return nil
+	} else if err == errVMisTerminating {
+		return svm.getStopError()
 	}
 
-	// Remove from the cache map.
-	delete(d.cache, id)
-	logrus.Debugf("%s: releasing cacheMutex. Ref count on cache item has dropped to zero, removed from cache", title)
-	d.cacheMutex.Unlock()
+	// Generate the mounts that Get() might have mounted
+	disks, err := d.getAllMounts(id)
+	if err != nil {
+		logrus.Debugf("%s failed to get all layer details for %s: %s", title, d.dir(id), err)
+		return fmt.Errorf("%s failed to get layer details for %s: %s", title, d.dir(id), err)
+	}
 
-	// If we have done a mount and we are in global mode, then remove it. We don't
-	// need to remove in safe mode as the service VM is going to be torn down anyway.
-	if d.globalMode {
-		logrus.Debugf("%s: locking cache item at zero ref-count", title)
-		item.Lock()
-		defer func() {
-			logrus.Debugf("%s: releasing cache item at zero ref-count", title)
-			item.Unlock()
-		}()
-		if item.isMounted {
-			svm, err := d.getServiceVM(id, false)
-			if err != nil {
-				return err
-			}
+	// Now, we want to perform the unmounts, hot-remove and stop the service vm.
+	// We want to go though all the steps even if we have an error to clean up properly
+	err = svm.deleteUnionMount(unionMountName(disks), disks...)
+	if err != nil {
+		logrus.Debugf("%s failed to delete union mount %s: %s", title, id, err)
+	}
 
-			logrus.Debugf("%s: Hot-Removing %s. Locking svm", title, item.hostPath)
-			svm.Lock()
-			if err := svm.config.HotRemoveVhd(item.hostPath); err != nil {
-				logrus.Debugf("%s: releasing svm on error path", title)
-				svm.Unlock()
-				return fmt.Errorf("%s failed to hot-remove %s from global service utility VM: %s", title, item.hostPath, err)
-			}
-			logrus.Debugf("%s: releasing svm", title)
-			svm.Unlock()
+	err1 := svm.hotRemoveVHDs(disks...)
+	if err1 != nil {
+		logrus.Debugf("%s failed to hot remove vhds %s: %s", title, id, err)
+		if err == nil {
+			err = err1
 		}
 	}
 
-	logrus.Debugf("%s %s: refCount 0. %s (%s) completed successfully", title, id, item.hostPath, item.uvmPath)
-	return nil
+	err1 = d.terminateServiceVM(id, fmt.Sprintf("Put %s", id), false)
+	if err1 != nil {
+		logrus.Debugf("%s failed to terminate service vm %s: %s", title, id, err1)
+		if err == nil {
+			err = err1
+		}
+	}
+	logrus.Debugf("Put succeeded on id %s", id)
+	return err
 }
 
 // Cleanup ensures the information the driver stores is properly removed.
@@ -761,15 +712,6 @@ func (d *Driver) Put(id string) error {
 func (d *Driver) Cleanup() error {
 	title := "lcowdriver: cleanup"
 
-	d.cacheMutex.Lock()
-	for k, v := range d.cache {
-		logrus.Debugf("%s cache item: %s: %+v", title, k, v)
-		if v.refCount > 0 {
-			logrus.Warnf("%s leaked %s: %+v", title, k, v)
-		}
-	}
-	d.cacheMutex.Unlock()
-
 	items, err := ioutil.ReadDir(d.dataRoot)
 	if err != nil {
 		if os.IsNotExist(err) {
@@ -794,8 +736,8 @@ func (d *Driver) Cleanup() error {
 
 	// Cleanup any service VMs we have running, along with their scratch spaces.
 	// We don't take the lock for this as it's taken in terminateServiceVm.
-	for k, v := range d.serviceVms {
-		logrus.Debugf("%s svm: %s: %+v", title, k, v)
+	for k, v := range d.serviceVms.svms {
+		logrus.Debugf("%s svm entry: %s: %+v", title, k, v)
 		d.terminateServiceVM(k, "cleanup", true)
 	}
 
@@ -812,65 +754,41 @@ func (d *Driver) Cleanup() error {
 func (d *Driver) Diff(id, parent string) (io.ReadCloser, error) {
 	title := fmt.Sprintf("lcowdriver: diff: %s", id)
 
-	logrus.Debugf("%s: locking cacheMutex", title)
-	d.cacheMutex.Lock()
-	if _, ok := d.cache[id]; !ok {
-		logrus.Debugf("%s: releasing cacheMutex on error path", title)
-		d.cacheMutex.Unlock()
-		return nil, fmt.Errorf("%s fail as %s is not in the cache", title, id)
-	}
-	ci := d.cache[id]
-	logrus.Debugf("%s: releasing cacheMutex", title)
-	d.cacheMutex.Unlock()
-
-	// Stat to get size
-	logrus.Debugf("%s: locking cacheItem", title)
-	ci.Lock()
-	fileInfo, err := os.Stat(ci.hostPath)
+	// Get VHDX info
+	ld, err := getLayerDetails(d.dir(id))
 	if err != nil {
-		logrus.Debugf("%s: releasing cacheItem on error path", title)
-		ci.Unlock()
-		return nil, fmt.Errorf("%s failed to stat %s: %s", title, ci.hostPath, err)
+		logrus.Debugf("%s: failed to get vhdx information of %s: %s", title, d.dir(id), err)
+		return nil, err
 	}
-	logrus.Debugf("%s: releasing cacheItem", title)
-	ci.Unlock()
 
 	// Start the SVM with a mapped virtual disk. Note that if the SVM is
 	// already running and we are in global mode, this will be
 	// hot-added.
-	mvd := &hcsshim.MappedVirtualDisk{
-		HostPath:          ci.hostPath,
-		ContainerPath:     ci.uvmPath,
+	mvd := hcsshim.MappedVirtualDisk{
+		HostPath:          ld.filename,
+		ContainerPath:     hostToGuest(ld.filename),
 		CreateInUtilityVM: true,
 		ReadOnly:          true,
 	}
 
 	logrus.Debugf("%s: starting service VM", title)
-	svm, err := d.startServiceVMIfNotRunning(id, mvd, fmt.Sprintf("diff %s", id))
+	svm, err := d.startServiceVMIfNotRunning(id, []hcsshim.MappedVirtualDisk{mvd}, fmt.Sprintf("diff %s", id))
 	if err != nil {
 		return nil, err
 	}
 
-	// Set `isMounted` for the cache item. Note that we re-scan the cache
-	// at this point as it's possible the cacheItem changed during the long-
-	// running operation above when we weren't holding the cacheMutex lock.
-	logrus.Debugf("%s: locking cacheMutex for updating isMounted", title)
-	d.cacheMutex.Lock()
-	if _, ok := d.cache[id]; !ok {
-		logrus.Debugf("%s: releasing cacheMutex on error path of isMounted", title)
-		d.cacheMutex.Unlock()
+	logrus.Debugf("lcowdriver: diff: waiting for svm to finish booting")
+	err = svm.getStartError()
+	if err != nil {
 		d.terminateServiceVM(id, fmt.Sprintf("diff %s", id), false)
-		return nil, fmt.Errorf("%s fail as %s is not in the cache when updating isMounted", title, id)
+		return nil, fmt.Errorf("lcowdriver: diff: svm failed to boot: %s", err)
 	}
-	ci = d.cache[id]
-	ci.setIsMounted()
-	logrus.Debugf("%s: releasing cacheMutex for updating isMounted", title)
-	d.cacheMutex.Unlock()
 
 	// Obtain the tar stream for it
-	logrus.Debugf("%s %s, size %d, isSandbox %t", title, ci.hostPath, fileInfo.Size(), ci.isSandbox)
-	tarReadCloser, err := svm.config.VhdToTar(ci.hostPath, ci.uvmPath, ci.isSandbox, fileInfo.Size())
+	logrus.Debugf("%s: %s %s, size %d, ReadOnly %t", title, ld.filename, mvd.ContainerPath, ld.size, ld.isSandbox)
+	tarReadCloser, err := svm.config.VhdToTar(mvd.HostPath, mvd.ContainerPath, ld.isSandbox, ld.size)
 	if err != nil {
+		svm.hotRemoveVHDs(mvd)
 		d.terminateServiceVM(id, fmt.Sprintf("diff %s", id), false)
 		return nil, fmt.Errorf("%s failed to export layer to tar stream for id: %s, parent: %s : %s", title, id, parent, err)
 	}
@@ -878,14 +796,12 @@ func (d *Driver) Diff(id, parent string) (io.ReadCloser, error) {
 	logrus.Debugf("%s id %s parent %s completed successfully", title, id, parent)
 
 	// In safe/non-global mode, we can't tear down the service VM until things have been read.
-	if !d.globalMode {
-		return ioutils.NewReadCloserWrapper(tarReadCloser, func() error {
-			tarReadCloser.Close()
-			d.terminateServiceVM(id, fmt.Sprintf("diff %s", id), false)
-			return nil
-		}), nil
-	}
-	return tarReadCloser, nil
+	return ioutils.NewReadCloserWrapper(tarReadCloser, func() error {
+		tarReadCloser.Close()
+		svm.hotRemoveVHDs(mvd)
+		d.terminateServiceVM(id, fmt.Sprintf("diff %s", id), false)
+		return nil
+	}), nil
 }
 
 // ApplyDiff extracts the changeset from the given diff into the
@@ -902,6 +818,12 @@ func (d *Driver) ApplyDiff(id, parent string, diff io.Reader) (int64, error) {
 	}
 	defer d.terminateServiceVM(id, fmt.Sprintf("applydiff %s", id), false)
 
+	logrus.Debugf("lcowdriver: applydiff: waiting for svm to finish booting")
+	err = svm.getStartError()
+	if err != nil {
+		return 0, fmt.Errorf("lcowdriver: applydiff: svm failed to boot: %s", err)
+	}
+
 	// TODO @jhowardmsft - the retries are temporary to overcome platform reliablity issues.
 	// Obviously this will be removed as platform bugs are fixed.
 	retries := 0
@@ -944,6 +866,11 @@ func (d *Driver) GetMetadata(id string) (map[string]string, error) {
 	return m, nil
 }
 
+// GetLayerPath gets the layer path on host (path to VHD/VHDX)
+func (d *Driver) GetLayerPath(id string) (string, error) {
+	return d.dir(id), nil
+}
+
 // dir returns the absolute path to the layer.
 func (d *Driver) dir(id string) string {
 	return filepath.Join(d.dataRoot, filepath.Base(id))
@@ -1006,3 +933,34 @@ func getLayerDetails(folder string) (*layerDetails, error) {
 
 	return ld, nil
 }
+
+func (d *Driver) getAllMounts(id string) ([]hcsshim.MappedVirtualDisk, error) {
+	layerChain, err := d.getLayerChain(id)
+	if err != nil {
+		return nil, err
+	}
+	layerChain = append([]string{d.dir(id)}, layerChain...)
+
+	logrus.Debugf("getting all  layers: %v", layerChain)
+	disks := make([]hcsshim.MappedVirtualDisk, len(layerChain), len(layerChain))
+	for i := range layerChain {
+		ld, err := getLayerDetails(layerChain[i])
+		if err != nil {
+			logrus.Debugf("Failed to get LayerVhdDetails from %s: %s", layerChain[i], err)
+			return nil, err
+		}
+		disks[i].HostPath = ld.filename
+		disks[i].ContainerPath = hostToGuest(ld.filename)
+		disks[i].CreateInUtilityVM = true
+		disks[i].ReadOnly = !ld.isSandbox
+	}
+	return disks, nil
+}
+
+func hostToGuest(hostpath string) string {
+	return fmt.Sprintf("/tmp/%s", filepath.Base(filepath.Dir(hostpath)))
+}
+
+func unionMountName(disks []hcsshim.MappedVirtualDisk) string {
+	return fmt.Sprintf("%s-mount", disks[0].ContainerPath)
+}

daemon/graphdriver/lcow/lcow_svm.go

@@ -0,0 +1,373 @@
+// +build windows
+
+package lcow
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/Microsoft/hcsshim"
+	"github.com/Microsoft/opengcs/client"
+	"github.com/sirupsen/logrus"
+)
+
+// Code for all the service VM management for the LCOW graphdriver
+
+var errVMisTerminating = errors.New("service VM is shutting down")
+var errVMUnknown = errors.New("service vm id is unknown")
+var errVMStillHasReference = errors.New("Attemping to delete a VM that is still being used")
+
+// serviceVMMap is the struct representing the id -> service VM mapping.
+type serviceVMMap struct {
+	sync.Mutex
+	svms map[string]*serviceVMMapItem
+}
+
+// serviceVMMapItem is our internal structure representing an item in our
+// map of service VMs we are maintaining.
+type serviceVMMapItem struct {
+	svm      *serviceVM // actual service vm object
+	refCount int        // refcount for VM
+}
+
+type serviceVM struct {
+	sync.Mutex                     // Serialises operations being performed in this service VM.
+	scratchAttached bool           // Has a scratch been attached?
+	config          *client.Config // Represents the service VM item.
+
+	// Indicates that the vm is started
+	startStatus chan interface{}
+	startError  error
+
+	// Indicates that the vm is stopped
+	stopStatus chan interface{}
+	stopError  error
+
+	attachedVHDs map[string]int // Map ref counting all the VHDS we've hot-added/hot-removed.
+	unionMounts  map[string]int // Map ref counting all the union filesystems we mounted.
+}
+
+// add will add an id to the service vm map. There are three cases:
+// 	- entry doesn't exist:
+// 		- add id to map and return a new vm that the caller can manually configure+start
+//	- entry does exist
+//  	- return vm in map and increment ref count
+//  - entry does exist but the ref count is 0
+//		- return the svm and errVMisTerminating. Caller can call svm.getStopError() to wait for stop
+func (svmMap *serviceVMMap) add(id string) (svm *serviceVM, alreadyExists bool, err error) {
+	svmMap.Lock()
+	defer svmMap.Unlock()
+	if svm, ok := svmMap.svms[id]; ok {
+		if svm.refCount == 0 {
+			return svm.svm, true, errVMisTerminating
+		}
+		svm.refCount++
+		return svm.svm, true, nil
+	}
+
+	// Doesn't exist, so create an empty svm to put into map and return
+	newSVM := &serviceVM{
+		startStatus:  make(chan interface{}),
+		stopStatus:   make(chan interface{}),
+		attachedVHDs: make(map[string]int),
+		unionMounts:  make(map[string]int),
+		config:       &client.Config{},
+	}
+	svmMap.svms[id] = &serviceVMMapItem{
+		svm:      newSVM,
+		refCount: 1,
+	}
+	return newSVM, false, nil
+}
+
+// get will get the service vm from the map. There are three cases:
+// 	- entry doesn't exist:
+// 		- return errVMUnknown
+//	- entry does exist
+//  	- return vm with no error
+//  - entry does exist but the ref count is 0
+//		- return the svm and errVMisTerminating. Caller can call svm.getStopError() to wait for stop
+func (svmMap *serviceVMMap) get(id string) (*serviceVM, error) {
+	svmMap.Lock()
+	defer svmMap.Unlock()
+	svm, ok := svmMap.svms[id]
+	if !ok {
+		return nil, errVMUnknown
+	}
+	if svm.refCount == 0 {
+		return svm.svm, errVMisTerminating
+	}
+	return svm.svm, nil
+}
+
+// decrementRefCount decrements the ref count of the given ID from the map. There are four cases:
+// 	- entry doesn't exist:
+// 		- return errVMUnknown
+//  - entry does exist but the ref count is 0
+//		- return the svm and errVMisTerminating. Caller can call svm.getStopError() to wait for stop
+//	- entry does exist but ref count is 1
+//  	- return vm and set lastRef to true. The caller can then stop the vm, delete the id from this map
+//      - and execute svm.signalStopFinished to signal the threads that the svm has been terminated.
+//	- entry does exist and ref count > 1
+//		- just reduce ref count and return svm
+func (svmMap *serviceVMMap) decrementRefCount(id string) (_ *serviceVM, lastRef bool, _ error) {
+	svmMap.Lock()
+	defer svmMap.Unlock()
+
+	svm, ok := svmMap.svms[id]
+	if !ok {
+		return nil, false, errVMUnknown
+	}
+	if svm.refCount == 0 {
+		return svm.svm, false, errVMisTerminating
+	}
+	svm.refCount--
+	return svm.svm, svm.refCount == 0, nil
+}
+
+// setRefCountZero works the same way as decrementRefCount, but sets ref count to 0 instead of decrementing it.
+func (svmMap *serviceVMMap) setRefCountZero(id string) (*serviceVM, error) {
+	svmMap.Lock()
+	defer svmMap.Unlock()
+
+	svm, ok := svmMap.svms[id]
+	if !ok {
+		return nil, errVMUnknown
+	}
+	if svm.refCount == 0 {
+		return svm.svm, errVMisTerminating
+	}
+	svm.refCount = 0
+	return svm.svm, nil
+}
+
+// deleteID deletes the given ID from the map. If the refcount is not 0 or the
+// VM does not exist, then this function returns an error.
+func (svmMap *serviceVMMap) deleteID(id string) error {
+	svmMap.Lock()
+	defer svmMap.Unlock()
+	svm, ok := svmMap.svms[id]
+	if !ok {
+		return errVMUnknown
+	}
+	if svm.refCount != 0 {
+		return errVMStillHasReference
+	}
+	delete(svmMap.svms, id)
+	return nil
+}
+
+func (svm *serviceVM) signalStartFinished(err error) {
+	svm.Lock()
+	svm.startError = err
+	svm.Unlock()
+	close(svm.startStatus)
+}
+
+func (svm *serviceVM) getStartError() error {
+	<-svm.startStatus
+	svm.Lock()
+	defer svm.Unlock()
+	return svm.startError
+}
+
+func (svm *serviceVM) signalStopFinished(err error) {
+	svm.Lock()
+	svm.stopError = err
+	svm.Unlock()
+	close(svm.stopStatus)
+}
+
+func (svm *serviceVM) getStopError() error {
+	<-svm.stopStatus
+	svm.Lock()
+	defer svm.Unlock()
+	return svm.stopError
+}
+
+// hotAddVHDs waits for the service vm to start and then attaches the vhds.
+func (svm *serviceVM) hotAddVHDs(mvds ...hcsshim.MappedVirtualDisk) error {
+	if err := svm.getStartError(); err != nil {
+		return err
+	}
+	return svm.hotAddVHDsAtStart(mvds...)
+}
+
+// hotAddVHDsAtStart works the same way as hotAddVHDs but does not wait for the VM to start.
+func (svm *serviceVM) hotAddVHDsAtStart(mvds ...hcsshim.MappedVirtualDisk) error {
+	svm.Lock()
+	defer svm.Unlock()
+	for i, mvd := range mvds {
+		if _, ok := svm.attachedVHDs[mvd.HostPath]; ok {
+			svm.attachedVHDs[mvd.HostPath]++
+			continue
+		}
+
+		if err := svm.config.HotAddVhd(mvd.HostPath, mvd.ContainerPath, mvd.ReadOnly, !mvd.AttachOnly); err != nil {
+			svm.hotRemoveVHDsAtStart(mvds[:i]...)
+			return err
+		}
+		svm.attachedVHDs[mvd.HostPath] = 1
+	}
+	return nil
+}
+
+// hotRemoveVHDs waits for the service vm to start and then removes the vhds.
+func (svm *serviceVM) hotRemoveVHDs(mvds ...hcsshim.MappedVirtualDisk) error {
+	if err := svm.getStartError(); err != nil {
+		return err
+	}
+	return svm.hotRemoveVHDsAtStart(mvds...)
+}
+
+// hotRemoveVHDsAtStart works the same way as hotRemoveVHDs but does not wait for the VM to start.
+func (svm *serviceVM) hotRemoveVHDsAtStart(mvds ...hcsshim.MappedVirtualDisk) error {
+	svm.Lock()
+	defer svm.Unlock()
+	var retErr error
+	for _, mvd := range mvds {
+		if _, ok := svm.attachedVHDs[mvd.HostPath]; !ok {
+			// We continue instead of returning an error if we try to hot remove a non-existent VHD.
+			// This is because one of the callers of the function is graphdriver.Put(). Since graphdriver.Get()
+			// defers the VM start to the first operation, it's possible that nothing have been hot-added
+			// when Put() is called. To avoid Put returning an error in that case, we simply continue if we
+			// don't find the vhd attached.
+			continue
+		}
+
+		if svm.attachedVHDs[mvd.HostPath] > 1 {
+			svm.attachedVHDs[mvd.HostPath]--
+			continue
+		}
+
+		// last VHD, so remove from VM and map
+		if err := svm.config.HotRemoveVhd(mvd.HostPath); err == nil {
+			delete(svm.attachedVHDs, mvd.HostPath)
+		} else {
+			// Take note of the error, but still continue to remove the other VHDs
+			logrus.Warnf("Failed to hot remove %s: %s", mvd.HostPath, err)
+			if retErr == nil {
+				retErr = err
+			}
+		}
+	}
+	return retErr
+}
+
+func (svm *serviceVM) createExt4VHDX(destFile string, sizeGB uint32, cacheFile string) error {
+	if err := svm.getStartError(); err != nil {
+		return err
+	}
+
+	svm.Lock()
+	defer svm.Unlock()
+	return svm.config.CreateExt4Vhdx(destFile, sizeGB, cacheFile)
+}
+
+func (svm *serviceVM) createUnionMount(mountName string, mvds ...hcsshim.MappedVirtualDisk) (err error) {
+	if len(mvds) == 0 {
+		return fmt.Errorf("createUnionMount: error must have at least 1 layer")
+	}
+
+	if err = svm.getStartError(); err != nil {
+		return err
+	}
+
+	svm.Lock()
+	defer svm.Unlock()
+	if _, ok := svm.unionMounts[mountName]; ok {
+		svm.unionMounts[mountName]++
+		return nil
+	}
+
+	var lowerLayers []string
+	if mvds[0].ReadOnly {
+		lowerLayers = append(lowerLayers, mvds[0].ContainerPath)
+	}
+
+	for i := 1; i < len(mvds); i++ {
+		lowerLayers = append(lowerLayers, mvds[i].ContainerPath)
+	}
+
+	logrus.Debugf("Doing the overlay mount with union directory=%s", mountName)
+	if err = svm.runProcess(fmt.Sprintf("mkdir -p %s", mountName), nil, nil, nil); err != nil {
+		return err
+	}
+
+	var cmd string
+	if mvds[0].ReadOnly {
+		// Readonly overlay
+		cmd = fmt.Sprintf("mount -t overlay overlay -olowerdir=%s %s",
+			strings.Join(lowerLayers, ","),
+			mountName)
+	} else {
+		upper := fmt.Sprintf("%s/upper", mvds[0].ContainerPath)
+		work := fmt.Sprintf("%s/work", mvds[0].ContainerPath)
+
+		if err = svm.runProcess(fmt.Sprintf("mkdir -p %s %s", upper, work), nil, nil, nil); err != nil {
+			return err
+		}
+
+		cmd = fmt.Sprintf("mount -t overlay overlay -olowerdir=%s,upperdir=%s,workdir=%s %s",
+			strings.Join(lowerLayers, ":"),
+			upper,
+			work,
+			mountName)
+	}
+
+	logrus.Debugf("createUnionMount: Executing mount=%s", cmd)
+	if err = svm.runProcess(cmd, nil, nil, nil); err != nil {
+		return err
+	}
+
+	svm.unionMounts[mountName] = 1
+	return nil
+}
+
+func (svm *serviceVM) deleteUnionMount(mountName string, disks ...hcsshim.MappedVirtualDisk) error {
+	if err := svm.getStartError(); err != nil {
+		return err
+	}
+
+	svm.Lock()
+	defer svm.Unlock()
+	if _, ok := svm.unionMounts[mountName]; !ok {
+		return nil
+	}
+
+	if svm.unionMounts[mountName] > 1 {
+		svm.unionMounts[mountName]--
+		return nil
+	}
+
+	logrus.Debugf("Removing union mount %s", mountName)
+	if err := svm.runProcess(fmt.Sprintf("umount %s", mountName), nil, nil, nil); err != nil {
+		return err
+	}
+
+	delete(svm.unionMounts, mountName)
+	return nil
+}
+
+func (svm *serviceVM) runProcess(command string, stdin io.Reader, stdout io.Writer, stderr io.Writer) error {
+	process, err := svm.config.RunProcess(command, stdin, stdout, stderr)
+	if err != nil {
+		return err
+	}
+	defer process.Close()
+
+	process.WaitTimeout(time.Duration(int(time.Second) * svm.config.UvmTimeoutSeconds))
+	exitCode, err := process.ExitCode()
+	if err != nil {
+		return err
+	}
+
+	if exitCode != 0 {
+		return fmt.Errorf("svm.runProcess: command %s failed with exit code %d", command, exitCode)
+	}
+	return nil
+}

daemon/graphdriver/lcow/remotefs.go

@@ -0,0 +1,139 @@
+// +build windows
+
+package lcow
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"runtime"
+	"strings"
+	"sync"
+
+	"github.com/Microsoft/hcsshim"
+	"github.com/Microsoft/opengcs/service/gcsutils/remotefs"
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/containerfs"
+	"github.com/sirupsen/logrus"
+)
+
+type lcowfs struct {
+	root        string
+	d           *Driver
+	mappedDisks []hcsshim.MappedVirtualDisk
+	vmID        string
+	currentSVM  *serviceVM
+	sync.Mutex
+}
+
+var _ containerfs.ContainerFS = &lcowfs{}
+
+// ErrNotSupported is an error for unsupported operations in the remotefs
+var ErrNotSupported = fmt.Errorf("not supported")
+
+// Functions to implement the ContainerFS interface
+func (l *lcowfs) Path() string {
+	return l.root
+}
+
+func (l *lcowfs) ResolveScopedPath(path string, rawPath bool) (string, error) {
+	logrus.Debugf("remotefs.resolvescopedpath inputs: %s %s ", path, l.root)
+
+	arg1 := l.Join(l.root, path)
+	if !rawPath {
+		// The l.Join("/", path) will make path an absolute path and then clean it
+		// so if path = ../../X, it will become /X.
+		arg1 = l.Join(l.root, l.Join("/", path))
+	}
+	arg2 := l.root
+
+	output := &bytes.Buffer{}
+	if err := l.runRemoteFSProcess(nil, output, remotefs.ResolvePathCmd, arg1, arg2); err != nil {
+		return "", err
+	}
+
+	logrus.Debugf("remotefs.resolvescopedpath success. Output: %s\n", output.String())
+	return output.String(), nil
+}
+
+func (l *lcowfs) OS() string {
+	return "linux"
+}
+
+func (l *lcowfs) Architecture() string {
+	return runtime.GOARCH
+}
+
+// Other functions that are used by docker like the daemon Archiver/Extractor
+func (l *lcowfs) ExtractArchive(src io.Reader, dst string, opts *archive.TarOptions) error {
+	logrus.Debugf("remotefs.ExtractArchve inputs: %s %+v", dst, opts)
+
+	tarBuf := &bytes.Buffer{}
+	if err := remotefs.WriteTarOptions(tarBuf, opts); err != nil {
+		return fmt.Errorf("failed to marshall tar opts: %s", err)
+	}
+
+	input := io.MultiReader(tarBuf, src)
+	if err := l.runRemoteFSProcess(input, nil, remotefs.ExtractArchiveCmd, dst); err != nil {
+		return fmt.Errorf("failed to extract archive to %s: %s", dst, err)
+	}
+	return nil
+}
+
+func (l *lcowfs) ArchivePath(src string, opts *archive.TarOptions) (io.ReadCloser, error) {
+	logrus.Debugf("remotefs.ArchivePath: %s %+v", src, opts)
+
+	tarBuf := &bytes.Buffer{}
+	if err := remotefs.WriteTarOptions(tarBuf, opts); err != nil {
+		return nil, fmt.Errorf("failed to marshall tar opts: %s", err)
+	}
+
+	r, w := io.Pipe()
+	go func() {
+		defer w.Close()
+		if err := l.runRemoteFSProcess(tarBuf, w, remotefs.ArchivePathCmd, src); err != nil {
+			logrus.Debugf("REMOTEFS: Failed to extract archive: %s %+v %s", src, opts, err)
+		}
+	}()
+	return r, nil
+}
+
+// Helper functions
+func (l *lcowfs) startVM() error {
+	l.Lock()
+	defer l.Unlock()
+	if l.currentSVM != nil {
+		return nil
+	}
+
+	svm, err := l.d.startServiceVMIfNotRunning(l.vmID, l.mappedDisks, fmt.Sprintf("lcowfs.startVM"))
+	if err != nil {
+		return err
+	}
+
+	if err = svm.createUnionMount(l.root, l.mappedDisks...); err != nil {
+		return err
+	}
+	l.currentSVM = svm
+	return nil
+}
+
+func (l *lcowfs) runRemoteFSProcess(stdin io.Reader, stdout io.Writer, args ...string) error {
+	if err := l.startVM(); err != nil {
+		return err
+	}
+
+	// Append remotefs prefix and setup as a command line string
+	cmd := fmt.Sprintf("%s %s", remotefs.RemotefsCmd, strings.Join(args, " "))
+	stderr := &bytes.Buffer{}
+	if err := l.currentSVM.runProcess(cmd, stdin, stdout, stderr); err != nil {
+		return err
+	}
+
+	eerr, err := remotefs.ReadError(stderr)
+	if eerr != nil {
+		// Process returned an error so return that.
+		return remotefs.ExportedToError(eerr)
+	}
+	return err
+}

daemon/graphdriver/lcow/remotefs_file.go

@@ -0,0 +1,211 @@
+// +build windows
+
+package lcow
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"io"
+	"os"
+	"strconv"
+
+	"github.com/Microsoft/hcsshim"
+	"github.com/Microsoft/opengcs/service/gcsutils/remotefs"
+	"github.com/containerd/continuity/driver"
+)
+
+type lcowfile struct {
+	process   hcsshim.Process
+	stdin     io.WriteCloser
+	stdout    io.ReadCloser
+	stderr    io.ReadCloser
+	fs        *lcowfs
+	guestPath string
+}
+
+func (l *lcowfs) Open(path string) (driver.File, error) {
+	return l.OpenFile(path, os.O_RDONLY, 0)
+}
+
+func (l *lcowfs) OpenFile(path string, flag int, perm os.FileMode) (_ driver.File, err error) {
+	flagStr := strconv.FormatInt(int64(flag), 10)
+	permStr := strconv.FormatUint(uint64(perm), 8)
+
+	commandLine := fmt.Sprintf("%s %s %s %s", remotefs.RemotefsCmd, remotefs.OpenFileCmd, flagStr, permStr)
+	env := make(map[string]string)
+	env["PATH"] = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:"
+	processConfig := &hcsshim.ProcessConfig{
+		EmulateConsole:    false,
+		CreateStdInPipe:   true,
+		CreateStdOutPipe:  true,
+		CreateStdErrPipe:  true,
+		CreateInUtilityVm: true,
+		WorkingDirectory:  "/bin",
+		Environment:       env,
+		CommandLine:       commandLine,
+	}
+
+	process, err := l.currentSVM.config.Uvm.CreateProcess(processConfig)
+	if err != nil {
+		return nil, fmt.Errorf("failed to open file %s: %s", path, err)
+	}
+
+	stdin, stdout, stderr, err := process.Stdio()
+	if err != nil {
+		process.Kill()
+		process.Close()
+		return nil, fmt.Errorf("failed to open file pipes %s: %s", path, err)
+	}
+
+	lf := &lcowfile{
+		process:   process,
+		stdin:     stdin,
+		stdout:    stdout,
+		stderr:    stderr,
+		fs:        l,
+		guestPath: path,
+	}
+
+	if _, err := lf.getResponse(); err != nil {
+		return nil, fmt.Errorf("failed to open file %s: %s", path, err)
+	}
+	return lf, nil
+}
+
+func (l *lcowfile) Read(b []byte) (int, error) {
+	hdr := &remotefs.FileHeader{
+		Cmd:  remotefs.Read,
+		Size: uint64(len(b)),
+	}
+
+	if err := remotefs.WriteFileHeader(l.stdin, hdr, nil); err != nil {
+		return 0, err
+	}
+
+	buf, err := l.getResponse()
+	if err != nil {
+		return 0, nil
+	}
+
+	n := copy(b, buf)
+	return n, nil
+}
+
+func (l *lcowfile) Write(b []byte) (int, error) {
+	hdr := &remotefs.FileHeader{
+		Cmd:  remotefs.Write,
+		Size: uint64(len(b)),
+	}
+
+	if err := remotefs.WriteFileHeader(l.stdin, hdr, b); err != nil {
+		return 0, err
+	}
+
+	_, err := l.getResponse()
+	if err != nil {
+		return 0, nil
+	}
+
+	return len(b), nil
+}
+
+func (l *lcowfile) Seek(offset int64, whence int) (int64, error) {
+	seekHdr := &remotefs.SeekHeader{
+		Offset: offset,
+		Whence: int32(whence),
+	}
+
+	buf := &bytes.Buffer{}
+	if err := binary.Write(buf, binary.BigEndian, seekHdr); err != nil {
+		return 0, err
+	}
+
+	hdr := &remotefs.FileHeader{
+		Cmd:  remotefs.Write,
+		Size: uint64(buf.Len()),
+	}
+	if err := remotefs.WriteFileHeader(l.stdin, hdr, buf.Bytes()); err != nil {
+		return 0, err
+	}
+
+	resBuf, err := l.getResponse()
+	if err != nil {
+		return 0, err
+	}
+
+	var res int64
+	if err := binary.Read(bytes.NewBuffer(resBuf), binary.BigEndian, &res); err != nil {
+		return 0, err
+	}
+	return res, nil
+}
+
+func (l *lcowfile) Close() error {
+	hdr := &remotefs.FileHeader{
+		Cmd:  remotefs.Close,
+		Size: 0,
+	}
+
+	if err := remotefs.WriteFileHeader(l.stdin, hdr, nil); err != nil {
+		return err
+	}
+
+	_, err := l.getResponse()
+	return err
+}
+
+func (l *lcowfile) Readdir(n int) ([]os.FileInfo, error) {
+	nStr := strconv.FormatInt(int64(n), 10)
+
+	// Unlike the other File functions, this one can just be run without maintaining state,
+	// so just do the normal runRemoteFSProcess way.
+	buf := &bytes.Buffer{}
+	if err := l.fs.runRemoteFSProcess(nil, buf, remotefs.ReadDirCmd, l.guestPath, nStr); err != nil {
+		return nil, err
+	}
+
+	var info []remotefs.FileInfo
+	if err := json.Unmarshal(buf.Bytes(), &info); err != nil {
+		return nil, nil
+	}
+
+	osInfo := make([]os.FileInfo, len(info))
+	for i := range info {
+		osInfo[i] = &info[i]
+	}
+	return osInfo, nil
+}
+
+func (l *lcowfile) getResponse() ([]byte, error) {
+	hdr, err := remotefs.ReadFileHeader(l.stdout)
+	if err != nil {
+		return nil, err
+	}
+
+	if hdr.Cmd != remotefs.CmdOK {
+		// Something went wrong during the openfile in the server.
+		// Parse stderr and return that as an error
+		eerr, err := remotefs.ReadError(l.stderr)
+		if eerr != nil {
+			return nil, remotefs.ExportedToError(eerr)
+		}
+
+		// Maybe the parsing went wrong?
+		if err != nil {
+			return nil, err
+		}
+
+		// At this point, we know something went wrong in the remotefs program, but
+		// we we don't know why.
+		return nil, fmt.Errorf("unknown error")
+	}
+
+	// Successful command, we might have some data to read (for Read + Seek)
+	buf := make([]byte, hdr.Size, hdr.Size)
+	if _, err := io.ReadFull(l.stdout, buf); err != nil {
+		return nil, err
+	}
+	return buf, nil
+}

daemon/graphdriver/lcow/remotefs_filedriver.go

@@ -0,0 +1,123 @@
+// +build windows
+
+package lcow
+
+import (
+	"bytes"
+	"encoding/json"
+	"os"
+	"strconv"
+
+	"github.com/Microsoft/opengcs/service/gcsutils/remotefs"
+
+	"github.com/containerd/continuity/driver"
+	"github.com/sirupsen/logrus"
+)
+
+var _ driver.Driver = &lcowfs{}
+
+func (l *lcowfs) Readlink(p string) (string, error) {
+	logrus.Debugf("removefs.readlink args: %s", p)
+
+	result := &bytes.Buffer{}
+	if err := l.runRemoteFSProcess(nil, result, remotefs.ReadlinkCmd, p); err != nil {
+		return "", err
+	}
+	return result.String(), nil
+}
+
+func (l *lcowfs) Mkdir(path string, mode os.FileMode) error {
+	return l.mkdir(path, mode, remotefs.MkdirCmd)
+}
+
+func (l *lcowfs) MkdirAll(path string, mode os.FileMode) error {
+	return l.mkdir(path, mode, remotefs.MkdirAllCmd)
+}
+
+func (l *lcowfs) mkdir(path string, mode os.FileMode, cmd string) error {
+	modeStr := strconv.FormatUint(uint64(mode), 8)
+	logrus.Debugf("remotefs.%s args: %s %s", cmd, path, modeStr)
+	return l.runRemoteFSProcess(nil, nil, cmd, path, modeStr)
+}
+
+func (l *lcowfs) Remove(path string) error {
+	return l.remove(path, remotefs.RemoveCmd)
+}
+
+func (l *lcowfs) RemoveAll(path string) error {
+	return l.remove(path, remotefs.RemoveAllCmd)
+}
+
+func (l *lcowfs) remove(path string, cmd string) error {
+	logrus.Debugf("remotefs.%s args: %s", cmd, path)
+	return l.runRemoteFSProcess(nil, nil, cmd, path)
+}
+
+func (l *lcowfs) Link(oldname, newname string) error {
+	return l.link(oldname, newname, remotefs.LinkCmd)
+}
+
+func (l *lcowfs) Symlink(oldname, newname string) error {
+	return l.link(oldname, newname, remotefs.SymlinkCmd)
+}
+
+func (l *lcowfs) link(oldname, newname, cmd string) error {
+	logrus.Debugf("remotefs.%s args: %s %s", cmd, oldname, newname)
+	return l.runRemoteFSProcess(nil, nil, cmd, oldname, newname)
+}
+
+func (l *lcowfs) Lchown(name string, uid, gid int64) error {
+	uidStr := strconv.FormatInt(uid, 10)
+	gidStr := strconv.FormatInt(gid, 10)
+
+	logrus.Debugf("remotefs.lchown args: %s %s %s", name, uidStr, gidStr)
+	return l.runRemoteFSProcess(nil, nil, remotefs.LchownCmd, name, uidStr, gidStr)
+}
+
+// Lchmod changes the mode of an file not following symlinks.
+func (l *lcowfs) Lchmod(path string, mode os.FileMode) error {
+	modeStr := strconv.FormatUint(uint64(mode), 8)
+	logrus.Debugf("remotefs.lchmod args: %s %s", path, modeStr)
+	return l.runRemoteFSProcess(nil, nil, remotefs.LchmodCmd, path, modeStr)
+}
+
+func (l *lcowfs) Mknod(path string, mode os.FileMode, major, minor int) error {
+	modeStr := strconv.FormatUint(uint64(mode), 8)
+	majorStr := strconv.FormatUint(uint64(major), 10)
+	minorStr := strconv.FormatUint(uint64(minor), 10)
+
+	logrus.Debugf("remotefs.mknod args: %s %s %s %s", path, modeStr, majorStr, minorStr)
+	return l.runRemoteFSProcess(nil, nil, remotefs.MknodCmd, path, modeStr, majorStr, minorStr)
+}
+
+func (l *lcowfs) Mkfifo(path string, mode os.FileMode) error {
+	modeStr := strconv.FormatUint(uint64(mode), 8)
+	logrus.Debugf("remotefs.mkfifo args: %s %s", path, modeStr)
+	return l.runRemoteFSProcess(nil, nil, remotefs.MkfifoCmd, path, modeStr)
+}
+
+func (l *lcowfs) Stat(p string) (os.FileInfo, error) {
+	return l.stat(p, remotefs.StatCmd)
+}
+
+func (l *lcowfs) Lstat(p string) (os.FileInfo, error) {
+	return l.stat(p, remotefs.LstatCmd)
+}
+
+func (l *lcowfs) stat(path string, cmd string) (os.FileInfo, error) {
+	logrus.Debugf("remotefs.stat inputs: %s %s", cmd, path)
+
+	output := &bytes.Buffer{}
+	err := l.runRemoteFSProcess(nil, output, cmd, path)
+	if err != nil {
+		return nil, err
+	}
+
+	var fi remotefs.FileInfo
+	if err := json.Unmarshal(output.Bytes(), &fi); err != nil {
+		return nil, err
+	}
+
+	logrus.Debugf("remotefs.stat success. got: %v\n", fi)
+	return &fi, nil
+}

daemon/graphdriver/lcow/remotefs_pathdriver.go

@@ -0,0 +1,212 @@
+// +build windows
+
+package lcow
+
+import (
+	"errors"
+	"os"
+	pathpkg "path"
+	"path/filepath"
+	"sort"
+	"strings"
+
+	"github.com/containerd/continuity/pathdriver"
+)
+
+var _ pathdriver.PathDriver = &lcowfs{}
+
+// Continuity Path functions can be done locally
+func (l *lcowfs) Join(path ...string) string {
+	return pathpkg.Join(path...)
+}
+
+func (l *lcowfs) IsAbs(path string) bool {
+	return pathpkg.IsAbs(path)
+}
+
+func sameWord(a, b string) bool {
+	return a == b
+}
+
+// Implementation taken from the Go standard library
+func (l *lcowfs) Rel(basepath, targpath string) (string, error) {
+	baseVol := ""
+	targVol := ""
+	base := l.Clean(basepath)
+	targ := l.Clean(targpath)
+	if sameWord(targ, base) {
+		return ".", nil
+	}
+	base = base[len(baseVol):]
+	targ = targ[len(targVol):]
+	if base == "." {
+		base = ""
+	}
+	// Can't use IsAbs - `\a` and `a` are both relative in Windows.
+	baseSlashed := len(base) > 0 && base[0] == l.Separator()
+	targSlashed := len(targ) > 0 && targ[0] == l.Separator()
+	if baseSlashed != targSlashed || !sameWord(baseVol, targVol) {
+		return "", errors.New("Rel: can't make " + targpath + " relative to " + basepath)
+	}
+	// Position base[b0:bi] and targ[t0:ti] at the first differing elements.
+	bl := len(base)
+	tl := len(targ)
+	var b0, bi, t0, ti int
+	for {
+		for bi < bl && base[bi] != l.Separator() {
+			bi++
+		}
+		for ti < tl && targ[ti] != l.Separator() {
+			ti++
+		}
+		if !sameWord(targ[t0:ti], base[b0:bi]) {
+			break
+		}
+		if bi < bl {
+			bi++
+		}
+		if ti < tl {
+			ti++
+		}
+		b0 = bi
+		t0 = ti
+	}
+	if base[b0:bi] == ".." {
+		return "", errors.New("Rel: can't make " + targpath + " relative to " + basepath)
+	}
+	if b0 != bl {
+		// Base elements left. Must go up before going down.
+		seps := strings.Count(base[b0:bl], string(l.Separator()))
+		size := 2 + seps*3
+		if tl != t0 {
+			size += 1 + tl - t0
+		}
+		buf := make([]byte, size)
+		n := copy(buf, "..")
+		for i := 0; i < seps; i++ {
+			buf[n] = l.Separator()
+			copy(buf[n+1:], "..")
+			n += 3
+		}
+		if t0 != tl {
+			buf[n] = l.Separator()
+			copy(buf[n+1:], targ[t0:])
+		}
+		return string(buf), nil
+	}
+	return targ[t0:], nil
+}
+
+func (l *lcowfs) Base(path string) string {
+	return pathpkg.Base(path)
+}
+
+func (l *lcowfs) Dir(path string) string {
+	return pathpkg.Dir(path)
+}
+
+func (l *lcowfs) Clean(path string) string {
+	return pathpkg.Clean(path)
+}
+
+func (l *lcowfs) Split(path string) (dir, file string) {
+	return pathpkg.Split(path)
+}
+
+func (l *lcowfs) Separator() byte {
+	return '/'
+}
+
+func (l *lcowfs) Abs(path string) (string, error) {
+	// Abs is supposed to add the current working directory, which is meaningless in lcow.
+	// So, return an error.
+	return "", ErrNotSupported
+}
+
+// Implementation taken from the Go standard library
+func (l *lcowfs) Walk(root string, walkFn filepath.WalkFunc) error {
+	info, err := l.Lstat(root)
+	if err != nil {
+		err = walkFn(root, nil, err)
+	} else {
+		err = l.walk(root, info, walkFn)
+	}
+	if err == filepath.SkipDir {
+		return nil
+	}
+	return err
+}
+
+// walk recursively descends path, calling w.
+func (l *lcowfs) walk(path string, info os.FileInfo, walkFn filepath.WalkFunc) error {
+	err := walkFn(path, info, nil)
+	if err != nil {
+		if info.IsDir() && err == filepath.SkipDir {
+			return nil
+		}
+		return err
+	}
+
+	if !info.IsDir() {
+		return nil
+	}
+
+	names, err := l.readDirNames(path)
+	if err != nil {
+		return walkFn(path, info, err)
+	}
+
+	for _, name := range names {
+		filename := l.Join(path, name)
+		fileInfo, err := l.Lstat(filename)
+		if err != nil {
+			if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir {
+				return err
+			}
+		} else {
+			err = l.walk(filename, fileInfo, walkFn)
+			if err != nil {
+				if !fileInfo.IsDir() || err != filepath.SkipDir {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// readDirNames reads the directory named by dirname and returns
+// a sorted list of directory entries.
+func (l *lcowfs) readDirNames(dirname string) ([]string, error) {
+	f, err := l.Open(dirname)
+	if err != nil {
+		return nil, err
+	}
+	files, err := f.Readdir(-1)
+	f.Close()
+	if err != nil {
+		return nil, err
+	}
+
+	names := make([]string, len(files), len(files))
+	for i := range files {
+		names[i] = files[i].Name()
+	}
+
+	sort.Strings(names)
+	return names, nil
+}
+
+// Note that Go's filepath.FromSlash/ToSlash convert between OS paths and '/'. Since the path separator
+// for LCOW (and Unix) is '/', they are no-ops.
+func (l *lcowfs) FromSlash(path string) string {
+	return path
+}
+
+func (l *lcowfs) ToSlash(path string) string {
+	return path
+}
+
+func (l *lcowfs) Match(pattern, name string) (matched bool, err error) {
+	return pathpkg.Match(pattern, name)
+}

daemon/graphdriver/overlay/overlay.go

@@ -15,6 +15,7 @@ import (
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/daemon/graphdriver/overlayutils"
 	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/fsutils"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/locker"
@@ -341,21 +342,21 @@ func (d *Driver) Remove(id string) error {
 }
 
 // Get creates and mounts the required file system for the given id and returns the mount path.
-func (d *Driver) Get(id string, mountLabel string) (s string, err error) {
+func (d *Driver) Get(id, mountLabel string) (_ containerfs.ContainerFS, err error) {
 	d.locker.Lock(id)
 	defer d.locker.Unlock(id)
 	dir := d.dir(id)
 	if _, err := os.Stat(dir); err != nil {
-		return "", err
+		return nil, err
 	}
 	// If id has a root, just return it
 	rootDir := path.Join(dir, "root")
 	if _, err := os.Stat(rootDir); err == nil {
-		return rootDir, nil
+		return containerfs.NewLocalContainerFS(rootDir), nil
 	}
 	mergedDir := path.Join(dir, "merged")
 	if count := d.ctr.Increment(mergedDir); count > 1 {
-		return mergedDir, nil
+		return containerfs.NewLocalContainerFS(mergedDir), nil
 	}
 	defer func() {
 		if err != nil {
@@ -366,7 +367,7 @@ func (d *Driver) Get(id string, mountLabel string) (s string, err error) {
 	}()
 	lowerID, err := ioutil.ReadFile(path.Join(dir, "lower-id"))
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 	var (
 		lowerDir = path.Join(d.dir(string(lowerID)), "root")
@@ -375,18 +376,18 @@ func (d *Driver) Get(id string, mountLabel string) (s string, err error) {
 		opts     = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", lowerDir, upperDir, workDir)
 	)
 	if err := unix.Mount("overlay", mergedDir, "overlay", 0, label.FormatMountLabel(opts, mountLabel)); err != nil {
-		return "", fmt.Errorf("error creating overlay mount to %s: %v", mergedDir, err)
+		return nil, fmt.Errorf("error creating overlay mount to %s: %v", mergedDir, err)
 	}
 	// chown "workdir/work" to the remapped root UID/GID. Overlay fs inside a
 	// user namespace requires this to move a directory from lower to upper.
 	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 	if err := os.Chown(path.Join(workDir, "work"), rootUID, rootGID); err != nil {
-		return "", err
+		return nil, err
 	}
-	return mergedDir, nil
+	return containerfs.NewLocalContainerFS(mergedDir), nil
 }
 
 // Put unmounts the mount path created for the give id.

daemon/graphdriver/overlay2/overlay.go

@@ -23,6 +23,7 @@ import (
 	"github.com/docker/docker/daemon/graphdriver/quota"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/chrootarchive"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/directory"
 	"github.com/docker/docker/pkg/fsutils"
 	"github.com/docker/docker/pkg/idtools"
@@ -514,12 +515,12 @@ func (d *Driver) Remove(id string) error {
 }
 
 // Get creates and mounts the required file system for the given id and returns the mount path.
-func (d *Driver) Get(id string, mountLabel string) (s string, err error) {
+func (d *Driver) Get(id, mountLabel string) (containerfs.ContainerFS, error) {
 	d.locker.Lock(id)
 	defer d.locker.Unlock(id)
 	dir := d.dir(id)
 	if _, err := os.Stat(dir); err != nil {
-		return "", err
+		return nil, err
 	}
 
 	diffDir := path.Join(dir, "diff")
@@ -527,14 +528,14 @@ func (d *Driver) Get(id string, mountLabel string) (s string, err error) {
 	if err != nil {
 		// If no lower, just return diff directory
 		if os.IsNotExist(err) {
-			return diffDir, nil
+			return containerfs.NewLocalContainerFS(diffDir), nil
 		}
-		return "", err
+		return nil, err
 	}
 
 	mergedDir := path.Join(dir, "merged")
 	if count := d.ctr.Increment(mergedDir); count > 1 {
-		return mergedDir, nil
+		return containerfs.NewLocalContainerFS(mergedDir), nil
 	}
 	defer func() {
 		if err != nil {
@@ -574,7 +575,7 @@ func (d *Driver) Get(id string, mountLabel string) (s string, err error) {
 		opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", string(lowers), path.Join(id, "diff"), path.Join(id, "work"))
 		mountData = label.FormatMountLabel(opts, mountLabel)
 		if len(mountData) > pageSize {
-			return "", fmt.Errorf("cannot mount layer, mount label too large %d", len(mountData))
+			return nil, fmt.Errorf("cannot mount layer, mount label too large %d", len(mountData))
 		}
 
 		mount = func(source string, target string, mType string, flags uintptr, label string) error {
@@ -584,21 +585,21 @@ func (d *Driver) Get(id string, mountLabel string) (s string, err error) {
 	}
 
 	if err := mount("overlay", mountTarget, "overlay", 0, mountData); err != nil {
-		return "", fmt.Errorf("error creating overlay mount to %s: %v", mergedDir, err)
+		return nil, fmt.Errorf("error creating overlay mount to %s: %v", mergedDir, err)
 	}
 
 	// chown "workdir/work" to the remapped root UID/GID. Overlay fs inside a
 	// user namespace requires this to move a directory from lower to upper.
 	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 
 	if err := os.Chown(path.Join(workDir, "work"), rootUID, rootGID); err != nil {
-		return "", err
+		return nil, err
 	}
 
-	return mergedDir, nil
+	return containerfs.NewLocalContainerFS(mergedDir), nil
 }
 
 // Put unmounts the mount path created for the give id.

daemon/graphdriver/proxy.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 
 	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/plugingetter"
 	"github.com/docker/docker/pkg/plugins"
@@ -129,20 +130,20 @@ func (d *graphDriverProxy) Remove(id string) error {
 	return nil
 }
 
-func (d *graphDriverProxy) Get(id, mountLabel string) (string, error) {
+func (d *graphDriverProxy) Get(id, mountLabel string) (containerfs.ContainerFS, error) {
 	args := &graphDriverRequest{
 		ID:         id,
 		MountLabel: mountLabel,
 	}
 	var ret graphDriverResponse
 	if err := d.p.Client().Call("GraphDriver.Get", args, &ret); err != nil {
-		return "", err
+		return nil, err
 	}
 	var err error
 	if ret.Err != "" {
 		err = errors.New(ret.Err)
 	}
-	return filepath.Join(d.p.BasePath(), ret.Dir), err
+	return containerfs.NewLocalContainerFS(filepath.Join(d.p.BasePath(), ret.Dir)), err
 }
 
 func (d *graphDriverProxy) Put(id string) error {

daemon/graphdriver/vfs/driver.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/pkg/chrootarchive"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/system"
 	"github.com/opencontainers/selinux/go-selinux/label"
@@ -94,7 +95,7 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error {
 	if err != nil {
 		return fmt.Errorf("%s: %s", parent, err)
 	}
-	return CopyWithTar(parentDir, dir)
+	return CopyWithTar(parentDir.Path(), dir)
 }
 
 func (d *Driver) dir(id string) string {
@@ -107,14 +108,14 @@ func (d *Driver) Remove(id string) error {
 }
 
 // Get returns the directory for the given id.
-func (d *Driver) Get(id, mountLabel string) (string, error) {
+func (d *Driver) Get(id, mountLabel string) (containerfs.ContainerFS, error) {
 	dir := d.dir(id)
 	if st, err := os.Stat(dir); err != nil {
-		return "", err
+		return nil, err
 	} else if !st.IsDir() {
-		return "", fmt.Errorf("%s: not a directory", dir)
+		return nil, fmt.Errorf("%s: not a directory", dir)
 	}
-	return dir, nil
+	return containerfs.NewLocalContainerFS(dir), nil
 }
 
 // Put is a noop for vfs that return nil for the error, since this driver has no runtime resources to clean up.

daemon/graphdriver/windows/windows.go

@@ -26,6 +26,7 @@ import (
 	"github.com/Microsoft/hcsshim"
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/ioutils"
 	"github.com/docker/docker/pkg/longpath"
@@ -354,36 +355,36 @@ func (d *Driver) Remove(id string) error {
 }
 
 // Get returns the rootfs path for the id. This will mount the dir at its given path.
-func (d *Driver) Get(id, mountLabel string) (string, error) {
+func (d *Driver) Get(id, mountLabel string) (containerfs.ContainerFS, error) {
 	panicIfUsedByLcow()
 	logrus.Debugf("WindowsGraphDriver Get() id %s mountLabel %s", id, mountLabel)
 	var dir string
 
 	rID, err := d.resolveID(id)
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 	if count := d.ctr.Increment(rID); count > 1 {
-		return d.cache[rID], nil
+		return containerfs.NewLocalContainerFS(d.cache[rID]), nil
 	}
 
 	// Getting the layer paths must be done outside of the lock.
 	layerChain, err := d.getLayerChain(rID)
 	if err != nil {
 		d.ctr.Decrement(rID)
-		return "", err
+		return nil, err
 	}
 
 	if err := hcsshim.ActivateLayer(d.info, rID); err != nil {
 		d.ctr.Decrement(rID)
-		return "", err
+		return nil, err
 	}
 	if err := hcsshim.PrepareLayer(d.info, rID, layerChain); err != nil {
 		d.ctr.Decrement(rID)
 		if err2 := hcsshim.DeactivateLayer(d.info, rID); err2 != nil {
 			logrus.Warnf("Failed to Deactivate %s: %s", id, err)
 		}
-		return "", err
+		return nil, err
 	}
 
 	mountPath, err := hcsshim.GetLayerMountPath(d.info, rID)
@@ -395,7 +396,7 @@ func (d *Driver) Get(id, mountLabel string) (string, error) {
 		if err2 := hcsshim.DeactivateLayer(d.info, rID); err2 != nil {
 			logrus.Warnf("Failed to Deactivate %s: %s", id, err)
 		}
-		return "", err
+		return nil, err
 	}
 	d.cacheMu.Lock()
 	d.cache[rID] = mountPath
@@ -409,7 +410,7 @@ func (d *Driver) Get(id, mountLabel string) (string, error) {
 		dir = d.dir(id)
 	}
 
-	return dir, nil
+	return containerfs.NewLocalContainerFS(dir), nil
 }
 
 // Put adds a new layer to the driver.
@@ -618,7 +619,7 @@ func (d *Driver) DiffSize(id, parent string) (size int64, err error) {
 	}
 	defer d.Put(id)
 
-	return archive.ChangesSize(layerFs, changes), nil
+	return archive.ChangesSize(layerFs.Path(), changes), nil
 }
 
 // GetMetadata returns custom driver information.

daemon/graphdriver/zfs/zfs.go

@@ -13,6 +13,7 @@ import (
 	"time"
 
 	"github.com/docker/docker/daemon/graphdriver"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/mount"
 	"github.com/docker/docker/pkg/parsers"
@@ -356,10 +357,10 @@ func (d *Driver) Remove(id string) error {
 }
 
 // Get returns the mountpoint for the given id after creating the target directories if necessary.
-func (d *Driver) Get(id, mountLabel string) (string, error) {
+func (d *Driver) Get(id, mountLabel string) (containerfs.ContainerFS, error) {
 	mountpoint := d.mountPath(id)
 	if count := d.ctr.Increment(mountpoint); count > 1 {
-		return mountpoint, nil
+		return containerfs.NewLocalContainerFS(mountpoint), nil
 	}
 
 	filesystem := d.zfsPath(id)
@@ -369,17 +370,17 @@ func (d *Driver) Get(id, mountLabel string) (string, error) {
 	rootUID, rootGID, err := idtools.GetRootUIDGID(d.uidMaps, d.gidMaps)
 	if err != nil {
 		d.ctr.Decrement(mountpoint)
-		return "", err
+		return nil, err
 	}
 	// Create the target directories if they don't exist
 	if err := idtools.MkdirAllAs(mountpoint, 0755, rootUID, rootGID); err != nil {
 		d.ctr.Decrement(mountpoint)
-		return "", err
+		return nil, err
 	}
 
 	if err := mount.Mount(filesystem, mountpoint, "zfs", options); err != nil {
 		d.ctr.Decrement(mountpoint)
-		return "", fmt.Errorf("error creating zfs mount of %s to %s: %v", filesystem, mountpoint, err)
+		return nil, fmt.Errorf("error creating zfs mount of %s to %s: %v", filesystem, mountpoint, err)
 	}
 
 	// this could be our first mount after creation of the filesystem, and the root dir may still have root
@@ -387,10 +388,10 @@ func (d *Driver) Get(id, mountLabel string) (string, error) {
 	if err := os.Chown(mountpoint, rootUID, rootGID); err != nil {
 		mount.Unmount(mountpoint)
 		d.ctr.Decrement(mountpoint)
-		return "", fmt.Errorf("error modifying zfs mountpoint (%s) directory ownership: %v", mountpoint, err)
+		return nil, fmt.Errorf("error modifying zfs mountpoint (%s) directory ownership: %v", mountpoint, err)
 	}
 
-	return mountpoint, nil
+	return containerfs.NewLocalContainerFS(mountpoint), nil
 }
 
 // Put removes the existing mountpoint for the given id if it exists.

daemon/initlayer/setup_solaris.go

@@ -2,12 +2,14 @@
 
 package initlayer
 
+import "github.com/docker/docker/pkg/containerfs"
+
 // Setup populates a directory with mountpoints suitable
 // for bind-mounting dockerinit into the container. The mountpoint is simply an
 // empty file at /.dockerinit
 //
 // This extra layer is used by all containers as the top-most ro layer. It protects
 // the container from unwanted side-effects on the rw layer.
-func Setup(initLayer string, rootUID, rootGID int) error {
+func Setup(initLayer containerfs.ContainerFS, rootUID, rootGID int) error {
 	return nil
 }

daemon/initlayer/setup_unix.go

@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 	"golang.org/x/sys/unix"
 )
@@ -16,7 +17,10 @@ import (
 //
 // This extra layer is used by all containers as the top-most ro layer. It protects
 // the container from unwanted side-effects on the rw layer.
-func Setup(initLayer string, rootIDs idtools.IDPair) error {
+func Setup(initLayerFs containerfs.ContainerFS, rootIDs idtools.IDPair) error {
+	// Since all paths are local to the container, we can just extract initLayerFs.Path()
+	initLayer := initLayerFs.Path()
+
 	for pth, typ := range map[string]string{
 		"/dev/pts":         "dir",
 		"/dev/shm":         "dir",

daemon/initlayer/setup_windows.go

@@ -3,6 +3,7 @@
 package initlayer
 
 import (
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 )
 
@@ -12,6 +13,6 @@ import (
 //
 // This extra layer is used by all containers as the top-most ro layer. It protects
 // the container from unwanted side-effects on the rw layer.
-func Setup(initLayer string, rootIDs idtools.IDPair) error {
+func Setup(initLayer containerfs.ContainerFS, rootIDs idtools.IDPair) error {
 	return nil
 }

daemon/oci_linux.go

@@ -19,7 +19,6 @@ import (
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/mount"
 	"github.com/docker/docker/pkg/stringutils"
-	"github.com/docker/docker/pkg/symlink"
 	"github.com/docker/docker/volume"
 	"github.com/opencontainers/runc/libcontainer/apparmor"
 	"github.com/opencontainers/runc/libcontainer/cgroups"
@@ -187,7 +186,7 @@ func setUser(s *specs.Spec, c *container.Container) error {
 }
 
 func readUserFile(c *container.Container, p string) (io.ReadCloser, error) {
-	fp, err := symlink.FollowSymlinkInScope(filepath.Join(c.BaseFS, p), c.BaseFS)
+	fp, err := c.GetResourcePath(p)
 	if err != nil {
 		return nil, err
 	}
@@ -632,7 +631,7 @@ func (daemon *Daemon) populateCommonSpec(s *specs.Spec, c *container.Container)
 		return err
 	}
 	s.Root = &specs.Root{
-		Path:     c.BaseFS,
+		Path:     c.BaseFS.Path(),
 		Readonly: c.HostConfig.ReadonlyRootfs,
 	}
 	if err := c.SetupWorkingDirectory(daemon.idMappings.RootPair()); err != nil {

daemon/oci_solaris.go

@@ -2,7 +2,6 @@ package daemon
 
 import (
 	"fmt"
-	"path/filepath"
 	"sort"
 	"strconv"
 
@@ -127,7 +126,7 @@ func (daemon *Daemon) populateCommonSpec(s *specs.Spec, c *container.Container)
 		return err
 	}
 	s.Root = specs.Root{
-		Path:     filepath.Dir(c.BaseFS),
+		Path:     c.BaseFS.Dir(c.BaseFS.Path()),
 		Readonly: c.HostConfig.ReadonlyRootfs,
 	}
 	if err := c.SetupWorkingDirectory(daemon.idMappings.RootPair()); err != nil {

daemon/oci_windows.go

@@ -233,7 +233,7 @@ func (daemon *Daemon) createSpecWindowsFields(c *container.Container, s *specs.S
 
 	s.Root.Readonly = false // Windows does not support a read-only root filesystem
 	if !isHyperV {
-		s.Root.Path = c.BaseFS // This is not set for Hyper-V containers
+		s.Root.Path = c.BaseFS.Path() // This is not set for Hyper-V containers
 		if !strings.HasSuffix(s.Root.Path, `\`) {
 			s.Root.Path = s.Root.Path + `\` // Ensure a correctly formatted volume GUID path \\?\Volume{GUID}\
 		}

daemon/start.go

@@ -204,7 +204,7 @@ func (daemon *Daemon) Cleanup(container *container.Container) {
 		daemon.unregisterExecCommand(container, eConfig)
 	}
 
-	if container.BaseFS != "" {
+	if container.BaseFS != nil && container.BaseFS.Path() != "" {
 		if err := container.UnmountVolumes(daemon.LogVolumeEvent); err != nil {
 			logrus.Warnf("%s cleanup: Failed to umount volumes: %v", container.ID, err)
 		}

integration-cli/docker_cli_external_graphdriver_unix_test.go

@@ -198,12 +198,13 @@ func (s *DockerExternalGraphdriverSuite) setUpPlugin(c *check.C, name string, ex
 			return
 		}
 
+		// TODO @gupta-ak: Figure out what to do here.
 		dir, err := driver.Get(req.ID, req.MountLabel)
 		if err != nil {
 			respond(w, err)
 			return
 		}
-		respond(w, &graphDriverResponse{Dir: dir})
+		respond(w, &graphDriverResponse{Dir: dir.Path()})
 	})
 
 	mux.HandleFunc("/GraphDriver.Put", func(w http.ResponseWriter, r *http.Request) {

layer/layer.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/docker/distribution"
 	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/opencontainers/go-digest"
 	"github.com/sirupsen/logrus"
 )
@@ -137,7 +138,7 @@ type RWLayer interface {
 
 	// Mount mounts the RWLayer and returns the filesystem path
 	// the to the writable layer.
-	Mount(mountLabel string) (string, error)
+	Mount(mountLabel string) (containerfs.ContainerFS, error)
 
 	// Unmount unmounts the RWLayer. This should be called
 	// for every mount. If there are multiple mount calls
@@ -178,7 +179,7 @@ type Metadata struct {
 // writable mount. Changes made here will
 // not be included in the Tar stream of the
 // RWLayer.
-type MountInit func(root string) error
+type MountInit func(root containerfs.ContainerFS) error
 
 // CreateRWLayerOpts contains optional arguments to be passed to CreateRWLayer
 type CreateRWLayerOpts struct {

layer/layer_store.go

@@ -749,5 +749,5 @@ func (n *naiveDiffPathDriver) DiffGetter(id string) (graphdriver.FileGetCloser,
 	if err != nil {
 		return nil, err
 	}
-	return &fileGetPutter{storage.NewPathFileGetter(p), n.Driver, id}, nil
+	return &fileGetPutter{storage.NewPathFileGetter(p.Path()), n.Driver, id}, nil
 }

layer/layer_test.go

@@ -10,9 +10,11 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/containerd/continuity/driver"
 	"github.com/docker/docker/daemon/graphdriver"
 	"github.com/docker/docker/daemon/graphdriver/vfs"
 	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/opencontainers/go-digest"
@@ -82,7 +84,7 @@ func newTestStore(t *testing.T) (Store, string, func()) {
 	}
 }
 
-type layerInit func(root string) error
+type layerInit func(root containerfs.ContainerFS) error
 
 func createLayer(ls Store, parent ChainID, layerFunc layerInit) (Layer, error) {
 	containerID := stringid.GenerateRandomID()
@@ -91,12 +93,12 @@ func createLayer(ls Store, parent ChainID, layerFunc layerInit) (Layer, error) {
 		return nil, err
 	}
 
-	path, err := mount.Mount("")
+	pathFS, err := mount.Mount("")
 	if err != nil {
 		return nil, err
 	}
 
-	if err := layerFunc(path); err != nil {
+	if err := layerFunc(pathFS); err != nil {
 		return nil, err
 	}
 
@@ -123,7 +125,7 @@ func createLayer(ls Store, parent ChainID, layerFunc layerInit) (Layer, error) {
 }
 
 type FileApplier interface {
-	ApplyFile(root string) error
+	ApplyFile(root containerfs.ContainerFS) error
 }
 
 type testFile struct {
@@ -140,25 +142,25 @@ func newTestFile(name string, content []byte, perm os.FileMode) FileApplier {
 	}
 }
 
-func (tf *testFile) ApplyFile(root string) error {
-	fullPath := filepath.Join(root, tf.name)
-	if err := os.MkdirAll(filepath.Dir(fullPath), 0755); err != nil {
+func (tf *testFile) ApplyFile(root containerfs.ContainerFS) error {
+	fullPath := root.Join(root.Path(), tf.name)
+	if err := root.MkdirAll(root.Dir(fullPath), 0755); err != nil {
 		return err
 	}
 	// Check if already exists
-	if stat, err := os.Stat(fullPath); err == nil && stat.Mode().Perm() != tf.permission {
-		if err := os.Chmod(fullPath, tf.permission); err != nil {
+	if stat, err := root.Stat(fullPath); err == nil && stat.Mode().Perm() != tf.permission {
+		if err := root.Lchmod(fullPath, tf.permission); err != nil {
 			return err
 		}
 	}
-	if err := ioutil.WriteFile(fullPath, tf.content, tf.permission); err != nil {
+	if err := driver.WriteFile(root, fullPath, tf.content, tf.permission); err != nil {
 		return err
 	}
 	return nil
 }
 
 func initWithFiles(files ...FileApplier) layerInit {
-	return func(root string) error {
+	return func(root containerfs.ContainerFS) error {
 		for _, f := range files {
 			if err := f.ApplyFile(root); err != nil {
 				return err
@@ -288,7 +290,7 @@ func TestMountAndRegister(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	b, err := ioutil.ReadFile(filepath.Join(path2, "testfile.txt"))
+	b, err := driver.ReadFile(path2, path2.Join(path2.Path(), "testfile.txt"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -391,12 +393,12 @@ func TestStoreRestore(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	path, err := m.Mount("")
+	pathFS, err := m.Mount("")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if err := ioutil.WriteFile(filepath.Join(path, "testfile.txt"), []byte("nothing here"), 0644); err != nil {
+	if err := driver.WriteFile(pathFS, pathFS.Join(pathFS.Path(), "testfile.txt"), []byte("nothing here"), 0644); err != nil {
 		t.Fatal(err)
 	}
 
@@ -430,20 +432,20 @@ func TestStoreRestore(t *testing.T) {
 
 	if mountPath, err := m2.Mount(""); err != nil {
 		t.Fatal(err)
-	} else if path != mountPath {
-		t.Fatalf("Unexpected path %s, expected %s", mountPath, path)
+	} else if pathFS.Path() != mountPath.Path() {
+		t.Fatalf("Unexpected path %s, expected %s", mountPath.Path(), pathFS.Path())
 	}
 
 	if mountPath, err := m2.Mount(""); err != nil {
 		t.Fatal(err)
-	} else if path != mountPath {
-		t.Fatalf("Unexpected path %s, expected %s", mountPath, path)
+	} else if pathFS.Path() != mountPath.Path() {
+		t.Fatalf("Unexpected path %s, expected %s", mountPath.Path(), pathFS.Path())
 	}
 	if err := m2.Unmount(); err != nil {
 		t.Fatal(err)
 	}
 
-	b, err := ioutil.ReadFile(filepath.Join(path, "testfile.txt"))
+	b, err := driver.ReadFile(pathFS, pathFS.Join(pathFS.Path(), "testfile.txt"))
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -618,7 +620,7 @@ func tarFromFiles(files ...FileApplier) ([]byte, error) {
 	defer os.RemoveAll(td)
 
 	for _, f := range files {
-		if err := f.ApplyFile(td); err != nil {
+		if err := f.ApplyFile(containerfs.NewLocalContainerFS(td)); err != nil {
 			return nil, err
 		}
 	}

layer/layer_windows.go

@@ -1,6 +1,15 @@
 package layer
 
-import "errors"
+import (
+	"errors"
+)
+
+// Getter is an interface to get the path to a layer on the host.
+type Getter interface {
+	// GetLayerPath gets the path for the layer. This is different from Get()
+	// since that returns an interface to account for umountable layers.
+	GetLayerPath(id string) (string, error)
+}
 
 // GetLayerPath returns the path to a layer
 func GetLayerPath(s Store, layer ChainID) (string, error) {
@@ -16,6 +25,10 @@ func GetLayerPath(s Store, layer ChainID) (string, error) {
 		return "", ErrLayerDoesNotExist
 	}
 
+	if layerGetter, ok := ls.driver.(Getter); ok {
+		return layerGetter.GetLayerPath(rl.cacheID)
+	}
+
 	path, err := ls.driver.Get(rl.cacheID, "")
 	if err != nil {
 		return "", err
@@ -25,7 +38,7 @@ func GetLayerPath(s Store, layer ChainID) (string, error) {
 		return "", err
 	}
 
-	return path, nil
+	return path.Path(), nil
 }
 
 func (ls *layerStore) mountID(name string) string {

layer/mount_test.go

@@ -2,13 +2,13 @@ package layer
 
 import (
 	"io/ioutil"
-	"os"
-	"path/filepath"
 	"runtime"
 	"sort"
 	"testing"
 
+	"github.com/containerd/continuity/driver"
 	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/containerfs"
 )
 
 func TestMountInit(t *testing.T) {
@@ -28,7 +28,7 @@ func TestMountInit(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	mountInit := func(root string) error {
+	mountInit := func(root containerfs.ContainerFS) error {
 		return initfile.ApplyFile(root)
 	}
 
@@ -40,21 +40,21 @@ func TestMountInit(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	path, err := m.Mount("")
+	pathFS, err := m.Mount("")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	f, err := os.Open(filepath.Join(path, "testfile.txt"))
+	fi, err := pathFS.Stat(pathFS.Join(pathFS.Path(), "testfile.txt"))
 	if err != nil {
 		t.Fatal(err)
 	}
-	defer f.Close()
 
-	fi, err := f.Stat()
+	f, err := pathFS.Open(pathFS.Join(pathFS.Path(), "testfile.txt"))
 	if err != nil {
 		t.Fatal(err)
 	}
+	defer f.Close()
 
 	b, err := ioutil.ReadAll(f)
 	if err != nil {
@@ -88,7 +88,7 @@ func TestMountSize(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	mountInit := func(root string) error {
+	mountInit := func(root containerfs.ContainerFS) error {
 		return newTestFile("file-init", contentInit, 0777).ApplyFile(root)
 	}
 	rwLayerOpts := &CreateRWLayerOpts{
@@ -100,12 +100,12 @@ func TestMountSize(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	path, err := m.Mount("")
+	pathFS, err := m.Mount("")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if err := ioutil.WriteFile(filepath.Join(path, "file2"), content2, 0755); err != nil {
+	if err := driver.WriteFile(pathFS, pathFS.Join(pathFS.Path(), "file2"), content2, 0755); err != nil {
 		t.Fatal(err)
 	}
 
@@ -140,7 +140,7 @@ func TestMountChanges(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	mountInit := func(root string) error {
+	mountInit := func(root containerfs.ContainerFS) error {
 		return initfile.ApplyFile(root)
 	}
 	rwLayerOpts := &CreateRWLayerOpts{
@@ -152,28 +152,28 @@ func TestMountChanges(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	path, err := m.Mount("")
+	pathFS, err := m.Mount("")
 	if err != nil {
 		t.Fatal(err)
 	}
 
-	if err := os.Chmod(filepath.Join(path, "testfile1.txt"), 0755); err != nil {
+	if err := pathFS.Lchmod(pathFS.Join(pathFS.Path(), "testfile1.txt"), 0755); err != nil {
 		t.Fatal(err)
 	}
 
-	if err := ioutil.WriteFile(filepath.Join(path, "testfile1.txt"), []byte("mount data!"), 0755); err != nil {
+	if err := driver.WriteFile(pathFS, pathFS.Join(pathFS.Path(), "testfile1.txt"), []byte("mount data!"), 0755); err != nil {
 		t.Fatal(err)
 	}
 
-	if err := os.Remove(filepath.Join(path, "testfile2.txt")); err != nil {
+	if err := pathFS.Remove(pathFS.Join(pathFS.Path(), "testfile2.txt")); err != nil {
 		t.Fatal(err)
 	}
 
-	if err := os.Chmod(filepath.Join(path, "testfile3.txt"), 0755); err != nil {
+	if err := pathFS.Lchmod(pathFS.Join(pathFS.Path(), "testfile3.txt"), 0755); err != nil {
 		t.Fatal(err)
 	}
 
-	if err := ioutil.WriteFile(filepath.Join(path, "testfile4.txt"), []byte("mount data!"), 0644); err != nil {
+	if err := driver.WriteFile(pathFS, pathFS.Join(pathFS.Path(), "testfile4.txt"), []byte("mount data!"), 0644); err != nil {
 		t.Fatal(err)
 	}
 

layer/mounted_layer.go

@@ -4,6 +4,7 @@ import (
 	"io"
 
 	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/containerfs"
 )
 
 type mountedLayer struct {
@@ -88,7 +89,7 @@ type referencedRWLayer struct {
 	*mountedLayer
 }
 
-func (rl *referencedRWLayer) Mount(mountLabel string) (string, error) {
+func (rl *referencedRWLayer) Mount(mountLabel string) (containerfs.ContainerFS, error) {
 	return rl.layerStore.driver.Get(rl.mountedLayer.mountID, mountLabel)
 }
 

pkg/archive/archive.go

@@ -55,18 +55,17 @@ type (
 	}
 )
 
-// Archiver allows the reuse of most utility functions of this package
-// with a pluggable Untar function. Also, to facilitate the passing of
-// specific id mappings for untar, an archiver can be created with maps
-// which will then be passed to Untar operations
+// Archiver implements the Archiver interface and allows the reuse of most utility functions of
+// this package with a pluggable Untar function. Also, to facilitate the passing of specific id
+// mappings for untar, an Archiver can be created with maps which will then be passed to Untar operations.
 type Archiver struct {
-	Untar      func(io.Reader, string, *TarOptions) error
-	IDMappings *idtools.IDMappings
+	Untar         func(io.Reader, string, *TarOptions) error
+	IDMappingsVar *idtools.IDMappings
 }
 
 // NewDefaultArchiver returns a new Archiver without any IDMappings
 func NewDefaultArchiver() *Archiver {
-	return &Archiver{Untar: Untar, IDMappings: &idtools.IDMappings{}}
+	return &Archiver{Untar: Untar, IDMappingsVar: &idtools.IDMappings{}}
 }
 
 // breakoutError is used to differentiate errors related to breaking out
@@ -1025,8 +1024,8 @@ func (archiver *Archiver) TarUntar(src, dst string) error {
 	}
 	defer archive.Close()
 	options := &TarOptions{
-		UIDMaps: archiver.IDMappings.UIDs(),
-		GIDMaps: archiver.IDMappings.GIDs(),
+		UIDMaps: archiver.IDMappingsVar.UIDs(),
+		GIDMaps: archiver.IDMappingsVar.GIDs(),
 	}
 	return archiver.Untar(archive, dst, options)
 }
@@ -1039,8 +1038,8 @@ func (archiver *Archiver) UntarPath(src, dst string) error {
 	}
 	defer archive.Close()
 	options := &TarOptions{
-		UIDMaps: archiver.IDMappings.UIDs(),
-		GIDMaps: archiver.IDMappings.GIDs(),
+		UIDMaps: archiver.IDMappingsVar.UIDs(),
+		GIDMaps: archiver.IDMappingsVar.GIDs(),
 	}
 	return archiver.Untar(archive, dst, options)
 }
@@ -1058,10 +1057,10 @@ func (archiver *Archiver) CopyWithTar(src, dst string) error {
 		return archiver.CopyFileWithTar(src, dst)
 	}
 
-	// if this archiver is set up with ID mapping we need to create
+	// if this Archiver is set up with ID mapping we need to create
 	// the new destination directory with the remapped root UID/GID pair
 	// as owner
-	rootIDs := archiver.IDMappings.RootPair()
+	rootIDs := archiver.IDMappingsVar.RootPair()
 	// Create dst, copy src's content into it
 	logrus.Debugf("Creating dest directory: %s", dst)
 	if err := idtools.MkdirAllAndChownNew(dst, 0755, rootIDs); err != nil {
@@ -1112,7 +1111,7 @@ func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {
 		hdr.Name = filepath.Base(dst)
 		hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
 
-		if err := remapIDs(archiver.IDMappings, hdr); err != nil {
+		if err := remapIDs(archiver.IDMappingsVar, hdr); err != nil {
 			return err
 		}
 
@@ -1139,6 +1138,11 @@ func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {
 	return err
 }
 
+// IDMappings returns the IDMappings of the archiver.
+func (archiver *Archiver) IDMappings() *idtools.IDMappings {
+	return archiver.IDMappingsVar
+}
+
 func remapIDs(idMappings *idtools.IDMappings, hdr *tar.Header) error {
 	ids, err := idMappings.ToHost(idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid})
 	hdr.Uid, hdr.Gid = ids.UID, ids.GID

pkg/archive/copy.go

@@ -27,23 +27,23 @@ var (
 // path (from before being processed by utility functions from the path or
 // filepath stdlib packages) ends with a trailing `/.` or `/`. If the cleaned
 // path already ends in a `.` path segment, then another is not added. If the
-// clean path already ends in a path separator, then another is not added.
-func PreserveTrailingDotOrSeparator(cleanedPath, originalPath string) string {
+// clean path already ends in the separator, then another is not added.
+func PreserveTrailingDotOrSeparator(cleanedPath string, originalPath string, sep byte) string {
 	// Ensure paths are in platform semantics
-	cleanedPath = normalizePath(cleanedPath)
-	originalPath = normalizePath(originalPath)
+	cleanedPath = strings.Replace(cleanedPath, "/", string(sep), -1)
+	originalPath = strings.Replace(originalPath, "/", string(sep), -1)
 
 	if !specifiesCurrentDir(cleanedPath) && specifiesCurrentDir(originalPath) {
-		if !hasTrailingPathSeparator(cleanedPath) {
+		if !hasTrailingPathSeparator(cleanedPath, sep) {
 			// Add a separator if it doesn't already end with one (a cleaned
 			// path would only end in a separator if it is the root).
-			cleanedPath += string(filepath.Separator)
+			cleanedPath += string(sep)
 		}
 		cleanedPath += "."
 	}
 
-	if !hasTrailingPathSeparator(cleanedPath) && hasTrailingPathSeparator(originalPath) {
-		cleanedPath += string(filepath.Separator)
+	if !hasTrailingPathSeparator(cleanedPath, sep) && hasTrailingPathSeparator(originalPath, sep) {
+		cleanedPath += string(sep)
 	}
 
 	return cleanedPath
@@ -52,14 +52,14 @@ func PreserveTrailingDotOrSeparator(cleanedPath, originalPath string) string {
 // assertsDirectory returns whether the given path is
 // asserted to be a directory, i.e., the path ends with
 // a trailing '/' or `/.`, assuming a path separator of `/`.
-func assertsDirectory(path string) bool {
-	return hasTrailingPathSeparator(path) || specifiesCurrentDir(path)
+func assertsDirectory(path string, sep byte) bool {
+	return hasTrailingPathSeparator(path, sep) || specifiesCurrentDir(path)
 }
 
 // hasTrailingPathSeparator returns whether the given
 // path ends with the system's path separator character.
-func hasTrailingPathSeparator(path string) bool {
-	return len(path) > 0 && os.IsPathSeparator(path[len(path)-1])
+func hasTrailingPathSeparator(path string, sep byte) bool {
+	return len(path) > 0 && path[len(path)-1] == sep
 }
 
 // specifiesCurrentDir returns whether the given path specifies
@@ -72,10 +72,10 @@ func specifiesCurrentDir(path string) bool {
 // basename by first cleaning the path but preserves a trailing "." if the
 // original path specified the current directory.
 func SplitPathDirEntry(path string) (dir, base string) {
-	cleanedPath := filepath.Clean(normalizePath(path))
+	cleanedPath := filepath.Clean(filepath.FromSlash(path))
 
 	if specifiesCurrentDir(path) {
-		cleanedPath += string(filepath.Separator) + "."
+		cleanedPath += string(os.PathSeparator) + "."
 	}
 
 	return filepath.Dir(cleanedPath), filepath.Base(cleanedPath)
@@ -106,19 +106,24 @@ func TarResourceRebase(sourcePath, rebaseName string) (content io.ReadCloser, er
 	// Separate the source path between its directory and
 	// the entry in that directory which we are archiving.
 	sourceDir, sourceBase := SplitPathDirEntry(sourcePath)
-
-	filter := []string{sourceBase}
+	opts := TarResourceRebaseOpts(sourceBase, rebaseName)
 
 	logrus.Debugf("copying %q from %q", sourceBase, sourceDir)
+	return TarWithOptions(sourceDir, opts)
+}
 
-	return TarWithOptions(sourceDir, &TarOptions{
+// TarResourceRebaseOpts does not preform the Tar, but instead just creates the rebase
+// parameters to be sent to TarWithOptions (the TarOptions struct)
+func TarResourceRebaseOpts(sourceBase string, rebaseName string) *TarOptions {
+	filter := []string{sourceBase}
+	return &TarOptions{
 		Compression:      Uncompressed,
 		IncludeFiles:     filter,
 		IncludeSourceDir: true,
 		RebaseNames: map[string]string{
 			sourceBase: rebaseName,
 		},
-	})
+	}
 }
 
 // CopyInfo holds basic info about the source
@@ -281,7 +286,7 @@ func PrepareArchiveCopy(srcContent io.Reader, srcInfo, dstInfo CopyInfo) (dstDir
 			srcBase = srcInfo.RebaseName
 		}
 		return dstDir, RebaseArchiveEntries(srcContent, srcBase, dstBase), nil
-	case assertsDirectory(dstInfo.Path):
+	case assertsDirectory(dstInfo.Path, os.PathSeparator):
 		// The destination does not exist and is asserted to be created as a
 		// directory, but the source content is not a directory. This is an
 		// error condition since you cannot create a directory from a file
@@ -351,6 +356,9 @@ func RebaseArchiveEntries(srcContent io.Reader, oldBase, newBase string) io.Read
 	return rebased
 }
 
+// TODO @gupta-ak. These might have to be changed in the future to be
+// continuity driver aware as well to support LCOW.
+
 // CopyResource performs an archive copy from the given source path to the
 // given destination path. The source path MUST exist and the destination
 // path's parent directory must exist.
@@ -365,8 +373,8 @@ func CopyResource(srcPath, dstPath string, followLink bool) error {
 	dstPath = normalizePath(dstPath)
 
 	// Clean the source and destination paths.
-	srcPath = PreserveTrailingDotOrSeparator(filepath.Clean(srcPath), srcPath)
-	dstPath = PreserveTrailingDotOrSeparator(filepath.Clean(dstPath), dstPath)
+	srcPath = PreserveTrailingDotOrSeparator(filepath.Clean(srcPath), srcPath, os.PathSeparator)
+	dstPath = PreserveTrailingDotOrSeparator(filepath.Clean(dstPath), dstPath, os.PathSeparator)
 
 	if srcInfo, err = CopyInfoSourcePath(srcPath, followLink); err != nil {
 		return err
@@ -429,7 +437,8 @@ func ResolveHostSourcePath(path string, followLink bool) (resolvedPath, rebaseNa
 		// resolvedDirPath will have been cleaned (no trailing path separators) so
 		// we can manually join it with the base path element.
 		resolvedPath = resolvedDirPath + string(filepath.Separator) + basePath
-		if hasTrailingPathSeparator(path) && filepath.Base(path) != filepath.Base(resolvedPath) {
+		if hasTrailingPathSeparator(path, os.PathSeparator) &&
+			filepath.Base(path) != filepath.Base(resolvedPath) {
 			rebaseName = filepath.Base(path)
 		}
 	}
@@ -442,11 +451,13 @@ func GetRebaseName(path, resolvedPath string) (string, string) {
 	// linkTarget will have been cleaned (no trailing path separators and dot) so
 	// we can manually join it with them
 	var rebaseName string
-	if specifiesCurrentDir(path) && !specifiesCurrentDir(resolvedPath) {
+	if specifiesCurrentDir(path) &&
+		!specifiesCurrentDir(resolvedPath) {
 		resolvedPath += string(filepath.Separator) + "."
 	}
 
-	if hasTrailingPathSeparator(path) && !hasTrailingPathSeparator(resolvedPath) {
+	if hasTrailingPathSeparator(path, os.PathSeparator) &&
+		!hasTrailingPathSeparator(resolvedPath, os.PathSeparator) {
 		resolvedPath += string(filepath.Separator)
 	}
 

pkg/chrootarchive/archive.go

@@ -16,7 +16,10 @@ func NewArchiver(idMappings *idtools.IDMappings) *archive.Archiver {
 	if idMappings == nil {
 		idMappings = &idtools.IDMappings{}
 	}
-	return &archive.Archiver{Untar: Untar, IDMappings: idMappings}
+	return &archive.Archiver{
+		Untar:         Untar,
+		IDMappingsVar: idMappings,
+	}
 }
 
 // Untar reads a stream of bytes from `archive`, parses it as a tar archive,

pkg/containerfs/archiver.go

@@ -0,0 +1,194 @@
+package containerfs
+
+import (
+	"archive/tar"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/idtools"
+	"github.com/docker/docker/pkg/promise"
+	"github.com/docker/docker/pkg/system"
+	"github.com/sirupsen/logrus"
+)
+
+// TarFunc provides a function definition for a custom Tar function
+type TarFunc func(string, *archive.TarOptions) (io.ReadCloser, error)
+
+// UntarFunc provides a function definition for a custom Untar function
+type UntarFunc func(io.Reader, string, *archive.TarOptions) error
+
+// Archiver provides a similar implementation of the archive.Archiver package with the rootfs abstraction
+type Archiver struct {
+	SrcDriver     Driver
+	DstDriver     Driver
+	Tar           TarFunc
+	Untar         UntarFunc
+	IDMappingsVar *idtools.IDMappings
+}
+
+// TarUntar is a convenience function which calls Tar and Untar, with the output of one piped into the other.
+// If either Tar or Untar fails, TarUntar aborts and returns the error.
+func (archiver *Archiver) TarUntar(src, dst string) error {
+	logrus.Debugf("TarUntar(%s %s)", src, dst)
+	tarArchive, err := archiver.Tar(src, &archive.TarOptions{Compression: archive.Uncompressed})
+	if err != nil {
+		return err
+	}
+	defer tarArchive.Close()
+	options := &archive.TarOptions{
+		UIDMaps: archiver.IDMappingsVar.UIDs(),
+		GIDMaps: archiver.IDMappingsVar.GIDs(),
+	}
+	return archiver.Untar(tarArchive, dst, options)
+}
+
+// UntarPath untar a file from path to a destination, src is the source tar file path.
+func (archiver *Archiver) UntarPath(src, dst string) error {
+	tarArchive, err := archiver.SrcDriver.Open(src)
+	if err != nil {
+		return err
+	}
+	defer tarArchive.Close()
+	options := &archive.TarOptions{
+		UIDMaps: archiver.IDMappingsVar.UIDs(),
+		GIDMaps: archiver.IDMappingsVar.GIDs(),
+	}
+	return archiver.Untar(tarArchive, dst, options)
+}
+
+// CopyWithTar creates a tar archive of filesystem path `src`, and
+// unpacks it at filesystem path `dst`.
+// The archive is streamed directly with fixed buffering and no
+// intermediary disk IO.
+func (archiver *Archiver) CopyWithTar(src, dst string) error {
+	srcSt, err := archiver.SrcDriver.Stat(src)
+	if err != nil {
+		return err
+	}
+	if !srcSt.IsDir() {
+		return archiver.CopyFileWithTar(src, dst)
+	}
+
+	// if this archiver is set up with ID mapping we need to create
+	// the new destination directory with the remapped root UID/GID pair
+	// as owner
+	rootIDs := archiver.IDMappingsVar.RootPair()
+	// Create dst, copy src's content into it
+	if err := idtools.MkdirAllAndChownNew(dst, 0755, rootIDs); err != nil {
+		return err
+	}
+	logrus.Debugf("Calling TarUntar(%s, %s)", src, dst)
+	return archiver.TarUntar(src, dst)
+}
+
+// CopyFileWithTar emulates the behavior of the 'cp' command-line
+// for a single file. It copies a regular file from path `src` to
+// path `dst`, and preserves all its metadata.
+func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) {
+	logrus.Debugf("CopyFileWithTar(%s, %s)", src, dst)
+	srcDriver := archiver.SrcDriver
+	dstDriver := archiver.DstDriver
+
+	srcSt, err := srcDriver.Stat(src)
+	if err != nil {
+		return err
+	}
+
+	if srcSt.IsDir() {
+		return fmt.Errorf("Can't copy a directory")
+	}
+
+	// Clean up the trailing slash. This must be done in an operating
+	// system specific manner.
+	if dst[len(dst)-1] == dstDriver.Separator() {
+		dst = dstDriver.Join(dst, srcDriver.Base(src))
+	}
+
+	// The original call was system.MkdirAll, which is just
+	// os.MkdirAll on not-Windows and changed for Windows.
+	if dstDriver.OS() == "windows" {
+		// Now we are WCOW
+		if err := system.MkdirAll(filepath.Dir(dst), 0700, ""); err != nil {
+			return err
+		}
+	} else {
+		// We can just use the driver.MkdirAll function
+		if err := dstDriver.MkdirAll(dstDriver.Dir(dst), 0700); err != nil {
+			return err
+		}
+	}
+
+	r, w := io.Pipe()
+	errC := promise.Go(func() error {
+		defer w.Close()
+
+		srcF, err := srcDriver.Open(src)
+		if err != nil {
+			return err
+		}
+		defer srcF.Close()
+
+		hdr, err := tar.FileInfoHeader(srcSt, "")
+		if err != nil {
+			return err
+		}
+		hdr.Name = dstDriver.Base(dst)
+		if dstDriver.OS() == "windows" {
+			hdr.Mode = int64(chmodTarEntry(os.FileMode(hdr.Mode)))
+		} else {
+			hdr.Mode = int64(os.FileMode(hdr.Mode))
+		}
+
+		if err := remapIDs(archiver.IDMappingsVar, hdr); err != nil {
+			return err
+		}
+
+		tw := tar.NewWriter(w)
+		defer tw.Close()
+		if err := tw.WriteHeader(hdr); err != nil {
+			return err
+		}
+		if _, err := io.Copy(tw, srcF); err != nil {
+			return err
+		}
+		return nil
+	})
+	defer func() {
+		if er := <-errC; err == nil && er != nil {
+			err = er
+		}
+	}()
+
+	err = archiver.Untar(r, dstDriver.Dir(dst), nil)
+	if err != nil {
+		r.CloseWithError(err)
+	}
+	return err
+}
+
+// IDMappings returns the IDMappings of the archiver.
+func (archiver *Archiver) IDMappings() *idtools.IDMappings {
+	return archiver.IDMappingsVar
+}
+
+func remapIDs(idMappings *idtools.IDMappings, hdr *tar.Header) error {
+	ids, err := idMappings.ToHost(idtools.IDPair{UID: hdr.Uid, GID: hdr.Gid})
+	hdr.Uid, hdr.Gid = ids.UID, ids.GID
+	return err
+}
+
+// chmodTarEntry is used to adjust the file permissions used in tar header based
+// on the platform the archival is done.
+func chmodTarEntry(perm os.FileMode) os.FileMode {
+	//perm &= 0755 // this 0-ed out tar flags (like link, regular file, directory marker etc.)
+	permPart := perm & os.ModePerm
+	noPermPart := perm &^ os.ModePerm
+	// Add the x bit: make everything +x from windows
+	permPart |= 0111
+	permPart &= 0755
+
+	return noPermPart | permPart
+}

pkg/containerfs/containerfs.go

@@ -0,0 +1,87 @@
+package containerfs
+
+import (
+	"path/filepath"
+	"runtime"
+
+	"github.com/containerd/continuity/driver"
+	"github.com/containerd/continuity/pathdriver"
+	"github.com/docker/docker/pkg/symlink"
+)
+
+// ContainerFS is that represents a root file system
+type ContainerFS interface {
+	// Path returns the path to the root. Note that this may not exist
+	// on the local system, so the continuity operations must be used
+	Path() string
+
+	// ResolveScopedPath evaluates the given path scoped to the root.
+	// For example, if root=/a, and path=/b/c, then this function would return /a/b/c.
+	// If rawPath is true, then the function will not preform any modifications
+	// before path resolution. Otherwise, the function will clean the given path
+	// by making it an absolute path.
+	ResolveScopedPath(path string, rawPath bool) (string, error)
+
+	Driver
+}
+
+// Driver combines both continuity's Driver and PathDriver interfaces with a Platform
+// field to determine the OS.
+type Driver interface {
+	// OS returns the OS where the rootfs is located. Essentially,
+	// runtime.GOOS for everything aside from LCOW, which is "linux"
+	OS() string
+
+	// Architecture returns the hardware architecture where the
+	// container is located.
+	Architecture() string
+
+	// Driver & PathDriver provide methods to manipulate files & paths
+	driver.Driver
+	pathdriver.PathDriver
+}
+
+// NewLocalContainerFS is a helper function to implement daemon's Mount interface
+// when the graphdriver mount point is a local path on the machine.
+func NewLocalContainerFS(path string) ContainerFS {
+	return &local{
+		path:       path,
+		Driver:     driver.LocalDriver,
+		PathDriver: pathdriver.LocalPathDriver,
+	}
+}
+
+// NewLocalDriver provides file and path drivers for a local file system. They are
+// essentially a wrapper around the `os` and `filepath` functions.
+func NewLocalDriver() Driver {
+	return &local{
+		Driver:     driver.LocalDriver,
+		PathDriver: pathdriver.LocalPathDriver,
+	}
+}
+
+type local struct {
+	path string
+	driver.Driver
+	pathdriver.PathDriver
+}
+
+func (l *local) Path() string {
+	return l.path
+}
+
+func (l *local) ResolveScopedPath(path string, rawPath bool) (string, error) {
+	cleanedPath := path
+	if !rawPath {
+		cleanedPath = cleanScopedPath(path)
+	}
+	return symlink.FollowSymlinkInScope(filepath.Join(l.path, cleanedPath), l.path)
+}
+
+func (l *local) OS() string {
+	return runtime.GOOS
+}
+
+func (l *local) Architecture() string {
+	return runtime.GOARCH
+}

pkg/containerfs/containerfs_unix.go

@@ -0,0 +1,10 @@
+// +build !windows
+
+package containerfs
+
+import "path/filepath"
+
+// cleanScopedPath preappends a to combine with a mnt path.
+func cleanScopedPath(path string) string {
+	return filepath.Join(string(filepath.Separator), path)
+}

pkg/containerfs/containerfs_windows.go

@@ -0,0 +1,15 @@
+package containerfs
+
+import "path/filepath"
+
+// cleanScopedPath removes the C:\ syntax, and prepares to combine
+// with a volume path
+func cleanScopedPath(path string) string {
+	if len(path) >= 2 {
+		c := path[0]
+		if path[1] == ':' && ('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') {
+			path = path[2:]
+		}
+	}
+	return filepath.Join(string(filepath.Separator), path)
+}

pkg/system/path.go

@@ -1,6 +1,13 @@
 package system
 
-import "runtime"
+import (
+	"fmt"
+	"path/filepath"
+	"runtime"
+	"strings"
+
+	"github.com/containerd/continuity/pathdriver"
+)
 
 const defaultUnixPathEnv = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
 
@@ -19,3 +26,35 @@ func DefaultPathEnv(platform string) string {
 	return defaultUnixPathEnv
 
 }
+
+// CheckSystemDriveAndRemoveDriveLetter verifies that a path, if it includes a drive letter,
+// is the system drive.
+// On Linux: this is a no-op.
+// On Windows: this does the following>
+// CheckSystemDriveAndRemoveDriveLetter verifies and manipulates a Windows path.
+// This is used, for example, when validating a user provided path in docker cp.
+// If a drive letter is supplied, it must be the system drive. The drive letter
+// is always removed. Also, it translates it to OS semantics (IOW / to \). We
+// need the path in this syntax so that it can ultimately be contatenated with
+// a Windows long-path which doesn't support drive-letters. Examples:
+// C:			--> Fail
+// C:\			--> \
+// a			--> a
+// /a			--> \a
+// d:\			--> Fail
+func CheckSystemDriveAndRemoveDriveLetter(path string, driver pathdriver.PathDriver) (string, error) {
+	if runtime.GOOS != "windows" || LCOWSupported() {
+		return path, nil
+	}
+
+	if len(path) == 2 && string(path[1]) == ":" {
+		return "", fmt.Errorf("No relative path specified in %q", path)
+	}
+	if !driver.IsAbs(path) || len(path) < 2 {
+		return filepath.FromSlash(path), nil
+	}
+	if string(path[1]) == ":" && !strings.EqualFold(string(path[0]), "c") {
+		return "", fmt.Errorf("The specified path is not on the system drive (C:)")
+	}
+	return filepath.FromSlash(path[2:]), nil
+}

pkg/system/path_unix.go

@@ -1,9 +0,0 @@
-// +build !windows
-
-package system
-
-// CheckSystemDriveAndRemoveDriveLetter verifies that a path, if it includes a drive letter,
-// is the system drive. This is a no-op on Linux.
-func CheckSystemDriveAndRemoveDriveLetter(path string) (string, error) {
-	return path, nil
-}

pkg/system/path_windows.go

@@ -1,33 +0,0 @@
-// +build windows
-
-package system
-
-import (
-	"fmt"
-	"path/filepath"
-	"strings"
-)
-
-// CheckSystemDriveAndRemoveDriveLetter verifies and manipulates a Windows path.
-// This is used, for example, when validating a user provided path in docker cp.
-// If a drive letter is supplied, it must be the system drive. The drive letter
-// is always removed. Also, it translates it to OS semantics (IOW / to \). We
-// need the path in this syntax so that it can ultimately be concatenated with
-// a Windows long-path which doesn't support drive-letters. Examples:
-// C:			--> Fail
-// C:\			--> \
-// a			--> a
-// /a			--> \a
-// d:\			--> Fail
-func CheckSystemDriveAndRemoveDriveLetter(path string) (string, error) {
-	if len(path) == 2 && string(path[1]) == ":" {
-		return "", fmt.Errorf("No relative path specified in %q", path)
-	}
-	if !filepath.IsAbs(path) || len(path) < 2 {
-		return filepath.FromSlash(path), nil
-	}
-	if string(path[1]) == ":" && !strings.EqualFold(string(path[0]), "c") {
-		return "", fmt.Errorf("The specified path is not on the system drive (C:)")
-	}
-	return filepath.FromSlash(path[2:]), nil
-}

pkg/system/path_windows_test.go

@@ -2,19 +2,23 @@
 
 package system
 
-import "testing"
+import (
+	"testing"
+
+	"github.com/containerd/continuity/pathdriver"
+)
 
 // TestCheckSystemDriveAndRemoveDriveLetter tests CheckSystemDriveAndRemoveDriveLetter
 func TestCheckSystemDriveAndRemoveDriveLetter(t *testing.T) {
 	// Fails if not C drive.
-	_, err := CheckSystemDriveAndRemoveDriveLetter(`d:\`)
+	_, err := CheckSystemDriveAndRemoveDriveLetter(`d:\`, pathdriver.LocalPathDriver)
 	if err == nil || (err != nil && err.Error() != "The specified path is not on the system drive (C:)") {
 		t.Fatalf("Expected error for d:")
 	}
 
 	// Single character is unchanged
 	var path string
-	if path, err = CheckSystemDriveAndRemoveDriveLetter("z"); err != nil {
+	if path, err = CheckSystemDriveAndRemoveDriveLetter("z", pathdriver.LocalPathDriver); err != nil {
 		t.Fatalf("Single character should pass")
 	}
 	if path != "z" {
@@ -22,7 +26,7 @@ func TestCheckSystemDriveAndRemoveDriveLetter(t *testing.T) {
 	}
 
 	// Two characters without colon is unchanged
-	if path, err = CheckSystemDriveAndRemoveDriveLetter("AB"); err != nil {
+	if path, err = CheckSystemDriveAndRemoveDriveLetter("AB", pathdriver.LocalPathDriver); err != nil {
 		t.Fatalf("2 characters without colon should pass")
 	}
 	if path != "AB" {
@@ -30,7 +34,7 @@ func TestCheckSystemDriveAndRemoveDriveLetter(t *testing.T) {
 	}
 
 	// Abs path without drive letter
-	if path, err = CheckSystemDriveAndRemoveDriveLetter(`\l`); err != nil {
+	if path, err = CheckSystemDriveAndRemoveDriveLetter(`\l`, pathdriver.LocalPathDriver); err != nil {
 		t.Fatalf("abs path no drive letter should pass")
 	}
 	if path != `\l` {
@@ -38,7 +42,7 @@ func TestCheckSystemDriveAndRemoveDriveLetter(t *testing.T) {
 	}
 
 	// Abs path without drive letter, linux style
-	if path, err = CheckSystemDriveAndRemoveDriveLetter(`/l`); err != nil {
+	if path, err = CheckSystemDriveAndRemoveDriveLetter(`/l`, pathdriver.LocalPathDriver); err != nil {
 		t.Fatalf("abs path no drive letter linux style should pass")
 	}
 	if path != `\l` {
@@ -46,7 +50,7 @@ func TestCheckSystemDriveAndRemoveDriveLetter(t *testing.T) {
 	}
 
 	// Drive-colon should be stripped
-	if path, err = CheckSystemDriveAndRemoveDriveLetter(`c:\`); err != nil {
+	if path, err = CheckSystemDriveAndRemoveDriveLetter(`c:\`, pathdriver.LocalPathDriver); err != nil {
 		t.Fatalf("An absolute path should pass")
 	}
 	if path != `\` {
@@ -54,7 +58,7 @@ func TestCheckSystemDriveAndRemoveDriveLetter(t *testing.T) {
 	}
 
 	// Verify with a linux-style path
-	if path, err = CheckSystemDriveAndRemoveDriveLetter(`c:/`); err != nil {
+	if path, err = CheckSystemDriveAndRemoveDriveLetter(`c:/`, pathdriver.LocalPathDriver); err != nil {
 		t.Fatalf("An absolute path should pass")
 	}
 	if path != `\` {
@@ -62,7 +66,7 @@ func TestCheckSystemDriveAndRemoveDriveLetter(t *testing.T) {
 	}
 
 	// Failure on c:
-	if path, err = CheckSystemDriveAndRemoveDriveLetter(`c:`); err == nil {
+	if path, err = CheckSystemDriveAndRemoveDriveLetter(`c:`, pathdriver.LocalPathDriver); err == nil {
 		t.Fatalf("c: should fail")
 	}
 	if err.Error() != `No relative path specified in "c:"` {
@@ -70,7 +74,7 @@ func TestCheckSystemDriveAndRemoveDriveLetter(t *testing.T) {
 	}
 
 	// Failure on d:
-	if path, err = CheckSystemDriveAndRemoveDriveLetter(`d:`); err == nil {
+	if path, err = CheckSystemDriveAndRemoveDriveLetter(`d:`, pathdriver.LocalPathDriver); err == nil {
 		t.Fatalf("c: should fail")
 	}
 	if err.Error() != `No relative path specified in "d:"` {

plugin/manager_linux.go

@@ -12,6 +12,7 @@ import (
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/daemon/initlayer"
 	"github.com/docker/docker/libcontainerd"
+	"github.com/docker/docker/pkg/containerfs"
 	"github.com/docker/docker/pkg/idtools"
 	"github.com/docker/docker/pkg/mount"
 	"github.com/docker/docker/pkg/plugins"
@@ -57,7 +58,8 @@ func (pm *Manager) enable(p *v2.Plugin, c *controller, force bool) error {
 		}
 	}
 
-	if err := initlayer.Setup(filepath.Join(pm.config.Root, p.PluginObj.ID, rootFSFileName), idtools.IDPair{0, 0}); err != nil {
+	rootFS := containerfs.NewLocalContainerFS(filepath.Join(pm.config.Root, p.PluginObj.ID, rootFSFileName))
+	if err := initlayer.Setup(rootFS, idtools.IDPair{0, 0}); err != nil {
 		return errors.WithStack(err)
 	}
 

vendor.conf

@@ -27,7 +27,7 @@ github.com/RackSec/srslog 456df3a81436d29ba874f3590eeeee25d666f8a5
 github.com/imdario/mergo 0.2.1
 golang.org/x/sync de49d9dcd27d4f764488181bea099dfe6179bcf0
 
-github.com/containerd/continuity cf279e6ac893682272b4479d4c67fd3abf878b4e
+github.com/containerd/continuity 22694c680ee48fb8f50015b44618517e2bde77e8
 
 #get libnetwork packages
 github.com/docker/libnetwork d5c822319097cc01cc9bd5ffedd74c7ce7c894f2

vendor/github.com/Microsoft/opengcs/service/gcsutils/README

@@ -0,0 +1,4 @@
+1. This program only runs in Linux. So you just first copy the files over to a Linux machine. 
+2. Get Go and and then run make get-deps && make. This is set the $GOPATH for you and build the binaries.
+3. vhd_to_tar and tar_to_vhd are the standalone executables that read/write to stdin/out and do the tar <-> vhd conversion.
+   tar2vhd_server is the service VM server that takes client requests over hvsock.

vendor/github.com/Microsoft/opengcs/service/gcsutils/remotefs/defs.go

@@ -0,0 +1,109 @@
+package remotefs
+
+import (
+	"errors"
+	"os"
+	"time"
+)
+
+// RemotefsCmd is the name of the remotefs meta command
+const RemotefsCmd = "remotefs"
+
+// Name of the commands when called from the cli context (remotefs <CMD> ...)
+const (
+	StatCmd           = "stat"
+	LstatCmd          = "lstat"
+	ReadlinkCmd       = "readlink"
+	MkdirCmd          = "mkdir"
+	MkdirAllCmd       = "mkdirall"
+	RemoveCmd         = "remove"
+	RemoveAllCmd      = "removeall"
+	LinkCmd           = "link"
+	SymlinkCmd        = "symlink"
+	LchmodCmd         = "lchmod"
+	LchownCmd         = "lchown"
+	MknodCmd          = "mknod"
+	MkfifoCmd         = "mkfifo"
+	OpenFileCmd       = "openfile"
+	ReadFileCmd       = "readfile"
+	WriteFileCmd      = "writefile"
+	ReadDirCmd        = "readdir"
+	ResolvePathCmd    = "resolvepath"
+	ExtractArchiveCmd = "extractarchive"
+	ArchivePathCmd    = "archivepath"
+)
+
+// ErrInvalid is returned if the parameters are invalid
+var ErrInvalid = errors.New("invalid arguments")
+
+// ErrUnknown is returned for an unknown remotefs command
+var ErrUnknown = errors.New("unkown command")
+
+// ExportedError is the serialized version of the a Go error.
+// It also provides a trivial implementation of the error interface.
+type ExportedError struct {
+	ErrString string
+	ErrNum    int `json:",omitempty"`
+}
+
+// Error returns an error string
+func (ee *ExportedError) Error() string {
+	return ee.ErrString
+}
+
+// FileInfo is the stat struct returned by the remotefs system. It
+// fulfills the os.FileInfo interface.
+type FileInfo struct {
+	NameVar    string
+	SizeVar    int64
+	ModeVar    os.FileMode
+	ModTimeVar int64 // Serialization of time.Time breaks in travis, so use an int
+	IsDirVar   bool
+}
+
+var _ os.FileInfo = &FileInfo{}
+
+// Name returns the filename from a FileInfo structure
+func (f *FileInfo) Name() string { return f.NameVar }
+
+// Size returns the size from a FileInfo structure
+func (f *FileInfo) Size() int64 { return f.SizeVar }
+
+// Mode returns the mode from a FileInfo structure
+func (f *FileInfo) Mode() os.FileMode { return f.ModeVar }
+
+// ModTime returns the modification time from a FileInfo structure
+func (f *FileInfo) ModTime() time.Time { return time.Unix(0, f.ModTimeVar) }
+
+// IsDir returns the is-directory indicator from a FileInfo structure
+func (f *FileInfo) IsDir() bool { return f.IsDirVar }
+
+// Sys provides an interface to a FileInfo structure
+func (f *FileInfo) Sys() interface{} { return nil }
+
+// FileHeader is a header for remote *os.File operations for remotefs.OpenFile
+type FileHeader struct {
+	Cmd  uint32
+	Size uint64
+}
+
+const (
+	// Read request command.
+	Read uint32 = iota
+	// Write request command.
+	Write
+	// Seek request command.
+	Seek
+	// Close request command.
+	Close
+	// CmdOK is a response meaning request succeeded.
+	CmdOK
+	// CmdFailed is a response meaning request failed.
+	CmdFailed
+)
+
+// SeekHeader is header for the Seek operation for remotefs.OpenFile
+type SeekHeader struct {
+	Offset int64
+	Whence int32
+}

vendor/github.com/Microsoft/opengcs/service/gcsutils/remotefs/remotefs.go

@@ -0,0 +1,546 @@
+// +build !windows
+
+package remotefs
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/json"
+	"io"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/symlink"
+	"golang.org/x/sys/unix"
+)
+
+// Func is the function definition for a generic remote fs function
+// The input to the function is any serialized structs / data from in and the string slice
+// from args. The output of the function will be serialized and written to out.
+type Func func(stdin io.Reader, stdout io.Writer, args []string) error
+
+// Commands provide a string -> remotefs function mapping.
+// This is useful for commandline programs that will receive a string
+// as the function to execute.
+var Commands = map[string]Func{
+	StatCmd:           Stat,
+	LstatCmd:          Lstat,
+	ReadlinkCmd:       Readlink,
+	MkdirCmd:          Mkdir,
+	MkdirAllCmd:       MkdirAll,
+	RemoveCmd:         Remove,
+	RemoveAllCmd:      RemoveAll,
+	LinkCmd:           Link,
+	SymlinkCmd:        Symlink,
+	LchmodCmd:         Lchmod,
+	LchownCmd:         Lchown,
+	MknodCmd:          Mknod,
+	MkfifoCmd:         Mkfifo,
+	OpenFileCmd:       OpenFile,
+	ReadFileCmd:       ReadFile,
+	WriteFileCmd:      WriteFile,
+	ReadDirCmd:        ReadDir,
+	ResolvePathCmd:    ResolvePath,
+	ExtractArchiveCmd: ExtractArchive,
+	ArchivePathCmd:    ArchivePath,
+}
+
+// Stat functions like os.Stat.
+// Args:
+// - args[0] is the path
+// Out:
+// - out = FileInfo object
+func Stat(in io.Reader, out io.Writer, args []string) error {
+	return stat(in, out, args, os.Stat)
+}
+
+// Lstat functions like os.Lstat.
+// Args:
+// - args[0] is the path
+// Out:
+// - out = FileInfo object
+func Lstat(in io.Reader, out io.Writer, args []string) error {
+	return stat(in, out, args, os.Lstat)
+}
+
+func stat(in io.Reader, out io.Writer, args []string, statfunc func(string) (os.FileInfo, error)) error {
+	if len(args) < 1 {
+		return ErrInvalid
+	}
+
+	fi, err := statfunc(args[0])
+	if err != nil {
+		return err
+	}
+
+	info := FileInfo{
+		NameVar:    fi.Name(),
+		SizeVar:    fi.Size(),
+		ModeVar:    fi.Mode(),
+		ModTimeVar: fi.ModTime().UnixNano(),
+		IsDirVar:   fi.IsDir(),
+	}
+
+	buf, err := json.Marshal(info)
+	if err != nil {
+		return err
+	}
+
+	if _, err := out.Write(buf); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Readlink works like os.Readlink
+// In:
+//  - args[0] is path
+// Out:
+//  - Write link result to out
+func Readlink(in io.Reader, out io.Writer, args []string) error {
+	if len(args) < 1 {
+		return ErrInvalid
+	}
+
+	l, err := os.Readlink(args[0])
+	if err != nil {
+		return err
+	}
+
+	if _, err := out.Write([]byte(l)); err != nil {
+		return err
+	}
+	return nil
+}
+
+// Mkdir works like os.Mkdir
+// Args:
+// - args[0] is the path
+// - args[1] is the permissions in octal (like 0755)
+func Mkdir(in io.Reader, out io.Writer, args []string) error {
+	return mkdir(in, out, args, os.Mkdir)
+}
+
+// MkdirAll works like os.MkdirAll.
+// Args:
+// - args[0] is the path
+// - args[1] is the permissions in octal (like 0755)
+func MkdirAll(in io.Reader, out io.Writer, args []string) error {
+	return mkdir(in, out, args, os.MkdirAll)
+}
+
+func mkdir(in io.Reader, out io.Writer, args []string, mkdirFunc func(string, os.FileMode) error) error {
+	if len(args) < 2 {
+		return ErrInvalid
+	}
+
+	perm, err := strconv.ParseUint(args[1], 8, 32)
+	if err != nil {
+		return err
+	}
+	return mkdirFunc(args[0], os.FileMode(perm))
+}
+
+// Remove works like os.Remove
+// Args:
+//	- args[0] is the path
+func Remove(in io.Reader, out io.Writer, args []string) error {
+	return remove(in, out, args, os.Remove)
+}
+
+// RemoveAll works like os.RemoveAll
+// Args:
+//  - args[0] is the path
+func RemoveAll(in io.Reader, out io.Writer, args []string) error {
+	return remove(in, out, args, os.RemoveAll)
+}
+
+func remove(in io.Reader, out io.Writer, args []string, removefunc func(string) error) error {
+	if len(args) < 1 {
+		return ErrInvalid
+	}
+	return removefunc(args[0])
+}
+
+// Link works like os.Link
+// Args:
+//  - args[0] = old path name (link source)
+//  - args[1] = new path name (link dest)
+func Link(in io.Reader, out io.Writer, args []string) error {
+	return link(in, out, args, os.Link)
+}
+
+// Symlink works like os.Symlink
+// Args:
+//  - args[0] = old path name (link source)
+//  - args[1] = new path name (link dest)
+func Symlink(in io.Reader, out io.Writer, args []string) error {
+	return link(in, out, args, os.Symlink)
+}
+
+func link(in io.Reader, out io.Writer, args []string, linkfunc func(string, string) error) error {
+	if len(args) < 2 {
+		return ErrInvalid
+	}
+	return linkfunc(args[0], args[1])
+}
+
+// Lchmod changes permission of the given file without following symlinks
+// Args:
+//  - args[0] = path
+//  - args[1] = permission mode in octal (like 0755)
+func Lchmod(in io.Reader, out io.Writer, args []string) error {
+	if len(args) < 2 {
+		return ErrInvalid
+	}
+
+	perm, err := strconv.ParseUint(args[1], 8, 32)
+	if err != nil {
+		return err
+	}
+
+	path := args[0]
+	if !filepath.IsAbs(path) {
+		path, err = filepath.Abs(path)
+		if err != nil {
+			return err
+		}
+	}
+	return unix.Fchmodat(0, path, uint32(perm), unix.AT_SYMLINK_NOFOLLOW)
+}
+
+// Lchown works like os.Lchown
+// Args:
+//  - args[0] = path
+//  - args[1] = uid in base 10
+//  - args[2] = gid in base 10
+func Lchown(in io.Reader, out io.Writer, args []string) error {
+	if len(args) < 3 {
+		return ErrInvalid
+	}
+
+	uid, err := strconv.ParseInt(args[1], 10, 64)
+	if err != nil {
+		return err
+	}
+
+	gid, err := strconv.ParseInt(args[2], 10, 64)
+	if err != nil {
+		return err
+	}
+	return os.Lchown(args[0], int(uid), int(gid))
+}
+
+// Mknod works like syscall.Mknod
+// Args:
+//  - args[0] = path
+//  - args[1] = permission mode in octal (like 0755)
+//  - args[2] = major device number in base 10
+//  - args[3] = minor device number in base 10
+func Mknod(in io.Reader, out io.Writer, args []string) error {
+	if len(args) < 4 {
+		return ErrInvalid
+	}
+
+	perm, err := strconv.ParseUint(args[1], 8, 32)
+	if err != nil {
+		return err
+	}
+
+	major, err := strconv.ParseInt(args[2], 10, 32)
+	if err != nil {
+		return err
+	}
+
+	minor, err := strconv.ParseInt(args[3], 10, 32)
+	if err != nil {
+		return err
+	}
+
+	dev := unix.Mkdev(uint32(major), uint32(minor))
+	return unix.Mknod(args[0], uint32(perm), int(dev))
+}
+
+// Mkfifo creates a FIFO special file with the given path name and permissions
+// Args:
+// 	- args[0] = path
+//  - args[1] = permission mode in octal (like 0755)
+func Mkfifo(in io.Reader, out io.Writer, args []string) error {
+	if len(args) < 2 {
+		return ErrInvalid
+	}
+
+	perm, err := strconv.ParseUint(args[1], 8, 32)
+	if err != nil {
+		return err
+	}
+	return unix.Mkfifo(args[0], uint32(perm))
+}
+
+// OpenFile works like os.OpenFile. To manage the file pointer state,
+// this function acts as a single file "file server" with Read/Write/Close
+// being serialized control codes from in.
+// Args:
+//  - args[0] = path
+//  - args[1] = flag in base 10
+//  - args[2] = permission mode in octal (like 0755)
+func OpenFile(in io.Reader, out io.Writer, args []string) (err error) {
+	defer func() {
+		if err != nil {
+			// error code will be serialized by the caller, so don't write it here
+			WriteFileHeader(out, &FileHeader{Cmd: CmdFailed}, nil)
+		}
+	}()
+
+	if len(args) < 3 {
+		return ErrInvalid
+	}
+
+	flag, err := strconv.ParseInt(args[1], 10, 32)
+	if err != nil {
+		return err
+	}
+
+	perm, err := strconv.ParseUint(args[2], 8, 32)
+	if err != nil {
+		return err
+	}
+
+	f, err := os.OpenFile(args[0], int(flag), os.FileMode(perm))
+	if err != nil {
+		return err
+	}
+
+	// Signal the client that OpenFile succeeded
+	if err := WriteFileHeader(out, &FileHeader{Cmd: CmdOK}, nil); err != nil {
+		return err
+	}
+
+	for {
+		hdr, err := ReadFileHeader(in)
+		if err != nil {
+			return err
+		}
+
+		var buf []byte
+		switch hdr.Cmd {
+		case Read:
+			buf = make([]byte, hdr.Size, hdr.Size)
+			n, err := f.Read(buf)
+			if err != nil {
+				return err
+			}
+			buf = buf[:n]
+		case Write:
+			if _, err := io.CopyN(f, in, int64(hdr.Size)); err != nil {
+				return err
+			}
+		case Seek:
+			seekHdr := &SeekHeader{}
+			if err := binary.Read(in, binary.BigEndian, seekHdr); err != nil {
+				return err
+			}
+			res, err := f.Seek(seekHdr.Offset, int(seekHdr.Whence))
+			if err != nil {
+				return err
+			}
+			buffer := &bytes.Buffer{}
+			if err := binary.Write(buffer, binary.BigEndian, res); err != nil {
+				return err
+			}
+			buf = buffer.Bytes()
+		case Close:
+			if err := f.Close(); err != nil {
+				return err
+			}
+		default:
+			return ErrUnknown
+		}
+
+		retHdr := &FileHeader{
+			Cmd:  CmdOK,
+			Size: uint64(len(buf)),
+		}
+		if err := WriteFileHeader(out, retHdr, buf); err != nil {
+			return err
+		}
+
+		if hdr.Cmd == Close {
+			break
+		}
+	}
+	return nil
+}
+
+// ReadFile works like ioutil.ReadFile but instead writes the file to a writer
+// Args:
+//  - args[0] = path
+// Out:
+//  - Write file contents to out
+func ReadFile(in io.Reader, out io.Writer, args []string) error {
+	if len(args) < 1 {
+		return ErrInvalid
+	}
+
+	f, err := os.Open(args[0])
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	if _, err := io.Copy(out, f); err != nil {
+		return nil
+	}
+	return nil
+}
+
+// WriteFile works like ioutil.WriteFile but instead reads the file from a reader
+// Args:
+//  - args[0] = path
+//  - args[1] = permission mode in octal (like 0755)
+//  - input data stream from in
+func WriteFile(in io.Reader, out io.Writer, args []string) error {
+	if len(args) < 2 {
+		return ErrInvalid
+	}
+
+	perm, err := strconv.ParseUint(args[1], 8, 32)
+	if err != nil {
+		return err
+	}
+
+	f, err := os.OpenFile(args[0], os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(perm))
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	if _, err := io.Copy(f, in); err != nil {
+		return err
+	}
+	return nil
+}
+
+// ReadDir works like *os.File.Readdir but instead writes the result to a writer
+// Args:
+//  - args[0] = path
+//  - args[1] = number of directory entries to return. If <= 0, return all entries in directory
+func ReadDir(in io.Reader, out io.Writer, args []string) error {
+	if len(args) < 2 {
+		return ErrInvalid
+	}
+
+	n, err := strconv.ParseInt(args[1], 10, 32)
+	if err != nil {
+		return err
+	}
+
+	f, err := os.Open(args[0])
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+
+	infos, err := f.Readdir(int(n))
+	if err != nil {
+		return err
+	}
+
+	fileInfos := make([]FileInfo, len(infos))
+	for i := range infos {
+		fileInfos[i] = FileInfo{
+			NameVar:    infos[i].Name(),
+			SizeVar:    infos[i].Size(),
+			ModeVar:    infos[i].Mode(),
+			ModTimeVar: infos[i].ModTime().UnixNano(),
+			IsDirVar:   infos[i].IsDir(),
+		}
+	}
+
+	buf, err := json.Marshal(fileInfos)
+	if err != nil {
+		return err
+	}
+
+	if _, err := out.Write(buf); err != nil {
+		return err
+	}
+	return nil
+}
+
+// ResolvePath works like docker's symlink.FollowSymlinkInScope.
+// It takens in a `path` and a `root` and evaluates symlinks in `path`
+// as if they were scoped in `root`. `path` must be a child path of `root`.
+// In other words, `path` must have `root` as a prefix.
+// Example:
+// path=/foo/bar -> /baz
+// root=/foo,
+// Expected result = /foo/baz
+//
+// Args:
+// - args[0] is `path`
+// - args[1] is `root`
+// Out:
+// - Write resolved path to stdout
+func ResolvePath(in io.Reader, out io.Writer, args []string) error {
+	if len(args) < 2 {
+		return ErrInvalid
+	}
+	res, err := symlink.FollowSymlinkInScope(args[0], args[1])
+	if err != nil {
+		return err
+	}
+	if _, err = out.Write([]byte(res)); err != nil {
+		return err
+	}
+	return nil
+}
+
+// ExtractArchive extracts the archive read from in.
+// Args:
+// - in = size of json | json of archive.TarOptions | input tar stream
+// - args[0] = extract directory name
+func ExtractArchive(in io.Reader, out io.Writer, args []string) error {
+	if len(args) < 1 {
+		return ErrInvalid
+	}
+
+	opts, err := ReadTarOptions(in)
+	if err != nil {
+		return err
+	}
+
+	if err := archive.Untar(in, args[0], opts); err != nil {
+		return err
+	}
+	return nil
+}
+
+// ArchivePath archives the given directory and writes it to out.
+// Args:
+// - in = size of json | json of archive.TarOptions
+// - args[0] = source directory name
+// Out:
+// - out = tar file of the archive
+func ArchivePath(in io.Reader, out io.Writer, args []string) error {
+	if len(args) < 1 {
+		return ErrInvalid
+	}
+
+	opts, err := ReadTarOptions(in)
+	if err != nil {
+		return err
+	}
+
+	r, err := archive.TarWithOptions(args[0], opts)
+	if err != nil {
+		return err
+	}
+
+	if _, err := io.Copy(out, r); err != nil {
+		return err
+	}
+	return nil
+}

vendor/github.com/Microsoft/opengcs/service/gcsutils/remotefs/utils.go

@@ -0,0 +1,168 @@
+package remotefs
+
+import (
+	"bytes"
+	"encoding/binary"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"os"
+	"syscall"
+
+	"github.com/docker/docker/pkg/archive"
+)
+
+// ReadError is an utility function that reads a serialized error from the given reader
+// and deserializes it.
+func ReadError(in io.Reader) (*ExportedError, error) {
+	b, err := ioutil.ReadAll(in)
+	if err != nil {
+		return nil, err
+	}
+
+	// No error
+	if len(b) == 0 {
+		return nil, nil
+	}
+
+	var exportedErr ExportedError
+	if err := json.Unmarshal(b, &exportedErr); err != nil {
+		return nil, err
+	}
+
+	return &exportedErr, nil
+}
+
+// ExportedToError will convert a ExportedError to an error. It will try to match
+// the error to any existing known error like os.ErrNotExist. Otherwise, it will just
+// return an implementation of the error interface.
+func ExportedToError(ee *ExportedError) error {
+	if ee.Error() == os.ErrNotExist.Error() {
+		return os.ErrNotExist
+	} else if ee.Error() == os.ErrExist.Error() {
+		return os.ErrExist
+	} else if ee.Error() == os.ErrPermission.Error() {
+		return os.ErrPermission
+	}
+	return ee
+}
+
+// WriteError is an utility function that serializes the error
+// and writes it to the output writer.
+func WriteError(err error, out io.Writer) error {
+	if err == nil {
+		return nil
+	}
+	err = fixOSError(err)
+
+	var errno int
+	switch typedError := err.(type) {
+	case *os.PathError:
+		if se, ok := typedError.Err.(syscall.Errno); ok {
+			errno = int(se)
+		}
+	case *os.LinkError:
+		if se, ok := typedError.Err.(syscall.Errno); ok {
+			errno = int(se)
+		}
+	case *os.SyscallError:
+		if se, ok := typedError.Err.(syscall.Errno); ok {
+			errno = int(se)
+		}
+	}
+
+	exportedError := &ExportedError{
+		ErrString: err.Error(),
+		ErrNum:    errno,
+	}
+
+	b, err1 := json.Marshal(exportedError)
+	if err1 != nil {
+		return err1
+	}
+
+	_, err1 = out.Write(b)
+	if err1 != nil {
+		return err1
+	}
+	return nil
+}
+
+// fixOSError converts possible platform dependent error into the portable errors in the
+// Go os package if possible.
+func fixOSError(err error) error {
+	// The os.IsExist, os.IsNotExist, and os.IsPermissions functions are platform
+	// dependent, so sending the raw error might break those functions on a different OS.
+	// Go defines portable errors for these.
+	if os.IsExist(err) {
+		return os.ErrExist
+	} else if os.IsNotExist(err) {
+		return os.ErrNotExist
+	} else if os.IsPermission(err) {
+		return os.ErrPermission
+	}
+	return err
+}
+
+// ReadTarOptions reads from the specified reader and deserializes an archive.TarOptions struct.
+func ReadTarOptions(r io.Reader) (*archive.TarOptions, error) {
+	var size uint64
+	if err := binary.Read(r, binary.BigEndian, &size); err != nil {
+		return nil, err
+	}
+
+	rawJSON := make([]byte, size)
+	if _, err := io.ReadFull(r, rawJSON); err != nil {
+		return nil, err
+	}
+
+	var opts archive.TarOptions
+	if err := json.Unmarshal(rawJSON, &opts); err != nil {
+		return nil, err
+	}
+	return &opts, nil
+}
+
+// WriteTarOptions serializes a archive.TarOptions struct and writes it to the writer.
+func WriteTarOptions(w io.Writer, opts *archive.TarOptions) error {
+	optsBuf, err := json.Marshal(opts)
+	if err != nil {
+		return err
+	}
+
+	optsSize := uint64(len(optsBuf))
+	optsSizeBuf := &bytes.Buffer{}
+	if err := binary.Write(optsSizeBuf, binary.BigEndian, optsSize); err != nil {
+		return err
+	}
+
+	if _, err := optsSizeBuf.WriteTo(w); err != nil {
+		return err
+	}
+
+	if _, err := w.Write(optsBuf); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// ReadFileHeader reads from r and returns a deserialized FileHeader
+func ReadFileHeader(r io.Reader) (*FileHeader, error) {
+	hdr := &FileHeader{}
+	if err := binary.Read(r, binary.BigEndian, hdr); err != nil {
+		return nil, err
+	}
+	return hdr, nil
+}
+
+// WriteFileHeader serializes a FileHeader and writes it to w, along with any extra data
+func WriteFileHeader(w io.Writer, hdr *FileHeader, extraData []byte) error {
+	if err := binary.Write(w, binary.BigEndian, hdr); err != nil {
+		return err
+	}
+	if _, err := w.Write(extraData); err != nil {
+		return err
+	}
+	return nil
+}