Merge pull request #15898 from Microsoft/15775-buildcontextfix

Windows: Fix long path handling for docker build

builder/internals.go

@@ -34,6 +34,7 @@ import (
 	"github.com/docker/docker/pkg/progressreader"
 	"github.com/docker/docker/pkg/stringid"
 	"github.com/docker/docker/pkg/stringutils"
+	"github.com/docker/docker/pkg/symlink"
 	"github.com/docker/docker/pkg/system"
 	"github.com/docker/docker/pkg/tarsum"
 	"github.com/docker/docker/pkg/urlutil"
@@ -42,7 +43,7 @@ import (
 )
 
 func (b *builder) readContext(context io.Reader) (err error) {
-	tmpdirPath, err := ioutil.TempDir("", "docker-build")
+	tmpdirPath, err := getTempDir("", "docker-build")
 	if err != nil {
 		return
 	}
@@ -305,7 +306,7 @@ func calcCopyInfo(b *builder, cmdName string, cInfos *[]*copyInfo, origPath stri
 		}
 
 		// Create a tmp dir
-		tmpDirName, err := ioutil.TempDir(b.contextPath, "docker-remote")
+		tmpDirName, err := getTempDir(b.contextPath, "docker-remote")
 		if err != nil {
 			return err
 		}
@@ -684,14 +685,14 @@ func (b *builder) run(c *daemon.Container) error {
 
 func (b *builder) checkPathForAddition(orig string) error {
 	origPath := filepath.Join(b.contextPath, orig)
-	origPath, err := filepath.EvalSymlinks(origPath)
+	origPath, err := symlink.EvalSymlinks(origPath)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return fmt.Errorf("%s: no such file or directory", orig)
 		}
 		return err
 	}
-	contextPath, err := filepath.EvalSymlinks(b.contextPath)
+	contextPath, err := symlink.EvalSymlinks(b.contextPath)
 	if err != nil {
 		return err
 	}

builder/internals_unix.go

@@ -3,10 +3,15 @@
 package builder
 
 import (
+	"io/ioutil"
 	"os"
 	"path/filepath"
 )
 
+func getTempDir(dir, prefix string) (string, error) {
+	return ioutil.TempDir(dir, prefix)
+}
+
 func fixPermissions(source, destination string, uid, gid int, destExisted bool) error {
 	// If the destination didn't already exist, or the destination isn't a
 	// directory, then we should Lchown the destination. Otherwise, we shouldn't

builder/internals_windows.go

@@ -2,6 +2,20 @@
 
 package builder
 
+import (
+	"io/ioutil"
+
+	"github.com/docker/docker/pkg/longpath"
+)
+
+func getTempDir(dir, prefix string) (string, error) {
+	tempDir, err := ioutil.TempDir(dir, prefix)
+	if err != nil {
+		return "", err
+	}
+	return longpath.AddPrefix(tempDir), nil
+}
+
 func fixPermissions(source, destination string, uid, gid int, destExisted bool) error {
 	// chown is not supported on Windows
 	return nil

pkg/archive/archive_windows.go

@@ -8,15 +8,14 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+
+	"github.com/docker/docker/pkg/longpath"
 )
 
 // fixVolumePathPrefix does platform specific processing to ensure that if
 // the path being passed in is not in a volume path format, convert it to one.
 func fixVolumePathPrefix(srcPath string) string {
-	if !strings.HasPrefix(srcPath, `\\?\`) {
-		srcPath = `\\?\` + srcPath
-	}
-	return srcPath
+	return longpath.AddPrefix(srcPath)
 }
 
 // getWalkRoot calculates the root path when performing a TarWithOptions.

pkg/chrootarchive/archive_windows.go

@@ -4,6 +4,7 @@ import (
 	"io"
 
 	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/longpath"
 )
 
 // chroot is not supported by Windows
@@ -17,5 +18,5 @@ func invokeUnpack(decompressedArchive io.ReadCloser,
 	// Windows is different to Linux here because Windows does not support
 	// chroot. Hence there is no point sandboxing a chrooted process to
 	// do the unpack. We call inline instead within the daemon process.
-	return archive.Unpack(decompressedArchive, dest, options)
+	return archive.Unpack(decompressedArchive, longpath.AddPrefix(dest), options)
 }

pkg/chrootarchive/diff_windows.go

@@ -5,9 +5,9 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
-	"strings"
 
 	"github.com/docker/docker/pkg/archive"
+	"github.com/docker/docker/pkg/longpath"
 )
 
 // applyLayerHandler parses a diff in the standard layer format from `layer`, and
@@ -17,9 +17,7 @@ func applyLayerHandler(dest string, layer archive.Reader, decompress bool) (size
 	dest = filepath.Clean(dest)
 
 	// Ensure it is a Windows-style volume path
-	if !strings.HasPrefix(dest, `\\?\`) {
-		dest = `\\?\` + dest
-	}
+	dest = longpath.AddPrefix(dest)
 
 	if decompress {
 		decompressed, err := archive.DecompressStream(layer)

pkg/directory/directory_windows.go

@@ -5,10 +5,18 @@ package directory
 import (
 	"os"
 	"path/filepath"
+	"strings"
+
+	"github.com/docker/docker/pkg/longpath"
 )
 
 // Size walks a directory tree and returns its total size in bytes.
 func Size(dir string) (size int64, err error) {
+	fixedPath, err := filepath.Abs(dir)
+	if err != nil {
+		return
+	}
+	fixedPath = longpath.AddPrefix(fixedPath)
 	err = filepath.Walk(dir, func(d string, fileInfo os.FileInfo, e error) error {
 		// Ignore directory sizes
 		if fileInfo == nil {

pkg/longpath/longpath.go

@@ -0,0 +1,21 @@
+// longpath introduces some constants and helper functions for handling long paths
+// in Windows, which are expected to be prepended with `\\?\` and followed by either
+// a drive letter, a UNC server\share, or a volume identifier.
+
+package longpath
+
+import (
+	"strings"
+)
+
+// Prefix is the longpath prefix for Windows file paths.
+const Prefix = `\\?\`
+
+// AddPrefix will add the Windows long path prefix to the path provided if
+// it does not already have it.
+func AddPrefix(path string) string {
+	if !strings.HasPrefix(path, Prefix) {
+		path = Prefix + path
+	}
+	return path
+}

pkg/symlink/README.md

@@ -1,4 +1,5 @@
-Package symlink implements EvalSymlinksInScope which is an extension of filepath.EvalSymlinks
+Package symlink implements EvalSymlinksInScope which is an extension of filepath.EvalSymlinks,
+as well as a Windows long-path aware version of filepath.EvalSymlinks
 from the [Go standard library](https://golang.org/pkg/path/filepath).
 
 The code from filepath.EvalSymlinks has been adapted in fs.go.

pkg/symlink/fs.go

@@ -132,3 +132,12 @@ func evalSymlinksInScope(path, root string) (string, error) {
 	// what's happening here
 	return filepath.Clean(root + filepath.Clean(string(filepath.Separator)+b.String())), nil
 }
+
+// EvalSymlinks returns the path name after the evaluation of any symbolic
+// links.
+// If path is relative the result will be relative to the current directory,
+// unless one of the components is an absolute symbolic link.
+// This version has been updated to support long paths prepended with `\\?\`.
+func EvalSymlinks(path string) (string, error) {
+	return evalSymlinks(path)
+}

pkg/symlink/fs_unix.go

@@ -0,0 +1,11 @@
+// +build !windows
+
+package symlink
+
+import (
+	"path/filepath"
+)
+
+func evalSymlinks(path string) (string, error) {
+	return filepath.EvalSymlinks(path)
+}

pkg/symlink/fs_windows.go

@@ -0,0 +1,156 @@
+package symlink
+
+import (
+	"bytes"
+	"errors"
+	"os"
+	"path/filepath"
+	"strings"
+	"syscall"
+
+	"github.com/docker/docker/pkg/longpath"
+)
+
+func toShort(path string) (string, error) {
+	p, err := syscall.UTF16FromString(path)
+	if err != nil {
+		return "", err
+	}
+	b := p // GetShortPathName says we can reuse buffer
+	n, err := syscall.GetShortPathName(&p[0], &b[0], uint32(len(b)))
+	if err != nil {
+		return "", err
+	}
+	if n > uint32(len(b)) {
+		b = make([]uint16, n)
+		n, err = syscall.GetShortPathName(&p[0], &b[0], uint32(len(b)))
+		if err != nil {
+			return "", err
+		}
+	}
+	return syscall.UTF16ToString(b), nil
+}
+
+func toLong(path string) (string, error) {
+	p, err := syscall.UTF16FromString(path)
+	if err != nil {
+		return "", err
+	}
+	b := p // GetLongPathName says we can reuse buffer
+	n, err := syscall.GetLongPathName(&p[0], &b[0], uint32(len(b)))
+	if err != nil {
+		return "", err
+	}
+	if n > uint32(len(b)) {
+		b = make([]uint16, n)
+		n, err = syscall.GetLongPathName(&p[0], &b[0], uint32(len(b)))
+		if err != nil {
+			return "", err
+		}
+	}
+	b = b[:n]
+	return syscall.UTF16ToString(b), nil
+}
+
+func evalSymlinks(path string) (string, error) {
+	path, err := walkSymlinks(path)
+	if err != nil {
+		return "", err
+	}
+
+	p, err := toShort(path)
+	if err != nil {
+		return "", err
+	}
+	p, err = toLong(p)
+	if err != nil {
+		return "", err
+	}
+	// syscall.GetLongPathName does not change the case of the drive letter,
+	// but the result of EvalSymlinks must be unique, so we have
+	// EvalSymlinks(`c:\a`) == EvalSymlinks(`C:\a`).
+	// Make drive letter upper case.
+	if len(p) >= 2 && p[1] == ':' && 'a' <= p[0] && p[0] <= 'z' {
+		p = string(p[0]+'A'-'a') + p[1:]
+	} else if len(p) >= 6 && p[5] == ':' && 'a' <= p[4] && p[4] <= 'z' {
+		p = p[:3] + string(p[4]+'A'-'a') + p[5:]
+	}
+	return filepath.Clean(p), nil
+}
+
+const utf8RuneSelf = 0x80
+
+func walkSymlinks(path string) (string, error) {
+	const maxIter = 255
+	originalPath := path
+	// consume path by taking each frontmost path element,
+	// expanding it if it's a symlink, and appending it to b
+	var b bytes.Buffer
+	for n := 0; path != ""; n++ {
+		if n > maxIter {
+			return "", errors.New("EvalSymlinks: too many links in " + originalPath)
+		}
+
+		// A path beginnging with `\\?\` represents the root, so automatically
+		// skip that part and begin processing the next segment.
+		if strings.HasPrefix(path, longpath.Prefix) {
+			b.WriteString(longpath.Prefix)
+			path = path[4:]
+			continue
+		}
+
+		// find next path component, p
+		var i = -1
+		for j, c := range path {
+			if c < utf8RuneSelf && os.IsPathSeparator(uint8(c)) {
+				i = j
+				break
+			}
+		}
+		var p string
+		if i == -1 {
+			p, path = path, ""
+		} else {
+			p, path = path[:i], path[i+1:]
+		}
+
+		if p == "" {
+			if b.Len() == 0 {
+				// must be absolute path
+				b.WriteRune(filepath.Separator)
+			}
+			continue
+		}
+
+		// If this is the first segment after the long path prefix, accept the
+		// current segment as a volume root or UNC share and move on to the next.
+		if b.String() == longpath.Prefix {
+			b.WriteString(p)
+			b.WriteRune(filepath.Separator)
+			continue
+		}
+
+		fi, err := os.Lstat(b.String() + p)
+		if err != nil {
+			return "", err
+		}
+		if fi.Mode()&os.ModeSymlink == 0 {
+			b.WriteString(p)
+			if path != "" || (b.Len() == 2 && len(p) == 2 && p[1] == ':') {
+				b.WriteRune(filepath.Separator)
+			}
+			continue
+		}
+
+		// it's a symlink, put it at the front of path
+		dest, err := os.Readlink(b.String() + p)
+		if err != nil {
+			return "", err
+		}
+		if filepath.IsAbs(dest) || os.IsPathSeparator(dest[0]) {
+			b.Reset()
+		}
+		path = dest + string(filepath.Separator) + path
+	}
+	return filepath.Clean(b.String()), nil
+}

utils/utils.go

@@ -200,9 +200,13 @@ func ReplaceOrAppendEnvValues(defaults, overrides []string) []string {
 // can be read and returns an error if some files can't be read
 // symlinks which point to non-existing files don't trigger an error
 func ValidateContextDirectory(srcPath string, excludes []string) error {
-	return filepath.Walk(filepath.Join(srcPath, "."), func(filePath string, f os.FileInfo, err error) error {
+	contextRoot, err := getContextRoot(srcPath)
+	if err != nil {
+		return err
+	}
+	return filepath.Walk(contextRoot, func(filePath string, f os.FileInfo, err error) error {
 		// skip this directory/file if it's not in the path, it won't get added to the context
-		if relFilePath, err := filepath.Rel(srcPath, filePath); err != nil {
+		if relFilePath, err := filepath.Rel(contextRoot, filePath); err != nil {
 			return err
 		} else if skip, err := fileutils.Matches(relFilePath, excludes); err != nil {
 			return err

utils/utils_unix.go

@@ -0,0 +1,11 @@
+// +build !windows
+
+package utils
+
+import (
+	"path/filepath"
+)
+
+func getContextRoot(srcPath string) (string, error) {
+	return filepath.Join(srcPath, "."), nil
+}

utils/utils_windows.go

@@ -0,0 +1,17 @@
+// +build windows
+
+package utils
+
+import (
+	"path/filepath"
+
+	"github.com/docker/docker/pkg/longpath"
+)
+
+func getContextRoot(srcPath string) (string, error) {
+	cr, err := filepath.Abs(srcPath)
+	if err != nil {
+		return "", err
+	}
+	return longpath.AddPrefix(cr), nil
+}