Merge pull request #34670 from sargun/use_copy_file_range

Use In-kernel File Copy for Overlayfs and VFS on Linux

daemon/graphdriver/overlay/copy.go → daemon/graphdriver/copy/copy.go

@@ -1,9 +1,18 @@
 // +build linux
 
-package overlay
+package copy
 
+/*
+#include <linux/fs.h>
+
+#ifndef FICLONE
+#define FICLONE		_IOW(0x94, 9, int)
+#endif
+*/
+import "C"
 import (
 	"fmt"
+	"io"
 	"os"
 	"path/filepath"
 	"syscall"
@@ -15,26 +24,71 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-type copyFlags int
+// Mode indicates whether to use hardlink or copy content
+type Mode int
 
 const (
-	copyHardlink copyFlags = 1 << iota
+	// Content creates a new file, and copies the content of the file
+	Content Mode = iota
+	// Hardlink creates a new hardlink to the existing file
+	Hardlink
 )
 
-func copyRegular(srcPath, dstPath string, mode os.FileMode) error {
+func copyRegular(srcPath, dstPath string, fileinfo os.FileInfo, copyWithFileRange, copyWithFileClone *bool) error {
 	srcFile, err := os.Open(srcPath)
 	if err != nil {
 		return err
 	}
 	defer srcFile.Close()
 
-	dstFile, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE, mode)
+	// If the destination file already exists, we shouldn't blow it away
+	dstFile, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, fileinfo.Mode())
 	if err != nil {
 		return err
 	}
 	defer dstFile.Close()
 
-	_, err = pools.Copy(dstFile, srcFile)
+	if *copyWithFileClone {
+		_, _, err = unix.Syscall(unix.SYS_IOCTL, dstFile.Fd(), C.FICLONE, srcFile.Fd())
+		if err == nil {
+			return nil
+		}
+
+		*copyWithFileClone = false
+		if err == unix.EXDEV {
+			*copyWithFileRange = false
+		}
+	}
+	if *copyWithFileRange {
+		err = doCopyWithFileRange(srcFile, dstFile, fileinfo)
+		// Trying the file_clone may not have caught the exdev case
+		// as the ioctl may not have been available (therefore EINVAL)
+		if err == unix.EXDEV || err == unix.ENOSYS {
+			*copyWithFileRange = false
+		} else if err != nil {
+			return err
+		}
+	}
+	return legacyCopy(srcFile, dstFile)
+}
+
+func doCopyWithFileRange(srcFile, dstFile *os.File, fileinfo os.FileInfo) error {
+	amountLeftToCopy := fileinfo.Size()
+
+	for amountLeftToCopy > 0 {
+		n, err := unix.CopyFileRange(int(srcFile.Fd()), nil, int(dstFile.Fd()), nil, int(amountLeftToCopy), 0)
+		if err != nil {
+			return err
+		}
+
+		amountLeftToCopy = amountLeftToCopy - int64(n)
+	}
+
+	return nil
+}
+
+func legacyCopy(srcFile io.Reader, dstFile io.Writer) error {
+	_, err := pools.Copy(dstFile, srcFile)
 
 	return err
 }
@@ -52,7 +106,11 @@ func copyXattr(srcPath, dstPath, attr string) error {
 	return nil
 }
 
-func copyDir(srcDir, dstDir string, flags copyFlags) error {
+// DirCopy copies or hardlinks the contents of one directory to another,
+// properly handling xattrs, and soft links
+func DirCopy(srcDir, dstDir string, copyMode Mode) error {
+	copyWithFileRange := true
+	copyWithFileClone := true
 	err := filepath.Walk(srcDir, func(srcPath string, f os.FileInfo, err error) error {
 		if err != nil {
 			return err
@@ -78,14 +136,14 @@ func copyDir(srcDir, dstDir string, flags copyFlags) error {
 
 		switch f.Mode() & os.ModeType {
 		case 0: // Regular file
-			if flags&copyHardlink != 0 {
+			if copyMode == Hardlink {
 				isHardlink = true
-				if err := os.Link(srcPath, dstPath); err != nil {
-					return err
+				if err2 := os.Link(srcPath, dstPath); err2 != nil {
+					return err2
 				}
 			} else {
-				if err := copyRegular(srcPath, dstPath, f.Mode()); err != nil {
-					return err
+				if err2 := copyRegular(srcPath, dstPath, f, &copyWithFileRange, &copyWithFileClone); err2 != nil {
+					return err2
 				}
 			}
 

daemon/graphdriver/copy/copy_test.go

@@ -0,0 +1,67 @@
+// +build linux
+
+package copy
+
+import (
+	"io/ioutil"
+	"math/rand"
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/docker/docker/pkg/parsers/kernel"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestIsCopyFileRangeSyscallAvailable(t *testing.T) {
+	// Verifies:
+	// 1. That copyFileRangeEnabled is being set to true when copy_file_range syscall is available
+	// 2. That isCopyFileRangeSyscallAvailable() works on "new" kernels
+	v, err := kernel.GetKernelVersion()
+	require.NoError(t, err)
+
+	copyWithFileRange := true
+	copyWithFileClone := false
+	doCopyTest(t, &copyWithFileRange, &copyWithFileClone)
+
+	if kernel.CompareKernelVersion(*v, kernel.VersionInfo{Kernel: 4, Major: 5, Minor: 0}) < 0 {
+		assert.False(t, copyWithFileRange)
+	} else {
+		assert.True(t, copyWithFileRange)
+	}
+
+}
+
+func TestCopy(t *testing.T) {
+	copyWithFileRange := true
+	copyWithFileClone := true
+	doCopyTest(t, &copyWithFileRange, &copyWithFileClone)
+}
+
+func TestCopyWithoutRange(t *testing.T) {
+	copyWithFileRange := false
+	copyWithFileClone := false
+	doCopyTest(t, &copyWithFileRange, &copyWithFileClone)
+}
+
+func doCopyTest(t *testing.T, copyWithFileRange, copyWithFileClone *bool) {
+	dir, err := ioutil.TempDir("", "docker-copy-check")
+	require.NoError(t, err)
+	defer os.RemoveAll(dir)
+	srcFilename := filepath.Join(dir, "srcFilename")
+	dstFilename := filepath.Join(dir, "dstilename")
+
+	r := rand.New(rand.NewSource(0))
+	buf := make([]byte, 1024)
+	_, err = r.Read(buf)
+	require.NoError(t, err)
+	require.NoError(t, ioutil.WriteFile(srcFilename, buf, 0777))
+	fileinfo, err := os.Stat(srcFilename)
+	require.NoError(t, err)
+
+	require.NoError(t, copyRegular(srcFilename, dstFilename, fileinfo, copyWithFileRange, copyWithFileClone))
+	readBuf, err := ioutil.ReadFile(dstFilename)
+	require.NoError(t, err)
+	assert.Equal(t, buf, readBuf)
+}

daemon/graphdriver/overlay/overlay.go

@@ -13,6 +13,7 @@ import (
 	"strconv"
 
 	"github.com/docker/docker/daemon/graphdriver"
+	"github.com/docker/docker/daemon/graphdriver/copy"
 	"github.com/docker/docker/daemon/graphdriver/overlayutils"
 	"github.com/docker/docker/pkg/archive"
 	"github.com/docker/docker/pkg/containerfs"
@@ -327,7 +328,7 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr
 		return err
 	}
 
-	return copyDir(parentUpperDir, upperDir, 0)
+	return copy.DirCopy(parentUpperDir, upperDir, copy.Content)
 }
 
 func (d *Driver) dir(id string) string {
@@ -443,7 +444,7 @@ func (d *Driver) ApplyDiff(id string, parent string, diff io.Reader) (size int64
 		}
 	}()
 
-	if err = copyDir(parentRootDir, tmpRootDir, copyHardlink); err != nil {
+	if err = copy.DirCopy(parentRootDir, tmpRootDir, copy.Hardlink); err != nil {
 		return 0, err
 	}