moby/pkg/archive/changes_test.go
Cory Snider 098a44c07f Finish refactor of UID/GID usage to a new struct
Finish the refactor which was partially completed with commit
34536c498d, passing around IdentityMapping structs instead of pairs of
[]IDMap slices.

Existing code which uses []IDMap relies on zero-valued fields to be
valid, empty mappings. So in order to successfully finish the
refactoring without introducing bugs, their replacement therefore also
needs to have a useful zero value which represents an empty mapping.
Change IdentityMapping to be a pass-by-value type so that there are no
nil pointers to worry about.

The functionality provided by the deprecated NewIDMappingsFromMaps
function is required by unit tests to to construct arbitrary
IdentityMapping values. And the daemon will always need to access the
mappings to pass them to the Linux kernel. Accommodate these use cases
by exporting the struct fields instead. BuildKit currently depends on
the UIDs and GIDs methods so we cannot get rid of them yet.

Signed-off-by: Cory Snider <csnider@mirantis.com>
2022-03-14 16:28:57 -04:00

550 lines
18 KiB
Go

package archive // import "github.com/docker/docker/pkg/archive"
import (
"os"
"os/exec"
"path"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
"syscall"
"testing"
"time"
"github.com/Microsoft/hcsshim/osversion"
"github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/parsers/kernel"
"github.com/docker/docker/pkg/system"
"gotest.tools/v3/assert"
"gotest.tools/v3/skip"
)
func max(x, y int) int {
if x >= y {
return x
}
return y
}
func copyDir(src, dst string) error {
if runtime.GOOS != "windows" {
return exec.Command("cp", "-a", src, dst).Run()
}
// Could have used xcopy src dst /E /I /H /Y /B. However, xcopy has the
// unfortunate side effect of not preserving timestamps of newly created
// directories in the target directory, so we don't get accurate changes.
// Use robocopy instead. Note this isn't available in microsoft/nanoserver.
// But it has gotchas. See https://weblogs.sqlteam.com/robv/archive/2010/02/17/61106.aspx
err := exec.Command("robocopy", filepath.FromSlash(src), filepath.FromSlash(dst), "/SL", "/COPYALL", "/MIR").Run()
if exiterr, ok := err.(*exec.ExitError); ok {
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
if status.ExitStatus()&24 == 0 {
return nil
}
}
}
return err
}
type FileType uint32
const (
Regular FileType = iota
Dir
Symlink
)
type FileData struct {
filetype FileType
path string
contents string
permissions os.FileMode
}
func createSampleDir(t *testing.T, root string) {
files := []FileData{
{filetype: Regular, path: "file1", contents: "file1\n", permissions: 0600},
{filetype: Regular, path: "file2", contents: "file2\n", permissions: 0666},
{filetype: Regular, path: "file3", contents: "file3\n", permissions: 0404},
{filetype: Regular, path: "file4", contents: "file4\n", permissions: 0600},
{filetype: Regular, path: "file5", contents: "file5\n", permissions: 0600},
{filetype: Regular, path: "file6", contents: "file6\n", permissions: 0600},
{filetype: Regular, path: "file7", contents: "file7\n", permissions: 0600},
{filetype: Dir, path: "dir1", contents: "", permissions: 0740},
{filetype: Regular, path: "dir1/file1-1", contents: "file1-1\n", permissions: 01444},
{filetype: Regular, path: "dir1/file1-2", contents: "file1-2\n", permissions: 0666},
{filetype: Dir, path: "dir2", contents: "", permissions: 0700},
{filetype: Regular, path: "dir2/file2-1", contents: "file2-1\n", permissions: 0666},
{filetype: Regular, path: "dir2/file2-2", contents: "file2-2\n", permissions: 0666},
{filetype: Dir, path: "dir3", contents: "", permissions: 0700},
{filetype: Regular, path: "dir3/file3-1", contents: "file3-1\n", permissions: 0666},
{filetype: Regular, path: "dir3/file3-2", contents: "file3-2\n", permissions: 0666},
{filetype: Dir, path: "dir4", contents: "", permissions: 0700},
{filetype: Regular, path: "dir4/file3-1", contents: "file4-1\n", permissions: 0666},
{filetype: Regular, path: "dir4/file3-2", contents: "file4-2\n", permissions: 0666},
{filetype: Symlink, path: "symlink1", contents: "target1", permissions: 0666},
{filetype: Symlink, path: "symlink2", contents: "target2", permissions: 0666},
{filetype: Symlink, path: "symlink3", contents: root + "/file1", permissions: 0666},
{filetype: Symlink, path: "symlink4", contents: root + "/symlink3", permissions: 0666},
{filetype: Symlink, path: "dirSymlink", contents: root + "/dir1", permissions: 0740},
}
provisionSampleDir(t, root, files)
}
func provisionSampleDir(t *testing.T, root string, files []FileData) {
now := time.Now()
for _, info := range files {
p := path.Join(root, info.path)
if info.filetype == Dir {
err := os.MkdirAll(p, info.permissions)
assert.NilError(t, err)
} else if info.filetype == Regular {
err := os.WriteFile(p, []byte(info.contents), info.permissions)
assert.NilError(t, err)
} else if info.filetype == Symlink {
err := os.Symlink(info.contents, p)
assert.NilError(t, err)
}
if info.filetype != Symlink {
// Set a consistent ctime, atime for all files and dirs
err := system.Chtimes(p, now, now)
assert.NilError(t, err)
}
}
}
func TestChangeString(t *testing.T) {
modifyChange := Change{"change", ChangeModify}
toString := modifyChange.String()
if toString != "C change" {
t.Fatalf("String() of a change with ChangeModify Kind should have been %s but was %s", "C change", toString)
}
addChange := Change{"change", ChangeAdd}
toString = addChange.String()
if toString != "A change" {
t.Fatalf("String() of a change with ChangeAdd Kind should have been %s but was %s", "A change", toString)
}
deleteChange := Change{"change", ChangeDelete}
toString = deleteChange.String()
if toString != "D change" {
t.Fatalf("String() of a change with ChangeDelete Kind should have been %s but was %s", "D change", toString)
}
}
func TestChangesWithNoChanges(t *testing.T) {
rwLayer, err := os.MkdirTemp("", "docker-changes-test")
assert.NilError(t, err)
defer os.RemoveAll(rwLayer)
layer, err := os.MkdirTemp("", "docker-changes-test-layer")
assert.NilError(t, err)
defer os.RemoveAll(layer)
createSampleDir(t, layer)
changes, err := Changes([]string{layer}, rwLayer)
assert.NilError(t, err)
if len(changes) != 0 {
t.Fatalf("Changes with no difference should have detect no changes, but detected %d", len(changes))
}
}
func TestChangesWithChanges(t *testing.T) {
// Mock the readonly layer
layer, err := os.MkdirTemp("", "docker-changes-test-layer")
assert.NilError(t, err)
defer os.RemoveAll(layer)
createSampleDir(t, layer)
os.MkdirAll(path.Join(layer, "dir1/subfolder"), 0740)
// Mock the RW layer
rwLayer, err := os.MkdirTemp("", "docker-changes-test")
assert.NilError(t, err)
defer os.RemoveAll(rwLayer)
// Create a folder in RW layer
dir1 := path.Join(rwLayer, "dir1")
os.MkdirAll(dir1, 0740)
deletedFile := path.Join(dir1, ".wh.file1-2")
os.WriteFile(deletedFile, []byte{}, 0600)
modifiedFile := path.Join(dir1, "file1-1")
os.WriteFile(modifiedFile, []byte{0x00}, 01444)
// Let's add a subfolder for a newFile
subfolder := path.Join(dir1, "subfolder")
os.MkdirAll(subfolder, 0740)
newFile := path.Join(subfolder, "newFile")
os.WriteFile(newFile, []byte{}, 0740)
changes, err := Changes([]string{layer}, rwLayer)
assert.NilError(t, err)
expectedChanges := []Change{
{filepath.FromSlash("/dir1"), ChangeModify},
{filepath.FromSlash("/dir1/file1-1"), ChangeModify},
{filepath.FromSlash("/dir1/file1-2"), ChangeDelete},
{filepath.FromSlash("/dir1/subfolder"), ChangeModify},
{filepath.FromSlash("/dir1/subfolder/newFile"), ChangeAdd},
}
checkChanges(expectedChanges, changes, t)
}
// See https://github.com/docker/docker/pull/13590
func TestChangesWithChangesGH13590(t *testing.T) {
// TODO Windows. Needs further investigation to identify the failure
if runtime.GOOS == "windows" {
t.Skip("needs more investigation")
}
baseLayer, err := os.MkdirTemp("", "docker-changes-test.")
assert.NilError(t, err)
defer os.RemoveAll(baseLayer)
dir3 := path.Join(baseLayer, "dir1/dir2/dir3")
os.MkdirAll(dir3, 07400)
file := path.Join(dir3, "file.txt")
os.WriteFile(file, []byte("hello"), 0666)
layer, err := os.MkdirTemp("", "docker-changes-test2.")
assert.NilError(t, err)
defer os.RemoveAll(layer)
// Test creating a new file
if err := copyDir(baseLayer+"/dir1", layer+"/"); err != nil {
t.Fatalf("Cmd failed: %q", err)
}
os.Remove(path.Join(layer, "dir1/dir2/dir3/file.txt"))
file = path.Join(layer, "dir1/dir2/dir3/file1.txt")
os.WriteFile(file, []byte("bye"), 0666)
changes, err := Changes([]string{baseLayer}, layer)
assert.NilError(t, err)
expectedChanges := []Change{
{"/dir1/dir2/dir3", ChangeModify},
{"/dir1/dir2/dir3/file1.txt", ChangeAdd},
}
checkChanges(expectedChanges, changes, t)
// Now test changing a file
layer, err = os.MkdirTemp("", "docker-changes-test3.")
assert.NilError(t, err)
defer os.RemoveAll(layer)
if err := copyDir(baseLayer+"/dir1", layer+"/"); err != nil {
t.Fatalf("Cmd failed: %q", err)
}
file = path.Join(layer, "dir1/dir2/dir3/file.txt")
os.WriteFile(file, []byte("bye"), 0666)
changes, err = Changes([]string{baseLayer}, layer)
assert.NilError(t, err)
expectedChanges = []Change{
{"/dir1/dir2/dir3/file.txt", ChangeModify},
}
checkChanges(expectedChanges, changes, t)
}
// Create a directory, copy it, make sure we report no changes between the two
func TestChangesDirsEmpty(t *testing.T) {
// Note we parse kernel.GetKernelVersion rather than system.GetOSVersion
// as test binaries aren't manifested, so would otherwise report the wrong
// build number.
if runtime.GOOS == "windows" {
v, err := kernel.GetKernelVersion()
assert.NilError(t, err)
build, _ := strconv.Atoi(strings.Split(strings.SplitN(v.String(), " ", 3)[2][1:], ".")[0])
if build >= osversion.V19H1 {
t.Skip("FIXME: broken on Windows 1903 and up; see #39846")
}
}
src, err := os.MkdirTemp("", "docker-changes-test")
assert.NilError(t, err)
defer os.RemoveAll(src)
createSampleDir(t, src)
dst := src + "-copy"
err = copyDir(src, dst)
assert.NilError(t, err)
defer os.RemoveAll(dst)
changes, err := ChangesDirs(dst, src)
assert.NilError(t, err)
if len(changes) != 0 {
t.Fatalf("Reported changes for identical dirs: %v", changes)
}
os.RemoveAll(src)
os.RemoveAll(dst)
}
func mutateSampleDir(t *testing.T, root string) {
// Remove a regular file
err := os.RemoveAll(path.Join(root, "file1"))
assert.NilError(t, err)
// Remove a directory
err = os.RemoveAll(path.Join(root, "dir1"))
assert.NilError(t, err)
// Remove a symlink
err = os.RemoveAll(path.Join(root, "symlink1"))
assert.NilError(t, err)
// Rewrite a file
err = os.WriteFile(path.Join(root, "file2"), []byte("fileNN\n"), 0777)
assert.NilError(t, err)
// Replace a file
err = os.RemoveAll(path.Join(root, "file3"))
assert.NilError(t, err)
err = os.WriteFile(path.Join(root, "file3"), []byte("fileMM\n"), 0404)
assert.NilError(t, err)
// Touch file
err = system.Chtimes(path.Join(root, "file4"), time.Now().Add(time.Second), time.Now().Add(time.Second))
assert.NilError(t, err)
// Replace file with dir
err = os.RemoveAll(path.Join(root, "file5"))
assert.NilError(t, err)
err = os.MkdirAll(path.Join(root, "file5"), 0666)
assert.NilError(t, err)
// Create new file
err = os.WriteFile(path.Join(root, "filenew"), []byte("filenew\n"), 0777)
assert.NilError(t, err)
// Create new dir
err = os.MkdirAll(path.Join(root, "dirnew"), 0766)
assert.NilError(t, err)
// Create a new symlink
err = os.Symlink("targetnew", path.Join(root, "symlinknew"))
assert.NilError(t, err)
// Change a symlink
err = os.RemoveAll(path.Join(root, "symlink2"))
assert.NilError(t, err)
err = os.Symlink("target2change", path.Join(root, "symlink2"))
assert.NilError(t, err)
// Replace dir with file
err = os.RemoveAll(path.Join(root, "dir2"))
assert.NilError(t, err)
err = os.WriteFile(path.Join(root, "dir2"), []byte("dir2\n"), 0777)
assert.NilError(t, err)
// Touch dir
err = system.Chtimes(path.Join(root, "dir3"), time.Now().Add(time.Second), time.Now().Add(time.Second))
assert.NilError(t, err)
}
func TestChangesDirsMutated(t *testing.T) {
// Note we parse kernel.GetKernelVersion rather than system.GetOSVersion
// as test binaries aren't manifested, so would otherwise report the wrong
// build number.
if runtime.GOOS == "windows" {
v, err := kernel.GetKernelVersion()
assert.NilError(t, err)
build, _ := strconv.Atoi(strings.Split(strings.SplitN(v.String(), " ", 3)[2][1:], ".")[0])
if build >= osversion.V19H1 {
t.Skip("FIXME: broken on Windows 1903 and up; see #39846")
}
}
src, err := os.MkdirTemp("", "docker-changes-test")
assert.NilError(t, err)
createSampleDir(t, src)
dst := src + "-copy"
err = copyDir(src, dst)
assert.NilError(t, err)
defer os.RemoveAll(src)
defer os.RemoveAll(dst)
mutateSampleDir(t, dst)
changes, err := ChangesDirs(dst, src)
assert.NilError(t, err)
sort.Sort(changesByPath(changes))
expectedChanges := []Change{
{filepath.FromSlash("/dir1"), ChangeDelete},
{filepath.FromSlash("/dir2"), ChangeModify},
}
// Note there is slight difference between the Linux and Windows
// implementations here. Due to https://github.com/moby/moby/issues/9874,
// and the fix at https://github.com/moby/moby/pull/11422, Linux does not
// consider a change to the directory time as a change. Windows on NTFS
// does. See https://github.com/moby/moby/pull/37982 for more information.
//
// Note also: https://github.com/moby/moby/pull/37982#discussion_r223523114
// that differences are ordered in the way the test is currently written, hence
// this is in the middle of the list of changes rather than at the start or
// end. Potentially can be addressed later.
if runtime.GOOS == "windows" {
expectedChanges = append(expectedChanges, Change{filepath.FromSlash("/dir3"), ChangeModify})
}
expectedChanges = append(expectedChanges, []Change{
{filepath.FromSlash("/dirnew"), ChangeAdd},
{filepath.FromSlash("/file1"), ChangeDelete},
{filepath.FromSlash("/file2"), ChangeModify},
{filepath.FromSlash("/file3"), ChangeModify},
{filepath.FromSlash("/file4"), ChangeModify},
{filepath.FromSlash("/file5"), ChangeModify},
{filepath.FromSlash("/filenew"), ChangeAdd},
{filepath.FromSlash("/symlink1"), ChangeDelete},
{filepath.FromSlash("/symlink2"), ChangeModify},
{filepath.FromSlash("/symlinknew"), ChangeAdd},
}...)
for i := 0; i < max(len(changes), len(expectedChanges)); i++ {
if i >= len(expectedChanges) {
t.Fatalf("unexpected change %s\n", changes[i].String())
}
if i >= len(changes) {
t.Fatalf("no change for expected change %s\n", expectedChanges[i].String())
}
if changes[i].Path == expectedChanges[i].Path {
if changes[i] != expectedChanges[i] {
t.Fatalf("Wrong change for %s, expected %s, got %s\n", changes[i].Path, changes[i].String(), expectedChanges[i].String())
}
} else if changes[i].Path < expectedChanges[i].Path {
t.Fatalf("unexpected change %q %q\n", changes[i].String(), expectedChanges[i].Path)
} else {
t.Fatalf("no change for expected change %s != %s\n", expectedChanges[i].String(), changes[i].String())
}
}
}
func TestApplyLayer(t *testing.T) {
// TODO Windows. This is very close to working, but it fails with changes
// to \symlinknew and \symlink2. The destination has an updated
// Access/Modify/Change/Birth date to the source (~3/100th sec different).
// Needs further investigation as to why, but I currently believe this is
// just the way NTFS works. I don't think it's a bug in this test or archive.
if runtime.GOOS == "windows" {
t.Skip("needs further investigation")
}
src, err := os.MkdirTemp("", "docker-changes-test")
assert.NilError(t, err)
createSampleDir(t, src)
defer os.RemoveAll(src)
dst := src + "-copy"
err = copyDir(src, dst)
assert.NilError(t, err)
mutateSampleDir(t, dst)
defer os.RemoveAll(dst)
changes, err := ChangesDirs(dst, src)
assert.NilError(t, err)
layer, err := ExportChanges(dst, changes, idtools.IdentityMapping{})
assert.NilError(t, err)
layerCopy, err := NewTempArchive(layer, "")
assert.NilError(t, err)
_, err = ApplyLayer(src, layerCopy)
assert.NilError(t, err)
changes2, err := ChangesDirs(src, dst)
assert.NilError(t, err)
if len(changes2) != 0 {
t.Fatalf("Unexpected differences after reapplying mutation: %v", changes2)
}
}
func TestChangesSizeWithHardlinks(t *testing.T) {
// TODO Windows. Needs further investigation. Likely in ChangeSizes not
// coping correctly with hardlinks on Windows.
if runtime.GOOS == "windows" {
t.Skip("needs further investigation")
}
srcDir, err := os.MkdirTemp("", "docker-test-srcDir")
assert.NilError(t, err)
defer os.RemoveAll(srcDir)
destDir, err := os.MkdirTemp("", "docker-test-destDir")
assert.NilError(t, err)
defer os.RemoveAll(destDir)
creationSize, err := prepareUntarSourceDirectory(100, destDir, true)
assert.NilError(t, err)
changes, err := ChangesDirs(destDir, srcDir)
assert.NilError(t, err)
got := ChangesSize(destDir, changes)
if got != int64(creationSize) {
t.Errorf("Expected %d bytes of changes, got %d", creationSize, got)
}
}
func TestChangesSizeWithNoChanges(t *testing.T) {
size := ChangesSize("/tmp", nil)
if size != 0 {
t.Fatalf("ChangesSizes with no changes should be 0, was %d", size)
}
}
func TestChangesSizeWithOnlyDeleteChanges(t *testing.T) {
changes := []Change{
{Path: "deletedPath", Kind: ChangeDelete},
}
size := ChangesSize("/tmp", changes)
if size != 0 {
t.Fatalf("ChangesSizes with only delete changes should be 0, was %d", size)
}
}
func TestChangesSize(t *testing.T) {
parentPath, err := os.MkdirTemp("", "docker-changes-test")
assert.NilError(t, err)
defer os.RemoveAll(parentPath)
addition := path.Join(parentPath, "addition")
err = os.WriteFile(addition, []byte{0x01, 0x01, 0x01}, 0744)
assert.NilError(t, err)
modification := path.Join(parentPath, "modification")
err = os.WriteFile(modification, []byte{0x01, 0x01, 0x01}, 0744)
assert.NilError(t, err)
changes := []Change{
{Path: "addition", Kind: ChangeAdd},
{Path: "modification", Kind: ChangeModify},
}
size := ChangesSize(parentPath, changes)
if size != 6 {
t.Fatalf("Expected 6 bytes of changes, got %d", size)
}
}
func checkChanges(expectedChanges, changes []Change, t *testing.T) {
skip.If(t, runtime.GOOS != "windows" && os.Getuid() != 0, "skipping test that requires root")
sort.Sort(changesByPath(expectedChanges))
sort.Sort(changesByPath(changes))
for i := 0; i < max(len(changes), len(expectedChanges)); i++ {
if i >= len(expectedChanges) {
t.Fatalf("unexpected change %s\n", changes[i].String())
}
if i >= len(changes) {
t.Fatalf("no change for expected change %s\n", expectedChanges[i].String())
}
if changes[i].Path == expectedChanges[i].Path {
if changes[i] != expectedChanges[i] {
t.Fatalf("Wrong change for %s, expected %s, got %s\n", changes[i].Path, changes[i].String(), expectedChanges[i].String())
}
} else if changes[i].Path < expectedChanges[i].Path {
t.Fatalf("unexpected change %s\n", changes[i].String())
} else {
t.Fatalf("no change for expected change %s != %s\n", expectedChanges[i].String(), changes[i].String())
}
}
}