moby/pkg/archive/changes_test.go
Sebastiaan van Stijn 55192de9e3
pkg/archive: rename max/min as it collides with go1.21 builtin
Signed-off-by: Sebastiaan van Stijn <github@gone.nl>
2023-08-26 22:02:25 +02:00

550 lines
18 KiB
Go

package archive // import "github.com/docker/docker/pkg/archive"
import (
"os"
"os/exec"
"path"
"path/filepath"
"runtime"
"sort"
"strconv"
"strings"
"syscall"
"testing"
"time"
"github.com/Microsoft/hcsshim/osversion"
"github.com/docker/docker/pkg/idtools"
"github.com/docker/docker/pkg/parsers/kernel"
"github.com/docker/docker/pkg/system"
"gotest.tools/v3/assert"
"gotest.tools/v3/skip"
)
func maxInt(x, y int) int {
if x >= y {
return x
}
return y
}
func copyDir(src, dst string) error {
if runtime.GOOS != "windows" {
return exec.Command("cp", "-a", src, dst).Run()
}
// Could have used xcopy src dst /E /I /H /Y /B. However, xcopy has the
// unfortunate side effect of not preserving timestamps of newly created
// directories in the target directory, so we don't get accurate changes.
// Use robocopy instead. Note this isn't available in microsoft/nanoserver.
// But it has gotchas. See https://weblogs.sqlteam.com/robv/archive/2010/02/17/61106.aspx
err := exec.Command("robocopy", filepath.FromSlash(src), filepath.FromSlash(dst), "/SL", "/COPYALL", "/MIR").Run()
if exiterr, ok := err.(*exec.ExitError); ok {
if status, ok := exiterr.Sys().(syscall.WaitStatus); ok {
if status.ExitStatus()&24 == 0 {
return nil
}
}
}
return err
}
type FileType uint32
const (
Regular FileType = iota
Dir
Symlink
)
type FileData struct {
filetype FileType
path string
contents string
permissions os.FileMode
}
func createSampleDir(t *testing.T, root string) {
files := []FileData{
{filetype: Regular, path: "file1", contents: "file1\n", permissions: 0o600},
{filetype: Regular, path: "file2", contents: "file2\n", permissions: 0o666},
{filetype: Regular, path: "file3", contents: "file3\n", permissions: 0o404},
{filetype: Regular, path: "file4", contents: "file4\n", permissions: 0o600},
{filetype: Regular, path: "file5", contents: "file5\n", permissions: 0o600},
{filetype: Regular, path: "file6", contents: "file6\n", permissions: 0o600},
{filetype: Regular, path: "file7", contents: "file7\n", permissions: 0o600},
{filetype: Dir, path: "dir1", contents: "", permissions: 0o740},
{filetype: Regular, path: "dir1/file1-1", contents: "file1-1\n", permissions: 0o1444},
{filetype: Regular, path: "dir1/file1-2", contents: "file1-2\n", permissions: 0o666},
{filetype: Dir, path: "dir2", contents: "", permissions: 0o700},
{filetype: Regular, path: "dir2/file2-1", contents: "file2-1\n", permissions: 0o666},
{filetype: Regular, path: "dir2/file2-2", contents: "file2-2\n", permissions: 0o666},
{filetype: Dir, path: "dir3", contents: "", permissions: 0o700},
{filetype: Regular, path: "dir3/file3-1", contents: "file3-1\n", permissions: 0o666},
{filetype: Regular, path: "dir3/file3-2", contents: "file3-2\n", permissions: 0o666},
{filetype: Dir, path: "dir4", contents: "", permissions: 0o700},
{filetype: Regular, path: "dir4/file3-1", contents: "file4-1\n", permissions: 0o666},
{filetype: Regular, path: "dir4/file3-2", contents: "file4-2\n", permissions: 0o666},
{filetype: Symlink, path: "symlink1", contents: "target1", permissions: 0o666},
{filetype: Symlink, path: "symlink2", contents: "target2", permissions: 0o666},
{filetype: Symlink, path: "symlink3", contents: root + "/file1", permissions: 0o666},
{filetype: Symlink, path: "symlink4", contents: root + "/symlink3", permissions: 0o666},
{filetype: Symlink, path: "dirSymlink", contents: root + "/dir1", permissions: 0o740},
}
provisionSampleDir(t, root, files)
}
func provisionSampleDir(t *testing.T, root string, files []FileData) {
now := time.Now()
for _, info := range files {
p := path.Join(root, info.path)
if info.filetype == Dir {
err := os.MkdirAll(p, info.permissions)
assert.NilError(t, err)
} else if info.filetype == Regular {
err := os.WriteFile(p, []byte(info.contents), info.permissions)
assert.NilError(t, err)
} else if info.filetype == Symlink {
err := os.Symlink(info.contents, p)
assert.NilError(t, err)
}
if info.filetype != Symlink {
// Set a consistent ctime, atime for all files and dirs
err := system.Chtimes(p, now, now)
assert.NilError(t, err)
}
}
}
func TestChangeString(t *testing.T) {
modifyChange := Change{"change", ChangeModify}
toString := modifyChange.String()
if toString != "C change" {
t.Fatalf("String() of a change with ChangeModify Kind should have been %s but was %s", "C change", toString)
}
addChange := Change{"change", ChangeAdd}
toString = addChange.String()
if toString != "A change" {
t.Fatalf("String() of a change with ChangeAdd Kind should have been %s but was %s", "A change", toString)
}
deleteChange := Change{"change", ChangeDelete}
toString = deleteChange.String()
if toString != "D change" {
t.Fatalf("String() of a change with ChangeDelete Kind should have been %s but was %s", "D change", toString)
}
}
func TestChangesWithNoChanges(t *testing.T) {
rwLayer, err := os.MkdirTemp("", "docker-changes-test")
assert.NilError(t, err)
defer os.RemoveAll(rwLayer)
layer, err := os.MkdirTemp("", "docker-changes-test-layer")
assert.NilError(t, err)
defer os.RemoveAll(layer)
createSampleDir(t, layer)
changes, err := Changes([]string{layer}, rwLayer)
assert.NilError(t, err)
if len(changes) != 0 {
t.Fatalf("Changes with no difference should have detect no changes, but detected %d", len(changes))
}
}
func TestChangesWithChanges(t *testing.T) {
// Mock the readonly layer
layer, err := os.MkdirTemp("", "docker-changes-test-layer")
assert.NilError(t, err)
defer os.RemoveAll(layer)
createSampleDir(t, layer)
os.MkdirAll(path.Join(layer, "dir1/subfolder"), 0o740)
// Mock the RW layer
rwLayer, err := os.MkdirTemp("", "docker-changes-test")
assert.NilError(t, err)
defer os.RemoveAll(rwLayer)
// Create a folder in RW layer
dir1 := path.Join(rwLayer, "dir1")
os.MkdirAll(dir1, 0o740)
deletedFile := path.Join(dir1, ".wh.file1-2")
os.WriteFile(deletedFile, []byte{}, 0o600)
modifiedFile := path.Join(dir1, "file1-1")
os.WriteFile(modifiedFile, []byte{0x00}, 0o1444)
// Let's add a subfolder for a newFile
subfolder := path.Join(dir1, "subfolder")
os.MkdirAll(subfolder, 0o740)
newFile := path.Join(subfolder, "newFile")
os.WriteFile(newFile, []byte{}, 0o740)
changes, err := Changes([]string{layer}, rwLayer)
assert.NilError(t, err)
expectedChanges := []Change{
{filepath.FromSlash("/dir1"), ChangeModify},
{filepath.FromSlash("/dir1/file1-1"), ChangeModify},
{filepath.FromSlash("/dir1/file1-2"), ChangeDelete},
{filepath.FromSlash("/dir1/subfolder"), ChangeModify},
{filepath.FromSlash("/dir1/subfolder/newFile"), ChangeAdd},
}
checkChanges(expectedChanges, changes, t)
}
// See https://github.com/docker/docker/pull/13590
func TestChangesWithChangesGH13590(t *testing.T) {
// TODO Windows. Needs further investigation to identify the failure
if runtime.GOOS == "windows" {
t.Skip("needs more investigation")
}
baseLayer, err := os.MkdirTemp("", "docker-changes-test.")
assert.NilError(t, err)
defer os.RemoveAll(baseLayer)
dir3 := path.Join(baseLayer, "dir1/dir2/dir3")
os.MkdirAll(dir3, 0o7400)
file := path.Join(dir3, "file.txt")
os.WriteFile(file, []byte("hello"), 0o666)
layer, err := os.MkdirTemp("", "docker-changes-test2.")
assert.NilError(t, err)
defer os.RemoveAll(layer)
// Test creating a new file
if err := copyDir(baseLayer+"/dir1", layer+"/"); err != nil {
t.Fatalf("Cmd failed: %q", err)
}
os.Remove(path.Join(layer, "dir1/dir2/dir3/file.txt"))
file = path.Join(layer, "dir1/dir2/dir3/file1.txt")
os.WriteFile(file, []byte("bye"), 0o666)
changes, err := Changes([]string{baseLayer}, layer)
assert.NilError(t, err)
expectedChanges := []Change{
{"/dir1/dir2/dir3", ChangeModify},
{"/dir1/dir2/dir3/file1.txt", ChangeAdd},
}
checkChanges(expectedChanges, changes, t)
// Now test changing a file
layer, err = os.MkdirTemp("", "docker-changes-test3.")
assert.NilError(t, err)
defer os.RemoveAll(layer)
if err := copyDir(baseLayer+"/dir1", layer+"/"); err != nil {
t.Fatalf("Cmd failed: %q", err)
}
file = path.Join(layer, "dir1/dir2/dir3/file.txt")
os.WriteFile(file, []byte("bye"), 0o666)
changes, err = Changes([]string{baseLayer}, layer)
assert.NilError(t, err)
expectedChanges = []Change{
{"/dir1/dir2/dir3/file.txt", ChangeModify},
}
checkChanges(expectedChanges, changes, t)
}
// Create a directory, copy it, make sure we report no changes between the two
func TestChangesDirsEmpty(t *testing.T) {
// Note we parse kernel.GetKernelVersion rather than system.GetOSVersion
// as test binaries aren't manifested, so would otherwise report the wrong
// build number.
if runtime.GOOS == "windows" {
v, err := kernel.GetKernelVersion()
assert.NilError(t, err)
build, _ := strconv.Atoi(strings.Split(strings.SplitN(v.String(), " ", 3)[2][1:], ".")[0])
if build >= osversion.V19H1 {
t.Skip("FIXME: broken on Windows 1903 and up; see #39846")
}
}
src, err := os.MkdirTemp("", "docker-changes-test")
assert.NilError(t, err)
defer os.RemoveAll(src)
createSampleDir(t, src)
dst := src + "-copy"
err = copyDir(src, dst)
assert.NilError(t, err)
defer os.RemoveAll(dst)
changes, err := ChangesDirs(dst, src)
assert.NilError(t, err)
if len(changes) != 0 {
t.Fatalf("Reported changes for identical dirs: %v", changes)
}
os.RemoveAll(src)
os.RemoveAll(dst)
}
func mutateSampleDir(t *testing.T, root string) {
// Remove a regular file
err := os.RemoveAll(path.Join(root, "file1"))
assert.NilError(t, err)
// Remove a directory
err = os.RemoveAll(path.Join(root, "dir1"))
assert.NilError(t, err)
// Remove a symlink
err = os.RemoveAll(path.Join(root, "symlink1"))
assert.NilError(t, err)
// Rewrite a file
err = os.WriteFile(path.Join(root, "file2"), []byte("fileNN\n"), 0o777)
assert.NilError(t, err)
// Replace a file
err = os.RemoveAll(path.Join(root, "file3"))
assert.NilError(t, err)
err = os.WriteFile(path.Join(root, "file3"), []byte("fileMM\n"), 0o404)
assert.NilError(t, err)
// Touch file
err = system.Chtimes(path.Join(root, "file4"), time.Now().Add(time.Second), time.Now().Add(time.Second))
assert.NilError(t, err)
// Replace file with dir
err = os.RemoveAll(path.Join(root, "file5"))
assert.NilError(t, err)
err = os.MkdirAll(path.Join(root, "file5"), 0o666)
assert.NilError(t, err)
// Create new file
err = os.WriteFile(path.Join(root, "filenew"), []byte("filenew\n"), 0o777)
assert.NilError(t, err)
// Create new dir
err = os.MkdirAll(path.Join(root, "dirnew"), 0o766)
assert.NilError(t, err)
// Create a new symlink
err = os.Symlink("targetnew", path.Join(root, "symlinknew"))
assert.NilError(t, err)
// Change a symlink
err = os.RemoveAll(path.Join(root, "symlink2"))
assert.NilError(t, err)
err = os.Symlink("target2change", path.Join(root, "symlink2"))
assert.NilError(t, err)
// Replace dir with file
err = os.RemoveAll(path.Join(root, "dir2"))
assert.NilError(t, err)
err = os.WriteFile(path.Join(root, "dir2"), []byte("dir2\n"), 0o777)
assert.NilError(t, err)
// Touch dir
err = system.Chtimes(path.Join(root, "dir3"), time.Now().Add(time.Second), time.Now().Add(time.Second))
assert.NilError(t, err)
}
func TestChangesDirsMutated(t *testing.T) {
// Note we parse kernel.GetKernelVersion rather than system.GetOSVersion
// as test binaries aren't manifested, so would otherwise report the wrong
// build number.
if runtime.GOOS == "windows" {
v, err := kernel.GetKernelVersion()
assert.NilError(t, err)
build, _ := strconv.Atoi(strings.Split(strings.SplitN(v.String(), " ", 3)[2][1:], ".")[0])
if build >= osversion.V19H1 {
t.Skip("FIXME: broken on Windows 1903 and up; see #39846")
}
}
src, err := os.MkdirTemp("", "docker-changes-test")
assert.NilError(t, err)
createSampleDir(t, src)
dst := src + "-copy"
err = copyDir(src, dst)
assert.NilError(t, err)
defer os.RemoveAll(src)
defer os.RemoveAll(dst)
mutateSampleDir(t, dst)
changes, err := ChangesDirs(dst, src)
assert.NilError(t, err)
sort.Sort(changesByPath(changes))
expectedChanges := []Change{
{filepath.FromSlash("/dir1"), ChangeDelete},
{filepath.FromSlash("/dir2"), ChangeModify},
}
// Note there is slight difference between the Linux and Windows
// implementations here. Due to https://github.com/moby/moby/issues/9874,
// and the fix at https://github.com/moby/moby/pull/11422, Linux does not
// consider a change to the directory time as a change. Windows on NTFS
// does. See https://github.com/moby/moby/pull/37982 for more information.
//
// Note also: https://github.com/moby/moby/pull/37982#discussion_r223523114
// that differences are ordered in the way the test is currently written, hence
// this is in the middle of the list of changes rather than at the start or
// end. Potentially can be addressed later.
if runtime.GOOS == "windows" {
expectedChanges = append(expectedChanges, Change{filepath.FromSlash("/dir3"), ChangeModify})
}
expectedChanges = append(expectedChanges, []Change{
{filepath.FromSlash("/dirnew"), ChangeAdd},
{filepath.FromSlash("/file1"), ChangeDelete},
{filepath.FromSlash("/file2"), ChangeModify},
{filepath.FromSlash("/file3"), ChangeModify},
{filepath.FromSlash("/file4"), ChangeModify},
{filepath.FromSlash("/file5"), ChangeModify},
{filepath.FromSlash("/filenew"), ChangeAdd},
{filepath.FromSlash("/symlink1"), ChangeDelete},
{filepath.FromSlash("/symlink2"), ChangeModify},
{filepath.FromSlash("/symlinknew"), ChangeAdd},
}...)
for i := 0; i < maxInt(len(changes), len(expectedChanges)); i++ {
if i >= len(expectedChanges) {
t.Fatalf("unexpected change %s\n", changes[i].String())
}
if i >= len(changes) {
t.Fatalf("no change for expected change %s\n", expectedChanges[i].String())
}
if changes[i].Path == expectedChanges[i].Path {
if changes[i] != expectedChanges[i] {
t.Fatalf("Wrong change for %s, expected %s, got %s\n", changes[i].Path, changes[i].String(), expectedChanges[i].String())
}
} else if changes[i].Path < expectedChanges[i].Path {
t.Fatalf("unexpected change %q %q\n", changes[i].String(), expectedChanges[i].Path)
} else {
t.Fatalf("no change for expected change %s != %s\n", expectedChanges[i].String(), changes[i].String())
}
}
}
func TestApplyLayer(t *testing.T) {
// TODO Windows. This is very close to working, but it fails with changes
// to \symlinknew and \symlink2. The destination has an updated
// Access/Modify/Change/Birth date to the source (~3/100th sec different).
// Needs further investigation as to why, but I currently believe this is
// just the way NTFS works. I don't think it's a bug in this test or archive.
if runtime.GOOS == "windows" {
t.Skip("needs further investigation")
}
src, err := os.MkdirTemp("", "docker-changes-test")
assert.NilError(t, err)
createSampleDir(t, src)
defer os.RemoveAll(src)
dst := src + "-copy"
err = copyDir(src, dst)
assert.NilError(t, err)
mutateSampleDir(t, dst)
defer os.RemoveAll(dst)
changes, err := ChangesDirs(dst, src)
assert.NilError(t, err)
layer, err := ExportChanges(dst, changes, idtools.IdentityMapping{})
assert.NilError(t, err)
layerCopy, err := NewTempArchive(layer, "")
assert.NilError(t, err)
_, err = ApplyLayer(src, layerCopy)
assert.NilError(t, err)
changes2, err := ChangesDirs(src, dst)
assert.NilError(t, err)
if len(changes2) != 0 {
t.Fatalf("Unexpected differences after reapplying mutation: %v", changes2)
}
}
func TestChangesSizeWithHardlinks(t *testing.T) {
// TODO Windows. Needs further investigation. Likely in ChangeSizes not
// coping correctly with hardlinks on Windows.
if runtime.GOOS == "windows" {
t.Skip("needs further investigation")
}
srcDir, err := os.MkdirTemp("", "docker-test-srcDir")
assert.NilError(t, err)
defer os.RemoveAll(srcDir)
destDir, err := os.MkdirTemp("", "docker-test-destDir")
assert.NilError(t, err)
defer os.RemoveAll(destDir)
creationSize, err := prepareUntarSourceDirectory(100, destDir, true)
assert.NilError(t, err)
changes, err := ChangesDirs(destDir, srcDir)
assert.NilError(t, err)
got := ChangesSize(destDir, changes)
if got != int64(creationSize) {
t.Errorf("Expected %d bytes of changes, got %d", creationSize, got)
}
}
func TestChangesSizeWithNoChanges(t *testing.T) {
size := ChangesSize("/tmp", nil)
if size != 0 {
t.Fatalf("ChangesSizes with no changes should be 0, was %d", size)
}
}
func TestChangesSizeWithOnlyDeleteChanges(t *testing.T) {
changes := []Change{
{Path: "deletedPath", Kind: ChangeDelete},
}
size := ChangesSize("/tmp", changes)
if size != 0 {
t.Fatalf("ChangesSizes with only delete changes should be 0, was %d", size)
}
}
func TestChangesSize(t *testing.T) {
parentPath, err := os.MkdirTemp("", "docker-changes-test")
assert.NilError(t, err)
defer os.RemoveAll(parentPath)
addition := path.Join(parentPath, "addition")
err = os.WriteFile(addition, []byte{0x01, 0x01, 0x01}, 0o744)
assert.NilError(t, err)
modification := path.Join(parentPath, "modification")
err = os.WriteFile(modification, []byte{0x01, 0x01, 0x01}, 0o744)
assert.NilError(t, err)
changes := []Change{
{Path: "addition", Kind: ChangeAdd},
{Path: "modification", Kind: ChangeModify},
}
size := ChangesSize(parentPath, changes)
if size != 6 {
t.Fatalf("Expected 6 bytes of changes, got %d", size)
}
}
func checkChanges(expectedChanges, changes []Change, t *testing.T) {
skip.If(t, runtime.GOOS != "windows" && os.Getuid() != 0, "skipping test that requires root")
sort.Sort(changesByPath(expectedChanges))
sort.Sort(changesByPath(changes))
for i := 0; i < maxInt(len(changes), len(expectedChanges)); i++ {
if i >= len(expectedChanges) {
t.Fatalf("unexpected change %s\n", changes[i].String())
}
if i >= len(changes) {
t.Fatalf("no change for expected change %s\n", expectedChanges[i].String())
}
if changes[i].Path == expectedChanges[i].Path {
if changes[i] != expectedChanges[i] {
t.Fatalf("Wrong change for %s, expected %s, got %s\n", changes[i].Path, changes[i].String(), expectedChanges[i].String())
}
} else if changes[i].Path < expectedChanges[i].Path {
t.Fatalf("unexpected change %s\n", changes[i].String())
} else {
t.Fatalf("no change for expected change %s != %s\n", expectedChanges[i].String(), changes[i].String())
}
}
}