Merge pull request #26119 from cpuguy83/lazily_load_fixtures

Move some test fixtures to go

hack/make/.ensure-frozen-images

@@ -1,67 +0,0 @@
-#!/bin/bash
-set -e
-
-# image list should match what's in the Dockerfile (minus the explicit images IDs)
-images=(
-	buildpack-deps:jessie
-	busybox:latest
-	debian:jessie
-	hello-world:latest
-)
-
-if [ "$TEST_IMAGE_NAMESPACE" ]; then
-	for (( i = 0; i < ${#images[@]}; i++ )); do
-		images[$i]="$TEST_IMAGE_NAMESPACE/${images[$i]}"
-	done
-fi
-
-if ! docker inspect "${images[@]}" &> /dev/null; then
-	hardCodedDir='/docker-frozen-images'
-	if [ -d "$hardCodedDir" ]; then
-		# Do not use a subshell for the following command. Windows to Linux CI
-		# runs bash 3.x so will not trap an error in a subshell.
-		# http://stackoverflow.com/questions/22630363/how-does-set-e-work-with-subshells
-		set -x; tar -cC "$hardCodedDir" . | docker load; set +x
-	else
-		dir="$DEST/frozen-images"
-		# extract the exact "RUN download-frozen-image-v2.sh" line from the Dockerfile itself for consistency
-		# NOTE: this will fail if either "curl" or "jq" is not installed or if the Dockerfile is not available/readable
-		awk '
-			$1 == "RUN" && $2 == "./contrib/download-frozen-image-v2.sh" {
-				for (i = 2; i < NF; i++)
-					printf ( $i == "'"$hardCodedDir"'" ? "'"$dir"'" : $i ) " ";
-				print $NF;
-				if (/\\$/) {
-					inCont = 1;
-					next;
-				}
-			}
-			inCont {
-				print;
-				if (!/\\$/) {
-					inCont = 0;
-				}
-			}
-		' "$DOCKERFILE" | sh -x
-		# Do not use a subshell for the following command. Windows to Linux CI
-		# runs bash 3.x so will not trap an error in a subshell.
-		# http://stackoverflow.com/questions/22630363/how-does-set-e-work-with-subshells
-		set -x; tar -cC "$dir" . | docker load; set +x
-	fi
-fi
-
-if [ "$TEST_IMAGE_NAMESPACE" ]; then
-	for image in "${images[@]}"; do
-		target="${image#$TEST_IMAGE_NAMESPACE/}"
-		if [ "$target" != "$image" ]; then
-			# tag images to ensure that all integrations work with the defined image names
-			docker tag "$image" "$target"
-			# then remove original tags as these make problems with later tests (e.g., TestInspectApiImageResponse)
-			docker rmi "$image"
-		fi
-	done
-fi
-
-# explicitly rename "hello-world:latest" to ":frozen" for the test that uses it
-docker tag hello-world:latest hello-world:frozen
-docker rmi hello-world:latest

hack/make/.ensure-frozen-images-windows

@@ -1,32 +0,0 @@
-#!/bin/bash
-set -e
-
-# This scripts sets up the required images for Windows to Windows CI
-
-# Tag (microsoft/)windowsservercore as latest
-set +e
-! BUILD=$(docker images | grep windowsservercore | grep -v latest | awk '{print $2}')
-if [ -z $BUILD ]; then
-	echo "ERROR: Could not find windowsservercore images"
-	exit 1
-fi
-
-# Get the name. Around 2016 6D TP5, these have the microsoft/ prefix, hence cater for both.
-! IMAGENAME=$(docker images | grep windowsservercore | grep -v latest | awk '{print $1}')
-if [ -z $IMAGENAME ]; then
-	echo "ERROR: Could not find windowsservercore image"
-	exit 1
-fi
-
-! LATESTCOUNT=$(docker images | grep windowsservercore | grep -v $BUILD | wc -l)
-if [ $LATESTCOUNT -ne 1 ]; then
-	set -e
-	docker tag $IMAGENAME:$BUILD windowsservercore:latest
-	echo "INFO: Tagged $IMAGENAME:$BUILD as windowsservercore:latest"
-fi
-
-# Busybox (requires windowsservercore)
-if [ -z "$(docker images | grep busybox)" ]; then
-	echo "INFO: Building busybox"
-	docker build -t busybox https://raw.githubusercontent.com/jhowardmsft/busybox/master/Dockerfile
-fi

hack/make/.ensure-httpserver

@@ -1,15 +0,0 @@
-#!/bin/bash
-set -e
-
-# Build a Go static web server on top of busybox image
-# and compile it for target daemon
-
-dir="$DEST/httpserver"
-mkdir -p "$dir"
-(
-	cd "$dir"
-	GOOS=${DOCKER_ENGINE_GOOS:="linux"} GOARCH=${DOCKER_ENGINE_GOARCH:="amd64"} CGO_ENABLED=0 go build -o httpserver github.com/docker/docker/contrib/httpserver
-	cp ../../../../contrib/httpserver/Dockerfile .
-	docker build -qt httpserver . > /dev/null
-)
-rm -rf "$dir"

hack/make/.ensure-nnp-test

@@ -1,22 +0,0 @@
-#!/bin/bash
-set -e
-
-# Build a C binary for testing no-new-privileges
-# and compile it for target daemon
-if [ "$DOCKER_ENGINE_GOOS" = "linux" ]; then
-	if [ "$DOCKER_ENGINE_OSARCH" = "$DOCKER_CLIENT_OSARCH" ]; then
-		tmpdir=$(mktemp -d)
-		gcc -g -Wall -static contrib/nnp-test/nnp-test.c -o "${tmpdir}/nnp-test"
-
-		dockerfile="${tmpdir}/Dockerfile"
-		cat <<-EOF > "$dockerfile"
-		FROM debian:jessie
-		COPY . /usr/bin/
-		RUN chmod +s /usr/bin/nnp-test
-		EOF
-		docker build --force-rm ${DOCKER_BUILD_ARGS} -qt nnp-test "${tmpdir}" > /dev/null
-		rm -rf "${tmpdir}"
-	else
-		docker build ${DOCKER_BUILD_ARGS} -qt nnp-test contrib/nnp-test > /dev/null
-	fi
-fi

hack/make/.ensure-syscall-test

@@ -1,26 +0,0 @@
-#!/bin/bash
-set -e
-
-# Build a C binary for cloning a userns for seccomp tests
-# and compile it for target daemon
-if [ "$DOCKER_ENGINE_GOOS" = "linux" ]; then
-	if [ "$DOCKER_ENGINE_OSARCH" = "$DOCKER_CLIENT_OSARCH" ]; then
-		tmpdir=$(mktemp -d)
-		gcc -g -Wall -static contrib/syscall-test/userns.c -o "${tmpdir}/userns-test"
-		gcc -g -Wall -static contrib/syscall-test/ns.c -o "${tmpdir}/ns-test"
-		gcc -g -Wall -static contrib/syscall-test/acct.c -o "${tmpdir}/acct-test"
-		if [ "$DOCKER_ENGINE_OSARCH" = "linux/amd64" ]; then
-			gcc -s -m32 -nostdlib contrib/syscall-test/exit32.s -o "${tmpdir}/exit32-test"
-		fi
-
-		dockerfile="${tmpdir}/Dockerfile"
-		cat <<-EOF > "$dockerfile"
-		FROM debian:jessie
-		COPY . /usr/bin/
-		EOF
-		docker build --force-rm ${DOCKER_BUILD_ARGS} -qt syscall-test "${tmpdir}" > /dev/null
-		rm -rf "${tmpdir}"
-	else
-		docker build ${DOCKER_BUILD_ARGS} -qt syscall-test contrib/syscall-test > /dev/null
-	fi
-fi

hack/make/.integration-daemon-setup

@@ -4,11 +4,4 @@ set -e
 bundle .detect-daemon-osarch
 if [ $DOCKER_ENGINE_GOOS != "windows" ]; then
 	bundle .ensure-emptyfs
-	bundle .ensure-frozen-images
-	bundle .ensure-httpserver
-	bundle .ensure-syscall-test
-	bundle .ensure-nnp-test
-else
-	# Note this is Windows to Windows CI, not Windows to Linux CI
-	bundle .ensure-frozen-images-windows
 fi

hack/make/.integration-test-helpers

@@ -67,11 +67,13 @@ test_env() {
 			DOCKER_HOST="$DOCKER_HOST" \
 			DOCKER_REMAP_ROOT="$DOCKER_REMAP_ROOT" \
 			DOCKER_REMOTE_DAEMON="$DOCKER_REMOTE_DAEMON" \
+			DOCKERFILE="$DOCKERFILE" \
 			GOPATH="$GOPATH" \
 			GOTRACEBACK=all \
 			HOME="$ABS_DEST/fake-HOME" \
 			PATH="$PATH" \
 			TEMP="$TEMP" \
+			TEST_IMAGE_NAMESPACE="$TEST_IMAGE_NAMESPACE" \
 			"$@"
 	)
 }

hack/make/validate-test

@@ -17,7 +17,10 @@ for f in "${files[@]}"; do
 
 	# we use "git show" here to validate that what's committed doesn't contain golang built-in testing
 	if git show "$VALIDATE_HEAD:$f" | grep -q testing.T; then
-		badFiles+=( "$f" )
+		if [ "$(echo $f | grep '_test')" ]; then
+			# allow testing.T for non- _test files
+			badFiles+=( "$f" )
+		fi
 	fi
 done
 

integration-cli/check_test.go

@@ -23,6 +23,9 @@ func Test(t *testing.T) {
 		fmt.Println("INFO: Testing against a local daemon")
 	}
 
+	if daemonPlatform == "linux" {
+		ensureFrozenImagesLinux(t)
+	}
 	check.TestingT(t)
 }
 

integration-cli/daemon.go

@@ -371,8 +371,9 @@ func (d *Daemon) LoadBusybox() error {
 			return fmt.Errorf("unexpected error on busybox.tar stat: %v", err)
 		}
 		// saving busybox image from main daemon
-		if err := exec.Command(dockerBinary, "save", "--output", bb, "busybox:latest").Run(); err != nil {
-			return fmt.Errorf("could not save busybox image: %v", err)
+		if out, err := exec.Command(dockerBinary, "save", "--output", bb, "busybox:latest").CombinedOutput(); err != nil {
+			imagesOut, _ := exec.Command(dockerBinary, "images", "--format", "{{ .Repository }}:{{ .Tag }}").CombinedOutput()
+			return fmt.Errorf("could not save busybox image: %s\n%s", string(out), strings.TrimSpace(string(imagesOut)))
 		}
 	}
 	// loading busybox image to this daemon

integration-cli/docker_cli_rmi_test.go

@@ -308,8 +308,13 @@ RUN echo 2 #layer2
 }
 
 func (*DockerSuite) TestRmiParentImageFail(c *check.C) {
-	parent := inspectField(c, "busybox", "Parent")
-	out, _, err := dockerCmdWithError("rmi", parent)
+	_, err := buildImage("test", `
+	FROM busybox
+	RUN echo hello`, false)
+	c.Assert(err, checker.IsNil)
+
+	id := inspectField(c, "busybox", "ID")
+	out, _, err := dockerCmdWithError("rmi", id)
 	c.Assert(err, check.NotNil)
 	if !strings.Contains(out, "image has dependent child images") {
 		c.Fatalf("rmi should have failed because it's a parent image, got %s", out)

integration-cli/docker_cli_run_unix_test.go

@@ -1019,6 +1019,7 @@ func (s *DockerSuite) TestRunSeccompProfileDenyUnshareUserns(c *check.C) {
 // with a the default seccomp profile exits with operation not permitted.
 func (s *DockerSuite) TestRunSeccompProfileDenyCloneUserns(c *check.C) {
 	testRequires(c, SameHostDaemon, seccompEnabled)
+	ensureSyscallTest(c)
 
 	runCmd := exec.Command(dockerBinary, "run", "syscall-test", "userns-test", "id")
 	out, _, err := runCommandWithOutput(runCmd)
@@ -1031,6 +1032,7 @@ func (s *DockerSuite) TestRunSeccompProfileDenyCloneUserns(c *check.C) {
 // 'docker run --security-opt seccomp=unconfined syscall-test' allows creating a userns.
 func (s *DockerSuite) TestRunSeccompUnconfinedCloneUserns(c *check.C) {
 	testRequires(c, SameHostDaemon, seccompEnabled, UserNamespaceInKernel, NotUserNamespace, unprivilegedUsernsClone)
+	ensureSyscallTest(c)
 
 	// make sure running w privileged is ok
 	runCmd := exec.Command(dockerBinary, "run", "--security-opt", "seccomp=unconfined", "syscall-test", "userns-test", "id")
@@ -1043,6 +1045,7 @@ func (s *DockerSuite) TestRunSeccompUnconfinedCloneUserns(c *check.C) {
 // allows creating a userns.
 func (s *DockerSuite) TestRunSeccompAllowPrivCloneUserns(c *check.C) {
 	testRequires(c, SameHostDaemon, seccompEnabled, UserNamespaceInKernel, NotUserNamespace)
+	ensureSyscallTest(c)
 
 	// make sure running w privileged is ok
 	runCmd := exec.Command(dockerBinary, "run", "--privileged", "syscall-test", "userns-test", "id")
@@ -1055,6 +1058,7 @@ func (s *DockerSuite) TestRunSeccompAllowPrivCloneUserns(c *check.C) {
 // with the default seccomp profile.
 func (s *DockerSuite) TestRunSeccompProfileAllow32Bit(c *check.C) {
 	testRequires(c, SameHostDaemon, seccompEnabled, IsAmd64)
+	ensureSyscallTest(c)
 
 	runCmd := exec.Command(dockerBinary, "run", "syscall-test", "exit32-test", "id")
 	if out, _, err := runCommandWithOutput(runCmd); err != nil {
@@ -1075,6 +1079,7 @@ func (s *DockerSuite) TestRunSeccompAllowSetrlimit(c *check.C) {
 
 func (s *DockerSuite) TestRunSeccompDefaultProfileAcct(c *check.C) {
 	testRequires(c, SameHostDaemon, seccompEnabled, NotUserNamespace)
+	ensureSyscallTest(c)
 
 	out, _, err := dockerCmdWithError("run", "syscall-test", "acct-test")
 	if err == nil || !strings.Contains(out, "Operation not permitted") {
@@ -1104,6 +1109,7 @@ func (s *DockerSuite) TestRunSeccompDefaultProfileAcct(c *check.C) {
 
 func (s *DockerSuite) TestRunSeccompDefaultProfileNS(c *check.C) {
 	testRequires(c, SameHostDaemon, seccompEnabled, NotUserNamespace)
+	ensureSyscallTest(c)
 
 	out, _, err := dockerCmdWithError("run", "syscall-test", "ns-test", "echo", "hello0")
 	if err == nil || !strings.Contains(out, "Operation not permitted") {
@@ -1140,6 +1146,7 @@ func (s *DockerSuite) TestRunSeccompDefaultProfileNS(c *check.C) {
 // effective uid transtions on executing setuid binaries.
 func (s *DockerSuite) TestRunNoNewPrivSetuid(c *check.C) {
 	testRequires(c, DaemonIsLinux, NotUserNamespace, SameHostDaemon)
+	ensureNNPTest(c)
 
 	// test that running a setuid binary results in no effective uid transition
 	runCmd := exec.Command(dockerBinary, "run", "--security-opt", "no-new-privileges", "--user", "1000", "nnp-test", "/usr/bin/nnp-test")

integration-cli/docker_utils.go

@@ -754,6 +754,10 @@ func newRemoteFileServer(ctx *FakeContext) (*remoteFileServer, error) {
 		container = fmt.Sprintf("fileserver-cnt-%s", strings.ToLower(stringutils.GenerateRandomAlphaOnlyString(10)))
 	)
 
+	if err := ensureHTTPServerImage(); err != nil {
+		return nil, err
+	}
+
 	// Build the image
 	if err := fakeContextAddDockerfile(ctx, `FROM httpserver
 COPY . /static`); err != nil {

integration-cli/fixtures.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"sync"
+)
+
+var ensureHTTPServerOnce sync.Once
+
+func ensureHTTPServerImage() error {
+	var doIt bool
+	ensureHTTPServerOnce.Do(func() {
+		doIt = true
+	})
+
+	if !doIt {
+		return nil
+	}
+
+	protectedImages["httpserver:latest"] = struct{}{}
+
+	tmp, err := ioutil.TempDir("", "docker-http-server-test")
+	if err != nil {
+		return fmt.Errorf("could not build http server: %v", err)
+	}
+	defer os.RemoveAll(tmp)
+
+	goos := daemonPlatform
+	if goos == "" {
+		goos = "linux"
+	}
+	goarch := os.Getenv("DOCKER_ENGINE_GOARCH")
+	if goarch == "" {
+		goarch = "amd64"
+	}
+
+	goCmd, lookErr := exec.LookPath("go")
+	if lookErr != nil {
+		return fmt.Errorf("could not build http server: %v", lookErr)
+	}
+
+	cmd := exec.Command(goCmd, "build", "-o", filepath.Join(tmp, "httpserver"), "github.com/docker/docker/contrib/httpserver")
+	cmd.Env = append(os.Environ(), []string{
+		"CGO_ENABLED=0",
+		"GOOS=" + goos,
+		"GOARCH=" + goarch,
+	}...)
+	var out []byte
+	if out, err = cmd.CombinedOutput(); err != nil {
+		return fmt.Errorf("could not build http server: %s", string(out))
+	}
+
+	cpCmd, lookErr := exec.LookPath("cp")
+	if lookErr != nil {
+		return fmt.Errorf("could not build http server: %v", lookErr)
+	}
+	if out, err = exec.Command(cpCmd, "../contrib/httpserver/Dockerfile", filepath.Join(tmp, "Dockerfile")).CombinedOutput(); err != nil {
+		return fmt.Errorf("could not build http server: %v", string(out))
+	}
+
+	if out, err = exec.Command(dockerBinary, "build", "-q", "-t", "httpserver", tmp).CombinedOutput(); err != nil {
+		return fmt.Errorf("could not build http server: %v", string(out))
+	}
+	return nil
+}

integration-cli/fixtures/load/frozen.go

@@ -0,0 +1,171 @@
+package load
+
+import (
+	"bufio"
+	"bytes"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/pkg/errors"
+)
+
+var frozenImgDir = "/docker-frozen-images"
+
+// FrozenImagesLinux loads the frozen image set for the integration suite
+// If the images are not available locally it will download them
+// TODO: This loads whatever is in the frozen image dir, regardless of what
+// images were passed in. If the images need to be downloaded, then it will respect
+// the passed in images
+func FrozenImagesLinux(dockerBinary string, images ...string) error {
+	imgNS := os.Getenv("TEST_IMAGE_NAMESPACE")
+	var loadImages []string
+	for _, img := range images {
+		if imgNS != "" {
+			img = imgNS + "/" + img
+		}
+		if err := exec.Command(dockerBinary, "inspect", "--type=image", img).Run(); err != nil {
+			loadImages = append(loadImages, img)
+		}
+	}
+
+	if len(loadImages) == 0 {
+		// everything is loaded, we're done
+		return nil
+	}
+
+	fi, err := os.Stat(frozenImgDir)
+	if err != nil || !fi.IsDir() {
+		if err := pullImages(dockerBinary, loadImages); err != nil {
+			return errors.Wrap(err, "error pulling image list")
+		}
+	} else {
+		if err := loadFrozenImags(dockerBinary); err != nil {
+			return err
+		}
+	}
+
+	if imgNS != "" {
+		for _, img := range loadImages {
+			target := strings.TrimPrefix(img, imgNS+"/")
+			if target != img {
+				if out, err := exec.Command(dockerBinary, "tag", img, target).CombinedOutput(); err != nil {
+					return errors.Errorf("%v: %s", err, string(out))
+				}
+				if out, err := exec.Command(dockerBinary, "rmi", img).CombinedOutput(); err != nil {
+					return errors.Errorf("%v: %s", err, string(out))
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func loadFrozenImags(dockerBinary string) error {
+	tar, err := exec.LookPath("tar")
+	if err != nil {
+		return errors.Wrap(err, "could not find tar binary")
+	}
+	tarCmd := exec.Command(tar, "-cC", frozenImgDir, ".")
+	out, err := tarCmd.StdoutPipe()
+	if err != nil {
+		return errors.Wrap(err, "error getting stdout pipe for tar command")
+	}
+
+	errBuf := bytes.NewBuffer(nil)
+	tarCmd.Stderr = errBuf
+	tarCmd.Start()
+	defer tarCmd.Wait()
+
+	cmd := exec.Command(dockerBinary, "load")
+	cmd.Stdin = out
+	if out, err := cmd.CombinedOutput(); err != nil {
+		return errors.Errorf("%v: %s", err, string(out))
+	}
+	return nil
+}
+
+func pullImages(dockerBinary string, images []string) error {
+	cwd, err := os.Getwd()
+	if err != nil {
+		return errors.Wrap(err, "error getting path to dockerfile")
+	}
+	dockerfile := os.Getenv("DOCKERFILE")
+	if dockerfile == "" {
+		dockerfile = "Dockerfile"
+	}
+	dockerfilePath := filepath.Join(filepath.Dir(filepath.Clean(cwd)), dockerfile)
+	pullRefs, err := readFrozenImageList(dockerfilePath, images)
+	if err != nil {
+		return errors.Wrap(err, "error reading frozen image list")
+	}
+
+	var wg sync.WaitGroup
+	chErr := make(chan error, len(images))
+	for tag, ref := range pullRefs {
+		wg.Add(1)
+		go func(tag, ref string) {
+			defer wg.Done()
+			if out, err := exec.Command(dockerBinary, "pull", ref).CombinedOutput(); err != nil {
+				chErr <- errors.Errorf("%v: %s", string(out), err)
+				return
+			}
+			if out, err := exec.Command(dockerBinary, "tag", ref, tag).CombinedOutput(); err != nil {
+				chErr <- errors.Errorf("%v: %s", string(out), err)
+				return
+			}
+			if out, err := exec.Command(dockerBinary, "rmi", ref).CombinedOutput(); err != nil {
+				chErr <- errors.Errorf("%v: %s", string(out), err)
+				return
+			}
+		}(tag, ref)
+	}
+	wg.Wait()
+	close(chErr)
+	return <-chErr
+}
+
+func readFrozenImageList(dockerfilePath string, images []string) (map[string]string, error) {
+	f, err := os.Open(dockerfilePath)
+	if err != nil {
+		return nil, errors.Wrap(err, "error reading dockerfile")
+	}
+	defer f.Close()
+	ls := make(map[string]string)
+
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := strings.Fields(scanner.Text())
+		if len(line) < 3 {
+			continue
+		}
+		if !(line[0] == "RUN" && line[1] == "./contrib/download-frozen-image-v2.sh") {
+			continue
+		}
+
+		frozenImgDir = line[2]
+		if line[2] == frozenImgDir {
+			frozenImgDir = filepath.Join(os.Getenv("DEST"), "frozen-images")
+		}
+
+		for scanner.Scan() {
+			img := strings.TrimSpace(scanner.Text())
+			img = strings.TrimSuffix(img, "\\")
+			img = strings.TrimSpace(img)
+			split := strings.Split(img, "@")
+			if len(split) < 2 {
+				break
+			}
+
+			for _, i := range images {
+				if split[0] == i {
+					ls[i] = img
+					break
+				}
+			}
+		}
+	}
+	return ls, nil
+}

integration-cli/fixtures_linux_daemon.go

@@ -0,0 +1,160 @@
+package main
+
+import (
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/docker/docker/integration-cli/fixtures/load"
+	"github.com/docker/docker/pkg/integration/checker"
+	"github.com/go-check/check"
+)
+
+func ensureFrozenImagesLinux(t *testing.T) {
+	images := []string{"busybox:latest", "hello-world:latest", "debian:jessie"}
+	err := load.FrozenImagesLinux(dockerBinary, images...)
+	if err != nil {
+		t.Log(dockerCmdWithError("images"))
+		t.Fatalf("%+v", err)
+	}
+
+	// hello-world:latest gets re-tagged as hello-world:frozen
+	// there are some tests that use hello-world:latest specifically so it pulls
+	// the image and hello-world:frozen is used for when we just want a super
+	// small image
+	if out, err := exec.Command(dockerBinary, "tag", "hello-world:latest", "hello-world:frozen").CombinedOutput(); err != nil {
+		t.Log(dockerCmdWithError("images"))
+		t.Fatal(string(out))
+	}
+	if out, err := exec.Command(dockerBinary, "rmi", "hello-world:latest").CombinedOutput(); err != nil {
+		t.Log(dockerCmdWithError("images"))
+		t.Fatal(string(out))
+	}
+
+	for _, img := range images {
+		if img == "hello-world:latest" {
+			img = "hello-world:frozen"
+		}
+		protectedImages[img] = struct{}{}
+	}
+}
+
+var ensureSyscallTestOnce sync.Once
+
+func ensureSyscallTest(c *check.C) {
+	var doIt bool
+	ensureSyscallTestOnce.Do(func() {
+		doIt = true
+	})
+	if !doIt {
+		return
+	}
+	protectedImages["syscall-test:latest"] = struct{}{}
+
+	// if no match, must build in docker, which is significantly slower
+	// (slower mostly because of the vfs graphdriver)
+	if daemonPlatform != runtime.GOOS {
+		ensureSyscallTestBuild(c)
+		return
+	}
+
+	tmp, err := ioutil.TempDir("", "syscall-test-build")
+	c.Assert(err, checker.IsNil, check.Commentf("couldn't create temp dir"))
+	defer os.RemoveAll(tmp)
+
+	gcc, err := exec.LookPath("gcc")
+	c.Assert(err, checker.IsNil, check.Commentf("could not find gcc"))
+
+	out, err := exec.Command(gcc, "-g", "-Wall", "-static", "../contrib/syscall-test/userns.c", "-o", tmp+"/"+"userns-test").CombinedOutput()
+	c.Assert(err, checker.IsNil, check.Commentf(string(out)))
+	out, err = exec.Command(gcc, "-g", "-Wall", "-static", "../contrib/syscall-test/ns.c", "-o", tmp+"/"+"ns-test").CombinedOutput()
+	c.Assert(err, checker.IsNil, check.Commentf(string(out)))
+	out, err = exec.Command(gcc, "-g", "-Wall", "-static", "../contrib/syscall-test/acct.c", "-o", tmp+"/"+"acct-test").CombinedOutput()
+	c.Assert(err, checker.IsNil, check.Commentf(string(out)))
+
+	if runtime.GOOS == "linux" && runtime.GOARCH == "amd64" {
+		out, err = exec.Command(gcc, "-s", "-m32", "-nostdlib", "../contrib/syscall-test/exit32.s", "-o", tmp+"/"+"exit32-test").CombinedOutput()
+		c.Assert(err, checker.IsNil, check.Commentf(string(out)))
+	}
+
+	dockerFile := filepath.Join(tmp, "Dockerfile")
+	content := []byte(`
+	FROM debian:jessie
+	COPY . /usr/bin/
+	`)
+	err = ioutil.WriteFile(dockerFile, content, 600)
+	c.Assert(err, checker.IsNil)
+
+	var buildArgs []string
+	if arg := os.Getenv("DOCKER_BUILD_ARGS"); strings.TrimSpace(arg) != "" {
+		buildArgs = strings.Split(arg, " ")
+	}
+	buildArgs = append(buildArgs, []string{"-q", "-t", "syscall-test", tmp}...)
+	buildArgs = append([]string{"build"}, buildArgs...)
+	dockerCmd(c, buildArgs...)
+}
+
+func ensureSyscallTestBuild(c *check.C) {
+	err := load.FrozenImagesLinux(dockerBinary, "buildpack-deps:jessie")
+	c.Assert(err, checker.IsNil)
+
+	var buildArgs []string
+	if arg := os.Getenv("DOCKER_BUILD_ARGS"); strings.TrimSpace(arg) != "" {
+		buildArgs = strings.Split(arg, " ")
+	}
+	buildArgs = append(buildArgs, []string{"-q", "-t", "syscall-test", "../contrib/syscall-test"}...)
+	buildArgs = append([]string{"build"}, buildArgs...)
+	dockerCmd(c, buildArgs...)
+}
+
+func ensureNNPTest(c *check.C) {
+	protectedImages["nnp-test:latest"] = struct{}{}
+	if daemonPlatform != runtime.GOOS {
+		ensureNNPTestBuild(c)
+		return
+	}
+
+	tmp, err := ioutil.TempDir("", "docker-nnp-test")
+	c.Assert(err, checker.IsNil)
+
+	gcc, err := exec.LookPath("gcc")
+	c.Assert(err, checker.IsNil, check.Commentf("could not find gcc"))
+
+	out, err := exec.Command(gcc, "-g", "-Wall", "-static", "../contrib/nnp-test/nnp-test.c", "-o", filepath.Join(tmp, "nnp-test")).CombinedOutput()
+	c.Assert(err, checker.IsNil, check.Commentf(string(out)))
+
+	dockerfile := filepath.Join(tmp, "Dockerfile")
+	content := `
+	FROM debian:jessie
+	COPY . /usr/bin
+	RUN chmod +s /usr/bin/nnp-test
+	`
+	err = ioutil.WriteFile(dockerfile, []byte(content), 600)
+	c.Assert(err, checker.IsNil, check.Commentf("could not write Dockerfile for nnp-test image"))
+
+	var buildArgs []string
+	if arg := os.Getenv("DOCKER_BUILD_ARGS"); strings.TrimSpace(arg) != "" {
+		buildArgs = strings.Split(arg, " ")
+	}
+	buildArgs = append(buildArgs, []string{"-q", "-t", "nnp-test", tmp}...)
+	buildArgs = append([]string{"build"}, buildArgs...)
+	dockerCmd(c, buildArgs...)
+}
+
+func ensureNNPTestBuild(c *check.C) {
+	err := load.FrozenImagesLinux(dockerBinary, "buildpack-deps:jessie")
+	c.Assert(err, checker.IsNil)
+
+	var buildArgs []string
+	if arg := os.Getenv("DOCKER_BUILD_ARGS"); strings.TrimSpace(arg) != "" {
+		buildArgs = strings.Split(arg, " ")
+	}
+	buildArgs = append(buildArgs, []string{"-q", "-t", "npp-test", "../contrib/nnp-test"}...)
+	buildArgs = append([]string{"build"}, buildArgs...)
+	dockerCmd(c, buildArgs...)
+}