Merge pull request #4505 from unclejack/cli_integration_tests

cli integration tests

Makefile

@@ -1,4 +1,4 @@
-.PHONY: all binary build cross default docs docs-build docs-shell shell test test-integration
+.PHONY: all binary build cross default docs docs-build docs-shell shell test test-integration test-integration-cli
 
 GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD)
 DOCKER_IMAGE := docker:$(GIT_BRANCH)
@@ -23,11 +23,14 @@ docs-shell: docs-build
 	docker run --rm -i -t -p 8000:8000 "$(DOCKER_DOCS_IMAGE)" bash
 
 test: build
-	$(DOCKER_RUN_DOCKER) hack/make.sh test test-integration
+	$(DOCKER_RUN_DOCKER) hack/make.sh binary test test-integration test-integration-cli
 
 test-integration: build
 	$(DOCKER_RUN_DOCKER) hack/make.sh test-integration
 
+test-integration-cli: build
+	$(DOCKER_RUN_DOCKER) hack/make.sh binary test-integration-cli
+
 shell: build
 	$(DOCKER_RUN_DOCKER) bash
 

hack/make.sh

@@ -139,6 +139,7 @@ find_dirs() {
 		\( \
 			-wholename './vendor' \
 			-o -wholename './integration' \
+			-o -wholename './integration-cli' \
 			-o -wholename './contrib' \
 			-o -wholename './pkg/mflag/example' \
 			-o -wholename './.git' \

hack/make/binary

@@ -11,5 +11,6 @@ go build \
 	" \
 	./docker
 echo "Created binary: $DEST/docker-$VERSION"
+ln -sf "docker-$VERSION" "$DEST/docker"
 
 hash_files "$DEST/docker-$VERSION"

hack/make/test-integration-cli

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+DEST=$1
+
+set -e
+
+# subshell so that we can export PATH without breaking other things
+(
+export PATH="$DEST/../binary:$DEST/../dynbinary:$PATH"
+
+bundle_test_integration_cli() {
+	go_test_dir ./integration-cli
+}
+
+if ! command -v docker &> /dev/null; then
+	echo >&2 'error: binary or dynbinary must be run before test-integration-cli'
+	false
+fi
+
+docker -d -D -p $DEST/docker.pid &> $DEST/docker.log &
+
+bundle_test_integration_cli 2>&1 \
+	| tee $DEST/test.log
+
+DOCKERD_PID=$(cat $DEST/docker.pid)
+kill $DOCKERD_PID
+wait $DOCKERD_PID
+)

integration-cli/build_tests/TestBuildSixtySteps/Dockerfile

@@ -0,0 +1,60 @@
+FROM busybox
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"
+RUN echo "foo"

integration-cli/docker_cli_build_test.go

@@ -0,0 +1,28 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"path/filepath"
+	"testing"
+)
+
+func TestBuildSixtySteps(t *testing.T) {
+	buildDirectory := filepath.Join(workingDirectory, "build_tests", "TestBuildSixtySteps")
+	buildCmd := exec.Command(dockerBinary, "build", "-t", "foobuildsixtysteps", ".")
+	buildCmd.Dir = buildDirectory
+	out, exitCode, err := runCommandWithOutput(buildCmd)
+	errorOut(err, t, fmt.Sprintf("build failed to complete: %v %v", out, err))
+
+	if err != nil || exitCode != 0 {
+		t.Fatal("failed to build the image")
+	}
+
+	go deleteImages("foobuildsixtysteps")
+
+	logDone("build - build an image with sixty build steps")
+}
+
+// TODO: TestCaching
+
+// TODO: TestADDCacheInvalidation

integration-cli/docker_cli_commit_test.go

@@ -0,0 +1,34 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"testing"
+)
+
+func TestCommitAfterContainerIsDone(t *testing.T) {
+	runCmd := exec.Command(dockerBinary, "run", "-i", "-a", "stdin", "busybox", "echo", "foo")
+	out, _, _, err := runCommandWithStdoutStderr(runCmd)
+	errorOut(err, t, fmt.Sprintf("failed to run container: %v %v", out, err))
+
+	cleanedContainerID := stripTrailingCharacters(out)
+
+	waitCmd := exec.Command(dockerBinary, "wait", cleanedContainerID)
+	_, _, err = runCommandWithOutput(waitCmd)
+	errorOut(err, t, fmt.Sprintf("error thrown while waiting for container: %s", out))
+
+	commitCmd := exec.Command(dockerBinary, "commit", cleanedContainerID)
+	out, _, err = runCommandWithOutput(commitCmd)
+	errorOut(err, t, fmt.Sprintf("failed to commit container to image: %v %v", out, err))
+
+	cleanedImageID := stripTrailingCharacters(out)
+
+	inspectCmd := exec.Command(dockerBinary, "inspect", cleanedImageID)
+	out, _, err = runCommandWithOutput(inspectCmd)
+	errorOut(err, t, fmt.Sprintf("failed to inspect image: %v %v", out, err))
+
+	go deleteContainer(cleanedContainerID)
+	go deleteImages(cleanedImageID)
+
+	logDone("commit - echo foo and commit the image")
+}

integration-cli/docker_cli_diff_test.go

@@ -0,0 +1,66 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+// ensure that an added file shows up in docker diff
+func TestDiffFilenameShownInOutput(t *testing.T) {
+	containerCmd := `echo foo > /root/bar`
+	runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "sh", "-c", containerCmd)
+	cid, _, err := runCommandWithOutput(runCmd)
+	errorOut(err, t, fmt.Sprintf("failed to start the container: %v", err))
+
+	cleanCID := stripTrailingCharacters(cid)
+
+	diffCmd := exec.Command(dockerBinary, "diff", cleanCID)
+	out, _, err := runCommandWithOutput(diffCmd)
+	errorOut(err, t, fmt.Sprintf("failed to run diff: %v %v", out, err))
+
+	found := false
+	for _, line := range strings.Split(out, "\n") {
+		if strings.Contains("A /root/bar", line) {
+			found = true
+			break
+		}
+	}
+	if !found {
+		t.Errorf("couldn't find the new file in docker diff's output: %v", out)
+	}
+	go deleteContainer(cleanCID)
+
+	logDone("diff - check if created file shows up")
+}
+
+// test to ensure GH #3840 doesn't occur any more
+func TestDiffEnsureDockerinitFilesAreIgnored(t *testing.T) {
+	// this is a list of files which shouldn't show up in `docker diff`
+	dockerinitFiles := []string{"/etc/resolv.conf", "/etc/hostname", "/etc/hosts", "/.dockerinit", "/.dockerenv"}
+
+	// we might not run into this problem from the first run, so start a few containers
+	for i := 0; i < 20; i++ {
+		containerCmd := `echo foo > /root/bar`
+		runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "sh", "-c", containerCmd)
+		cid, _, err := runCommandWithOutput(runCmd)
+		errorOut(err, t, fmt.Sprintf("%s", err))
+
+		cleanCID := stripTrailingCharacters(cid)
+
+		diffCmd := exec.Command(dockerBinary, "diff", cleanCID)
+		out, _, err := runCommandWithOutput(diffCmd)
+		errorOut(err, t, fmt.Sprintf("failed to run diff: %v %v", out, err))
+
+		go deleteContainer(cleanCID)
+
+		for _, filename := range dockerinitFiles {
+			if strings.Contains(out, filename) {
+				t.Errorf("found file which should've been ignored %v in diff output", filename)
+			}
+		}
+	}
+
+	logDone("diff - check if ignored files show up in diff")
+}

integration-cli/docker_cli_export_import_test.go

@@ -0,0 +1,50 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"testing"
+)
+
+// export an image and try to import it into a new one
+func TestExportContainerAndImportImage(t *testing.T) {
+	runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "true")
+	out, _, err := runCommandWithOutput(runCmd)
+	if err != nil {
+		t.Fatal("failed to create a container", out, err)
+	}
+
+	cleanedContainerID := stripTrailingCharacters(out)
+
+	inspectCmd := exec.Command(dockerBinary, "inspect", cleanedContainerID)
+	out, _, err = runCommandWithOutput(inspectCmd)
+	if err != nil {
+		t.Fatalf("output should've been a container id: %s %s ", cleanedContainerID, err)
+	}
+
+	exportCmdTemplate := `%v export %v > /tmp/testexp.tar`
+	exportCmdFinal := fmt.Sprintf(exportCmdTemplate, dockerBinary, cleanedContainerID)
+	exportCmd := exec.Command("bash", "-c", exportCmdFinal)
+	out, _, err = runCommandWithOutput(exportCmd)
+	errorOut(err, t, fmt.Sprintf("failed to export container: %v %v", out, err))
+
+	importCmdFinal := `cat /tmp/testexp.tar | docker import - testexp`
+	importCmd := exec.Command("bash", "-c", importCmdFinal)
+	out, _, err = runCommandWithOutput(importCmd)
+	errorOut(err, t, fmt.Sprintf("failed to import image: %v %v", out, err))
+
+	cleanedImageID := stripTrailingCharacters(out)
+
+	inspectCmd = exec.Command(dockerBinary, "inspect", cleanedImageID)
+	out, _, err = runCommandWithOutput(inspectCmd)
+	errorOut(err, t, fmt.Sprintf("output should've been an image id: %v %v", out, err))
+
+	go deleteImages("testexp")
+	go deleteContainer(cleanedContainerID)
+
+	os.Remove("/tmp/testexp.tar")
+
+	logDone("export - export a container")
+	logDone("import - import an image")
+}

integration-cli/docker_cli_images_test.go

@@ -0,0 +1,20 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+func TestImagesEnsureImageIsListed(t *testing.T) {
+	imagesCmd := exec.Command(dockerBinary, "images")
+	out, _, err := runCommandWithOutput(imagesCmd)
+	errorOut(err, t, fmt.Sprintf("listing images failed with errors: %v", err))
+
+	if !strings.Contains(out, "busybox") {
+		t.Fatal("images should've listed busybox")
+	}
+
+	logDone("images - busybox should be listed")
+}

integration-cli/docker_cli_info_test.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+// ensure docker info succeeds
+func TestInfoEnsureSucceeds(t *testing.T) {
+	versionCmd := exec.Command(dockerBinary, "info")
+	out, exitCode, err := runCommandWithOutput(versionCmd)
+	errorOut(err, t, fmt.Sprintf("encountered error while running docker info: %v", err))
+
+	if err != nil || exitCode != 0 {
+		t.Fatal("failed to execute docker info")
+	}
+
+	stringsToCheck := []string{"Containers:", "Execution Driver:", "Kernel Version:"}
+
+	for _, linePrefix := range stringsToCheck {
+		if !strings.Contains(out, linePrefix) {
+			t.Errorf("couldn't find string %v in output", linePrefix)
+		}
+	}
+
+	logDone("info - verify that it works")
+}

integration-cli/docker_cli_kill_test.go

@@ -0,0 +1,36 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+func TestKillContainer(t *testing.T) {
+	runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "sh", "-c", "sleep 10")
+	out, _, err := runCommandWithOutput(runCmd)
+	errorOut(err, t, fmt.Sprintf("run failed with errors: %v", err))
+
+	cleanedContainerID := stripTrailingCharacters(out)
+
+	inspectCmd := exec.Command(dockerBinary, "inspect", cleanedContainerID)
+	inspectOut, _, err := runCommandWithOutput(inspectCmd)
+	errorOut(err, t, fmt.Sprintf("out should've been a container id: %v %v", inspectOut, err))
+
+	killCmd := exec.Command(dockerBinary, "kill", cleanedContainerID)
+	out, _, err = runCommandWithOutput(killCmd)
+	errorOut(err, t, fmt.Sprintf("failed to kill container: %v %v", out, err))
+
+	listRunningContainersCmd := exec.Command(dockerBinary, "ps", "-q")
+	out, _, err = runCommandWithOutput(listRunningContainersCmd)
+	errorOut(err, t, fmt.Sprintf("failed to list running containers: %v", err))
+
+	if strings.Contains(out, cleanedContainerID) {
+		t.Fatal("killed container is still running")
+	}
+
+	go deleteContainer(cleanedContainerID)
+
+	logDone("kill - kill container running sleep 10")
+}

integration-cli/docker_cli_pull_test.go

@@ -0,0 +1,30 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"testing"
+)
+
+// pulling an image from the central registry should work
+func TestPullImageFromCentralRegistry(t *testing.T) {
+	pullCmd := exec.Command(dockerBinary, "pull", "busybox")
+	out, exitCode, err := runCommandWithOutput(pullCmd)
+	errorOut(err, t, fmt.Sprintf("%s %s", out, err))
+
+	if err != nil || exitCode != 0 {
+		t.Fatal("pulling the busybox image from the registry has failed")
+	}
+	logDone("pull - pull busybox")
+}
+
+// pulling a non-existing image from the central registry should return a non-zero exit code
+func TestPullNonExistingImage(t *testing.T) {
+	pullCmd := exec.Command(dockerBinary, "pull", "fooblahblah1234")
+	_, exitCode, err := runCommandWithOutput(pullCmd)
+
+	if err == nil || exitCode == 0 {
+		t.Fatal("expected non-zero exit status when pulling non-existing image")
+	}
+	logDone("pull - pull fooblahblah1234 (non-existing image)")
+}

integration-cli/docker_cli_push_test.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"testing"
+)
+
+// these tests need a freshly started empty private docker registry
+
+// pulling an image from the central registry should work
+func TestPushBusyboxImage(t *testing.T) {
+	// skip this test until we're able to use a registry
+	t.Skip()
+	// tag the image to upload it tot he private registry
+	repoName := fmt.Sprintf("%v/busybox", privateRegistryURL)
+	tagCmd := exec.Command(dockerBinary, "tag", "busybox", repoName)
+	out, exitCode, err := runCommandWithOutput(tagCmd)
+	errorOut(err, t, fmt.Sprintf("%v %v", out, err))
+
+	if err != nil || exitCode != 0 {
+		t.Fatal("image tagging failed")
+	}
+
+	pushCmd := exec.Command(dockerBinary, "push", repoName)
+	out, exitCode, err = runCommandWithOutput(pushCmd)
+	errorOut(err, t, fmt.Sprintf("%v %v", out, err))
+
+	go deleteImages(repoName)
+
+	if err != nil || exitCode != 0 {
+		t.Fatal("pushing the image to the private registry has failed")
+	}
+	logDone("push - push busybox to private registry")
+}
+
+// pushing an image without a prefix should throw an error
+func TestPushUnprefixedRepo(t *testing.T) {
+	// skip this test until we're able to use a registry
+	t.Skip()
+	pushCmd := exec.Command(dockerBinary, "push", "busybox")
+	_, exitCode, err := runCommandWithOutput(pushCmd)
+
+	if err == nil || exitCode == 0 {
+		t.Fatal("pushing an unprefixed repo didn't result in a non-zero exit status")
+	}
+	logDone("push - push unprefixed busybox repo --> must fail")
+}

integration-cli/docker_cli_run_test.go

@@ -0,0 +1,255 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+// "test123" should be printed by docker run
+func TestDockerRunEchoStdout(t *testing.T) {
+	runCmd := exec.Command(dockerBinary, "run", "busybox", "echo", "test123")
+	out, _, _, err := runCommandWithStdoutStderr(runCmd)
+	errorOut(err, t, out)
+
+	if out != "test123\n" {
+		t.Errorf("container should've printed 'test123'")
+	}
+
+	deleteAllContainers()
+
+	logDone("run - echo test123")
+}
+
+// "test" should be printed
+func TestDockerRunEchoStdoutWithMemoryLimit(t *testing.T) {
+	runCmd := exec.Command(dockerBinary, "run", "-m", "2786432", "busybox", "echo", "test")
+	out, _, _, err := runCommandWithStdoutStderr(runCmd)
+	errorOut(err, t, out)
+
+	if out != "test\n" {
+		t.Errorf("container should've printed 'test'")
+
+	}
+
+	deleteAllContainers()
+
+	logDone("run - echo with memory limit")
+}
+
+// "test" should be printed
+func TestDockerRunEchoStdoutWitCPULimit(t *testing.T) {
+	runCmd := exec.Command(dockerBinary, "run", "-c", "1000", "busybox", "echo", "test")
+	out, _, _, err := runCommandWithStdoutStderr(runCmd)
+	errorOut(err, t, out)
+
+	if out != "test\n" {
+		t.Errorf("container should've printed 'test'")
+	}
+
+	deleteAllContainers()
+
+	logDone("run - echo with CPU limit")
+}
+
+// "test" should be printed
+func TestDockerRunEchoStdoutWithCPUAndMemoryLimit(t *testing.T) {
+	runCmd := exec.Command(dockerBinary, "run", "-c", "1000", "-m", "2786432", "busybox", "echo", "test")
+	out, _, _, err := runCommandWithStdoutStderr(runCmd)
+	errorOut(err, t, out)
+
+	if out != "test\n" {
+		t.Errorf("container should've printed 'test'")
+	}
+
+	deleteAllContainers()
+
+	logDone("run - echo with CPU and memory limit")
+}
+
+// "test" should be printed
+func TestDockerRunEchoNamedContainer(t *testing.T) {
+	runCmd := exec.Command(dockerBinary, "run", "--name", "testfoonamedcontainer", "busybox", "echo", "test")
+	out, _, _, err := runCommandWithStdoutStderr(runCmd)
+	errorOut(err, t, out)
+
+	if out != "test\n" {
+		t.Errorf("container should've printed 'test'")
+	}
+
+	if err := deleteContainer("testfoonamedcontainer"); err != nil {
+		t.Errorf("failed to remove the named container: %v", err)
+	}
+
+	deleteAllContainers()
+
+	logDone("run - echo with named container")
+}
+
+// it should be possible to ping Google DNS resolver
+// this will fail when Internet access is unavailable
+func TestDockerRunPingGoogle(t *testing.T) {
+	runCmd := exec.Command(dockerBinary, "run", "busybox", "ping", "-c", "1", "8.8.8.8")
+	out, _, _, err := runCommandWithStdoutStderr(runCmd)
+	errorOut(err, t, out)
+
+	errorOut(err, t, "container should've been able to ping 8.8.8.8")
+
+	deleteAllContainers()
+
+	logDone("run - ping 8.8.8.8")
+}
+
+// the exit code should be 0
+// some versions of lxc might make this test fail
+func TestDockerRunExitCodeZero(t *testing.T) {
+	runCmd := exec.Command(dockerBinary, "run", "busybox", "true")
+	exitCode, err := runCommand(runCmd)
+	errorOut(err, t, fmt.Sprintf("%s", err))
+
+	if exitCode != 0 {
+		t.Errorf("container should've exited with exit code 0")
+	}
+
+	deleteAllContainers()
+
+	logDone("run - exit with 0")
+}
+
+// the exit code should be 1
+// some versions of lxc might make this test fail
+func TestDockerRunExitCodeOne(t *testing.T) {
+	runCmd := exec.Command(dockerBinary, "run", "busybox", "false")
+	exitCode, err := runCommand(runCmd)
+	if err != nil && !strings.Contains("exit status 1", fmt.Sprintf("%s", err)) {
+		t.Fatal(err)
+	}
+	if exitCode != 1 {
+		t.Errorf("container should've exited with exit code 1")
+	}
+
+	deleteAllContainers()
+
+	logDone("run - exit with 1")
+}
+
+// it should be possible to pipe in data via stdin to a process running in a container
+// some versions of lxc might make this test fail
+func TestRunStdinPipe(t *testing.T) {
+	runCmd := exec.Command("bash", "-c", `echo "blahblah" | docker run -i -a stdin busybox cat`)
+	out, _, _, err := runCommandWithStdoutStderr(runCmd)
+	errorOut(err, t, out)
+
+	out = stripTrailingCharacters(out)
+
+	inspectCmd := exec.Command(dockerBinary, "inspect", out)
+	inspectOut, _, err := runCommandWithOutput(inspectCmd)
+	errorOut(err, t, fmt.Sprintf("out should've been a container id: %s %s", out, inspectOut))
+
+	waitCmd := exec.Command(dockerBinary, "wait", out)
+	_, _, err = runCommandWithOutput(waitCmd)
+	errorOut(err, t, fmt.Sprintf("error thrown while waiting for container: %s", out))
+
+	logsCmd := exec.Command(dockerBinary, "logs", out)
+	containerLogs, _, err := runCommandWithOutput(logsCmd)
+	errorOut(err, t, fmt.Sprintf("error thrown while trying to get container logs: %s", err))
+
+	containerLogs = stripTrailingCharacters(containerLogs)
+
+	if containerLogs != "blahblah" {
+		t.Errorf("logs didn't print the container's logs %s", containerLogs)
+	}
+
+	rmCmd := exec.Command(dockerBinary, "rm", out)
+	_, _, err = runCommandWithOutput(rmCmd)
+	errorOut(err, t, fmt.Sprintf("rm failed to remove container %s", err))
+
+	deleteAllContainers()
+
+	logDone("run - pipe in with -i -a stdin")
+}
+
+// the container's ID should be printed when starting a container in detached mode
+func TestDockerRunDetachedContainerIDPrinting(t *testing.T) {
+	runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "true")
+	out, _, _, err := runCommandWithStdoutStderr(runCmd)
+	errorOut(err, t, out)
+
+	out = stripTrailingCharacters(out)
+
+	inspectCmd := exec.Command(dockerBinary, "inspect", out)
+	inspectOut, _, err := runCommandWithOutput(inspectCmd)
+	errorOut(err, t, fmt.Sprintf("out should've been a container id: %s %s", out, inspectOut))
+
+	waitCmd := exec.Command(dockerBinary, "wait", out)
+	_, _, err = runCommandWithOutput(waitCmd)
+	errorOut(err, t, fmt.Sprintf("error thrown while waiting for container: %s", out))
+
+	rmCmd := exec.Command(dockerBinary, "rm", out)
+	rmOut, _, err := runCommandWithOutput(rmCmd)
+	errorOut(err, t, "rm failed to remove container")
+
+	rmOut = stripTrailingCharacters(rmOut)
+	if rmOut != out {
+		t.Errorf("rm didn't print the container ID %s %s", out, rmOut)
+	}
+
+	deleteAllContainers()
+
+	logDone("run - print container ID in detached mode")
+}
+
+// the working directory should be set correctly
+func TestDockerRunWorkingDirectory(t *testing.T) {
+	runCmd := exec.Command(dockerBinary, "run", "-w", "/root", "busybox", "pwd")
+	out, _, _, err := runCommandWithStdoutStderr(runCmd)
+	errorOut(err, t, out)
+
+	out = stripTrailingCharacters(out)
+
+	if out != "/root" {
+		t.Errorf("-w failed to set working directory")
+	}
+
+	runCmd = exec.Command(dockerBinary, "run", "--workdir", "/root", "busybox", "pwd")
+	out, _, _, err = runCommandWithStdoutStderr(runCmd)
+	errorOut(err, t, out)
+
+	out = stripTrailingCharacters(out)
+
+	if out != "/root" {
+		t.Errorf("--workdir failed to set working directory")
+	}
+
+	deleteAllContainers()
+
+	logDone("run - run with working directory set by -w")
+	logDone("run - run with working directory set by --workdir")
+}
+
+// pinging Google's DNS resolver should fail when we disable the networking
+func TestDockerRunWithoutNetworking(t *testing.T) {
+	runCmd := exec.Command(dockerBinary, "run", "--networking=false", "busybox", "ping", "-c", "1", "8.8.8.8")
+	out, _, exitCode, err := runCommandWithStdoutStderr(runCmd)
+	if err != nil && exitCode != 1 {
+		t.Fatal(out, err)
+	}
+	if exitCode != 1 {
+		t.Errorf("--networking=false should've disabled the network; the container shouldn't have been able to ping 8.8.8.8")
+	}
+
+	runCmd = exec.Command(dockerBinary, "run", "-n=false", "busybox", "ping", "-c", "1", "8.8.8.8")
+	out, _, exitCode, err = runCommandWithStdoutStderr(runCmd)
+	if err != nil && exitCode != 1 {
+		t.Fatal(out, err)
+	}
+	if exitCode != 1 {
+		t.Errorf("-n=false should've disabled the network; the container shouldn't have been able to ping 8.8.8.8")
+	}
+
+	deleteAllContainers()
+
+	logDone("run - disable networking with --networking=false")
+	logDone("run - disable networking with -n=false")
+}

integration-cli/docker_cli_save_load_test.go

@@ -0,0 +1,52 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"os/exec"
+	"testing"
+)
+
+// save a repo and try to load it
+func TestSaveAndLoadRepo(t *testing.T) {
+	runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "true")
+	out, _, err := runCommandWithOutput(runCmd)
+	errorOut(err, t, fmt.Sprintf("failed to create a container: %v %v", out, err))
+
+	cleanedContainerID := stripTrailingCharacters(out)
+
+	repoName := "foobar-save-load-test"
+
+	inspectCmd := exec.Command(dockerBinary, "inspect", cleanedContainerID)
+	out, _, err = runCommandWithOutput(inspectCmd)
+	errorOut(err, t, fmt.Sprintf("output should've been a container id: %v %v", cleanedContainerID, err))
+
+	commitCmd := exec.Command(dockerBinary, "commit", cleanedContainerID, repoName)
+	out, _, err = runCommandWithOutput(commitCmd)
+	errorOut(err, t, fmt.Sprintf("failed to commit container: %v %v", out, err))
+
+	saveCmdTemplate := `%v save %v > /tmp/foobar-save-load-test.tar`
+	saveCmdFinal := fmt.Sprintf(saveCmdTemplate, dockerBinary, repoName)
+	saveCmd := exec.Command("bash", "-c", saveCmdFinal)
+	out, _, err = runCommandWithOutput(saveCmd)
+	errorOut(err, t, fmt.Sprintf("failed to save repo: %v %v", out, err))
+
+	deleteImages(repoName)
+
+	loadCmdFinal := `cat /tmp/foobar-save-load-test.tar | docker load`
+	loadCmd := exec.Command("bash", "-c", loadCmdFinal)
+	out, _, err = runCommandWithOutput(loadCmd)
+	errorOut(err, t, fmt.Sprintf("failed to load repo: %v %v", out, err))
+
+	inspectCmd = exec.Command(dockerBinary, "inspect", repoName)
+	out, _, err = runCommandWithOutput(inspectCmd)
+	errorOut(err, t, fmt.Sprintf("the repo should exist after loading it: %v %v", out, err))
+
+	go deleteImages(repoName)
+	go deleteContainer(cleanedContainerID)
+
+	os.Remove("/tmp/foobar-save-load-test.tar")
+
+	logDone("save - save a repo")
+	logDone("load - load a repo")
+}

integration-cli/docker_cli_search_test.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+// search for repos named  "registry" on the central registry
+func TestSearchOnCentralRegistry(t *testing.T) {
+	searchCmd := exec.Command(dockerBinary)
+	out, exitCode, err := runCommandWithOutput(searchCmd)
+	errorOut(err, t, fmt.Sprintf("encountered error while searching: %v", err))
+
+	if err != nil || exitCode != 0 {
+		t.Fatal("failed to search on the central registry")
+	}
+
+	if !strings.Contains(out, "registry") {
+		t.Fatal("couldn't find any repository named (or containing) 'registry'")
+	}
+
+	logDone("search - search for repositories named (or containing) 'registry'")
+}

integration-cli/docker_cli_tag_test.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"testing"
+)
+
+// tagging a named image in a new unprefixed repo should work
+func TestTagUnprefixedRepoByName(t *testing.T) {
+	pullCmd := exec.Command(dockerBinary, "pull", "busybox")
+	out, exitCode, err := runCommandWithOutput(pullCmd)
+	errorOut(err, t, fmt.Sprintf("%s %s", out, err))
+
+	if err != nil || exitCode != 0 {
+		t.Fatal("pulling the busybox image from the registry has failed")
+	}
+
+	tagCmd := exec.Command(dockerBinary, "tag", "busybox", "testfoobarbaz")
+	out, _, err = runCommandWithOutput(tagCmd)
+	errorOut(err, t, fmt.Sprintf("%v %v", out, err))
+
+	deleteImages("testfoobarbaz")
+
+	logDone("tag - busybox -> testfoobarbaz")
+}
+
+// tagging an image by ID in a new unprefixed repo should work
+func TestTagUnprefixedRepoByID(t *testing.T) {
+	getIDCmd := exec.Command(dockerBinary, "inspect", "-f", "{{.id}}", "busybox")
+	out, _, err := runCommandWithOutput(getIDCmd)
+	errorOut(err, t, fmt.Sprintf("failed to get the image ID of busybox: %v", err))
+
+	cleanedImageID := stripTrailingCharacters(out)
+	tagCmd := exec.Command(dockerBinary, "tag", cleanedImageID, "testfoobarbaz")
+	out, _, err = runCommandWithOutput(tagCmd)
+	errorOut(err, t, fmt.Sprintf("%s %s", out, err))
+
+	deleteImages("testfoobarbaz")
+
+	logDone("tag - busybox's image ID -> testfoobarbaz")
+}
+
+// ensure we don't allow the use of invalid tags; these tag operations should fail
+func TestTagInvalidUnprefixedRepo(t *testing.T) {
+	// skip this until we start blocking bad tags
+	t.Skip()
+
+	invalidRepos := []string{"-foo", "fo$z$", "Foo@3cc", "Foo$3", "Foo*3", "Fo^3", "Foo!3", "F)xcz(", "fo", "f"}
+
+	for _, repo := range invalidRepos {
+		tagCmd := exec.Command(dockerBinary, "tag", "busybox", repo)
+		_, _, err := runCommandWithOutput(tagCmd)
+		if err == nil {
+			t.Errorf("tag busybox %v should have failed", repo)
+			continue
+		}
+		logMessage := fmt.Sprintf("tag - busybox %v --> must fail", repo)
+		logDone(logMessage)
+	}
+}
+
+// ensure we allow the use of valid tags
+func TestTagValidPrefixedRepo(t *testing.T) {
+	pullCmd := exec.Command(dockerBinary, "pull", "busybox")
+	out, exitCode, err := runCommandWithOutput(pullCmd)
+	errorOut(err, t, fmt.Sprintf("%s %s", out, err))
+
+	if err != nil || exitCode != 0 {
+		t.Fatal("pulling the busybox image from the registry has failed")
+	}
+
+	validRepos := []string{"fooo/bar", "fooaa/test"}
+
+	for _, repo := range validRepos {
+		tagCmd := exec.Command(dockerBinary, "tag", "busybox", repo)
+		_, _, err := runCommandWithOutput(tagCmd)
+		if err != nil {
+			t.Errorf("tag busybox %v should have worked: %s", repo, err)
+			continue
+		}
+		go deleteImages(repo)
+		logMessage := fmt.Sprintf("tag - busybox %v", repo)
+		logDone(logMessage)
+	}
+}

integration-cli/docker_cli_top_test.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+func TestTop(t *testing.T) {
+	runCmd := exec.Command(dockerBinary, "run", "-i", "-d", "busybox", "sleep", "20")
+	out, _, err := runCommandWithOutput(runCmd)
+	errorOut(err, t, fmt.Sprintf("failed to start the container: %v", err))
+
+	cleanedContainerID := stripTrailingCharacters(out)
+
+	topCmd := exec.Command(dockerBinary, "top", cleanedContainerID)
+	out, _, err = runCommandWithOutput(topCmd)
+	errorOut(err, t, fmt.Sprintf("failed to run top: %v %v", out, err))
+
+	killCmd := exec.Command(dockerBinary, "kill", cleanedContainerID)
+	_, err = runCommand(killCmd)
+	errorOut(err, t, fmt.Sprintf("failed to kill container: %v", err))
+
+	go deleteContainer(cleanedContainerID)
+
+	if !strings.Contains(out, "sleep 20") {
+		t.Fatal("top should've listed sleep 20 in the process list")
+	}
+
+	logDone("top - sleep process should be listed")
+}

integration-cli/docker_cli_version_test.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+	"testing"
+)
+
+// ensure docker version works
+func TestVersionEnsureSucceeds(t *testing.T) {
+	versionCmd := exec.Command(dockerBinary, "version")
+	out, exitCode, err := runCommandWithOutput(versionCmd)
+	errorOut(err, t, fmt.Sprintf("encountered error while running docker version: %v", err))
+
+	if err != nil || exitCode != 0 {
+		t.Fatal("failed to execute docker version")
+	}
+
+	stringsToCheck := []string{"Client version:", "Go version (client):", "Git commit (client):", "Server version:", "Git commit (server):", "Go version (server):", "Last stable version:"}
+
+	for _, linePrefix := range stringsToCheck {
+		if !strings.Contains(out, linePrefix) {
+			t.Errorf("couldn't find string %v in output", linePrefix)
+		}
+	}
+
+	logDone("version - verify that it works and that the output is properly formatted")
+}

integration-cli/docker_test_vars.go

@@ -0,0 +1,29 @@
+package main
+
+import (
+	"os"
+)
+
+// the docker binary to use
+var dockerBinary = "docker"
+
+// the private registry image to use for tests involving the registry
+var registryImageName = "registry"
+
+// the private registry to use for tests
+var privateRegistryURL = "127.0.0.1:5000"
+
+var workingDirectory string
+
+func init() {
+	if dockerBin := os.Getenv("DOCKER_BINARY"); dockerBin != "" {
+		dockerBinary = dockerBin
+	}
+	if registryImage := os.Getenv("REGISTRY_IMAGE"); registryImage != "" {
+		registryImageName = registryImage
+	}
+	if registry := os.Getenv("REGISTRY_URL"); registry != "" {
+		privateRegistryURL = registry
+	}
+	workingDirectory, _ = os.Getwd()
+}

integration-cli/docker_utils.go

@@ -0,0 +1,56 @@
+package main
+
+import (
+	"fmt"
+	"os/exec"
+	"strings"
+)
+
+func deleteContainer(container string) error {
+	container = strings.Replace(container, "\n", " ", -1)
+	container = strings.Trim(container, " ")
+	rmArgs := fmt.Sprintf("rm %v", container)
+	rmSplitArgs := strings.Split(rmArgs, " ")
+	rmCmd := exec.Command(dockerBinary, rmSplitArgs...)
+	exitCode, err := runCommand(rmCmd)
+	// set error manually if not set
+	if exitCode != 0 && err == nil {
+		err = fmt.Errorf("failed to remove container: `docker rm` exit is non-zero")
+	}
+
+	return err
+}
+
+func getAllContainers() (string, error) {
+	getContainersCmd := exec.Command(dockerBinary, "ps", "-q", "-a")
+	out, exitCode, err := runCommandWithOutput(getContainersCmd)
+	if exitCode != 0 && err == nil {
+		err = fmt.Errorf("failed to get a list of containers: %v\n", out)
+	}
+
+	return out, err
+}
+
+func deleteAllContainers() error {
+	containers, err := getAllContainers()
+	if err != nil {
+		fmt.Println(containers)
+		return err
+	}
+
+	if err = deleteContainer(containers); err != nil {
+		return err
+	}
+	return nil
+}
+
+func deleteImages(images string) error {
+	rmiCmd := exec.Command(dockerBinary, "rmi", images)
+	exitCode, err := runCommand(rmiCmd)
+	// set error manually if not set
+	if exitCode != 0 && err == nil {
+		err = fmt.Errorf("failed to remove image: `docker rmi` exit is non-zero")
+	}
+
+	return err
+}

integration-cli/utils.go

@@ -0,0 +1,109 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"os/exec"
+	"strings"
+	"syscall"
+	"testing"
+)
+
+func getExitCode(err error) (int, error) {
+	exitCode := 0
+	if exiterr, ok := err.(*exec.ExitError); ok {
+		if procExit := exiterr.Sys().(syscall.WaitStatus); ok {
+			return procExit.ExitStatus(), nil
+		}
+	}
+	return exitCode, fmt.Errorf("failed to get exit code")
+}
+
+func runCommandWithOutput(cmd *exec.Cmd) (output string, exitCode int, err error) {
+	exitCode = 0
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		var exiterr error
+		if exitCode, exiterr = getExitCode(err); exiterr != nil {
+			// TODO: Fix this so we check the error's text.
+			// we've failed to retrieve exit code, so we set it to 127
+			exitCode = 127
+		}
+	}
+	output = string(out)
+	return
+}
+
+func runCommandWithStdoutStderr(cmd *exec.Cmd) (stdout string, stderr string, exitCode int, err error) {
+	exitCode = 0
+	var stderrBuffer bytes.Buffer
+	stderrPipe, err := cmd.StderrPipe()
+	if err != nil {
+		return "", "", -1, err
+	}
+	go io.Copy(&stderrBuffer, stderrPipe)
+	out, err := cmd.Output()
+
+	if err != nil {
+		var exiterr error
+		if exitCode, exiterr = getExitCode(err); exiterr != nil {
+			// TODO: Fix this so we check the error's text.
+			// we've failed to retrieve exit code, so we set it to 127
+			exitCode = 127
+		}
+	}
+	stdout = string(out)
+	stderr = string(stderrBuffer.Bytes())
+	return
+}
+
+func runCommand(cmd *exec.Cmd) (exitCode int, err error) {
+	exitCode = 0
+	err = cmd.Run()
+	if err != nil {
+		var exiterr error
+		if exitCode, exiterr = getExitCode(err); exiterr != nil {
+			// TODO: Fix this so we check the error's text.
+			// we've failed to retrieve exit code, so we set it to 127
+			exitCode = 127
+		}
+	}
+	return
+}
+
+func startCommand(cmd *exec.Cmd) (exitCode int, err error) {
+	exitCode = 0
+	err = cmd.Start()
+	if err != nil {
+		var exiterr error
+		if exitCode, exiterr = getExitCode(err); exiterr != nil {
+			// TODO: Fix this so we check the error's text.
+			// we've failed to retrieve exit code, so we set it to 127
+			exitCode = 127
+		}
+	}
+	return
+}
+
+func logDone(message string) {
+	fmt.Printf("[PASSED]: %s\n", message)
+}
+
+func stripTrailingCharacters(target string) string {
+	target = strings.Trim(target, "\n")
+	target = strings.Trim(target, " ")
+	return target
+}
+
+func errorOut(err error, t *testing.T, message string) {
+	if err != nil {
+		t.Fatal(message)
+	}
+}
+
+func errorOutOnNonNilError(err error, t *testing.T, message string) {
+	if err == nil {
+		t.Fatalf(message)
+	}
+}