integ-cli: Implement remote FakeStorage server for build via URL tests

Implemented a FakeStorage alternative that supports spinning
up a remote container on DOCKER_TEST_HOST to serve files over
an offline-compiled Go static web server image so that tests which
use URLs in Dockerfile can build them over at the daemon side.

`fakeStorage` function now automatically chooses if it should
use a local httptest.Server or a remote container.

This fixes the following tests when running against a remote
daemon:

- `TestBuildCacheADD`
- `TestBuildCopyWildcardNoFind`
- `TestBuildCopyWildcardCache`
- `TestBuildADDRemoteFileWithCache`
- `TestBuildADDRemoteFileWithoutCache`
- `TestBuildADDRemoteFileMTime`
- `TestBuildADDLocalAndRemoteFilesWithCache`
- `TestBuildADDLocalAndRemoteFilesWithoutCache`
- `TestBuildFromURLWithF`
- `TestBuildApiDockerFileRemote`

Signed-off-by: Ahmet Alp Balkan <ahmetalpbalkan@gmail.com>

contrib/httpserver/Dockerfile

@@ -0,0 +1,4 @@
+FROM busybox
+EXPOSE 80/tcp
+COPY httpserver .
+CMD ["./httpserver"]

contrib/httpserver/server.go

@@ -0,0 +1,12 @@
+package main
+
+import (
+	"log"
+	"net/http"
+)
+
+func main() {
+	fs := http.FileServer(http.Dir("/static"))
+	http.Handle("/", fs)
+	log.Panic(http.ListenAndServe(":80", nil))
+}

integration-cli/docker_api_containers_test.go

@@ -364,7 +364,7 @@ RUN find /tmp/`,
 	}
 	defer server.Close()
 
-	buf, err := sockRequestRaw("POST", "/build?dockerfile=baz&remote="+server.URL+"/testD", nil, "application/json")
+	buf, err := sockRequestRaw("POST", "/build?dockerfile=baz&remote="+server.URL()+"/testD", nil, "application/json")
 	if err != nil {
 		t.Fatalf("Build failed: %s", err)
 	}

integration-cli/docker_cli_build_test.go

@@ -8,14 +8,12 @@ import (
 	"io/ioutil"
 	"os"
 	"os/exec"
-	"path"
 	"path/filepath"
 	"reflect"
 	"regexp"
 	"runtime"
 	"strconv"
 	"strings"
-	"syscall"
 	"testing"
 	"text/template"
 	"time"
@@ -645,9 +643,10 @@ func TestBuildCacheADD(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer server.Close()
+
 	if _, err := buildImage(name,
 		fmt.Sprintf(`FROM scratch
-		ADD %s/robots.txt /`, server.URL),
+		ADD %s/robots.txt /`, server.URL()),
 		true); err != nil {
 		t.Fatal(err)
 	}
@@ -657,7 +656,7 @@ func TestBuildCacheADD(t *testing.T) {
 	deleteImages(name)
 	_, out, err := buildImageWithOut(name,
 		fmt.Sprintf(`FROM scratch
-		ADD %s/index.html /`, server.URL),
+		ADD %s/index.html /`, server.URL()),
 		true)
 	if err != nil {
 		t.Fatal(err)
@@ -797,7 +796,7 @@ RUN [ $(ls -l /exists/test_file4 | awk '{print $3":"$4}') = 'root:root' ]
 RUN [ $(ls -l /exists/robots.txt | awk '{print $3":"$4}') = 'root:root' ]
 
 RUN [ $(ls -l /exists/exists_file | awk '{print $3":"$4}') = 'dockerio:dockerio' ]
-`, server.URL),
+`, server.URL()),
 		map[string]string{
 			"test_file1": "test1",
 			"test_file2": "test2",
@@ -1084,6 +1083,7 @@ func TestBuildCopyWildcard(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer server.Close()
+
 	ctx, err := fakeContext(fmt.Sprintf(`FROM busybox
 	COPY file*.txt /tmp/
 	RUN ls /tmp/file1.txt /tmp/file2.txt
@@ -1093,7 +1093,7 @@ func TestBuildCopyWildcard(t *testing.T) {
 	RUN mkdir /tmp2
         ADD dir/*dir %s/robots.txt /tmp2/
 	RUN ls /tmp2/nest_nest_file /tmp2/robots.txt
-	`, server.URL),
+	`, server.URL()),
 		map[string]string{
 			"file1.txt":                     "test1",
 			"file2.txt":                     "test2",
@@ -2831,10 +2831,11 @@ func TestBuildADDRemoteFileWithCache(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer server.Close()
+
 	id1, err := buildImage(name,
 		fmt.Sprintf(`FROM scratch
         MAINTAINER dockerio
-        ADD %s/baz /usr/lib/baz/quux`, server.URL),
+        ADD %s/baz /usr/lib/baz/quux`, server.URL()),
 		true)
 	if err != nil {
 		t.Fatal(err)
@@ -2842,7 +2843,7 @@ func TestBuildADDRemoteFileWithCache(t *testing.T) {
 	id2, err := buildImage(name,
 		fmt.Sprintf(`FROM scratch
         MAINTAINER dockerio
-        ADD %s/baz /usr/lib/baz/quux`, server.URL),
+        ADD %s/baz /usr/lib/baz/quux`, server.URL()),
 		true)
 	if err != nil {
 		t.Fatal(err)
@@ -2864,10 +2865,11 @@ func TestBuildADDRemoteFileWithoutCache(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer server.Close()
+
 	id1, err := buildImage(name,
 		fmt.Sprintf(`FROM scratch
         MAINTAINER dockerio
-        ADD %s/baz /usr/lib/baz/quux`, server.URL),
+        ADD %s/baz /usr/lib/baz/quux`, server.URL()),
 		true)
 	if err != nil {
 		t.Fatal(err)
@@ -2875,7 +2877,7 @@ func TestBuildADDRemoteFileWithoutCache(t *testing.T) {
 	id2, err := buildImage(name2,
 		fmt.Sprintf(`FROM scratch
         MAINTAINER dockerio
-        ADD %s/baz /usr/lib/baz/quux`, server.URL),
+        ADD %s/baz /usr/lib/baz/quux`, server.URL()),
 		false)
 	if err != nil {
 		t.Fatal(err)
@@ -2894,7 +2896,8 @@ func TestBuildADDRemoteFileMTime(t *testing.T) {
 
 	defer deleteImages(name, name2, name3, name4)
 
-	server, err := fakeStorage(map[string]string{"baz": "hello"})
+	files := map[string]string{"baz": "hello"}
+	server, err := fakeStorage(files)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -2902,7 +2905,7 @@ func TestBuildADDRemoteFileMTime(t *testing.T) {
 
 	ctx, err := fakeContext(fmt.Sprintf(`FROM scratch
         MAINTAINER dockerio
-        ADD %s/baz /usr/lib/baz/quux`, server.URL), nil)
+        ADD %s/baz /usr/lib/baz/quux`, server.URL()), nil)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -2921,15 +2924,26 @@ func TestBuildADDRemoteFileMTime(t *testing.T) {
 		t.Fatal("The cache should have been used but wasn't - #1")
 	}
 
-	// Now set baz's times to anything else and redo the build
+	// Now create a different server withsame contents (causes different mtim)
 	// This time the cache should not be used
-	bazPath := path.Join(server.FakeContext.Dir, "baz")
-	err = syscall.UtimesNano(bazPath, make([]syscall.Timespec, 2))
+
+	// allow some time for clock to pass as mtime precision is only 1s
+	time.Sleep(2 * time.Second)
+
+	server2, err := fakeStorage(files)
 	if err != nil {
-		t.Fatalf("Error setting mtime on %q: %v", bazPath, err)
+		t.Fatal(err)
 	}
+	defer server2.Close()
 
-	id3, err := buildImageFromContext(name3, ctx, true)
+	ctx2, err := fakeContext(fmt.Sprintf(`FROM scratch
+        MAINTAINER dockerio
+        ADD %s/baz /usr/lib/baz/quux`, server2.URL()), nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer ctx2.Close()
+	id3, err := buildImageFromContext(name3, ctx2, true)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -2938,7 +2952,7 @@ func TestBuildADDRemoteFileMTime(t *testing.T) {
 	}
 
 	// And for good measure do it again and make sure cache is used this time
-	id4, err := buildImageFromContext(name4, ctx, true)
+	id4, err := buildImageFromContext(name4, ctx2, true)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -2958,10 +2972,11 @@ func TestBuildADDLocalAndRemoteFilesWithCache(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer server.Close()
+
 	ctx, err := fakeContext(fmt.Sprintf(`FROM scratch
         MAINTAINER dockerio
         ADD foo /usr/lib/bla/bar
-        ADD %s/baz /usr/lib/baz/quux`, server.URL),
+        ADD %s/baz /usr/lib/baz/quux`, server.URL()),
 		map[string]string{
 			"foo": "hello world",
 		})
@@ -3047,10 +3062,11 @@ func TestBuildADDLocalAndRemoteFilesWithoutCache(t *testing.T) {
 		t.Fatal(err)
 	}
 	defer server.Close()
+
 	ctx, err := fakeContext(fmt.Sprintf(`FROM scratch
         MAINTAINER dockerio
         ADD foo /usr/lib/bla/bar
-        ADD %s/baz /usr/lib/baz/quux`, server.URL),
+        ADD %s/baz /usr/lib/baz/quux`, server.URL()),
 		map[string]string{
 			"foo": "hello world",
 		})
@@ -4773,7 +4789,7 @@ RUN echo from Dockerfile`,
 
 	// Make sure that -f is ignored and that we don't use the Dockerfile
 	// that's in the current dir
-	out, _, err := dockerCmdInDir(t, ctx.Dir, "build", "-f", "baz", "-t", "test1", server.URL+"/baz")
+	out, _, err := dockerCmdInDir(t, ctx.Dir, "build", "-f", "baz", "-t", "test1", server.URL()+"/baz")
 	if err != nil {
 		t.Fatalf("Failed to build: %s\n%s", out, err)
 	}

integration-cli/docker_utils.go

@@ -581,17 +581,42 @@ func fakeContext(dockerfile string, files map[string]string) (*FakeContext, erro
 	return ctx, nil
 }
 
-type FakeStorage struct {
+// FakeStorage is a static file server. It might be running locally or remotely
+// on test host.
+type FakeStorage interface {
+	Close() error
+	URL() string
+	CtxDir() string
+}
+
+// fakeStorage returns either a local or remote (at daemon machine) file server
+func fakeStorage(files map[string]string) (FakeStorage, error) {
+	if isLocalDaemon {
+		return newLocalFakeStorage(files)
+	}
+	return newRemoteFileServer(files)
+}
+
+// localFileStorage is a file storage on the running machine
+type localFileStorage struct {
 	*FakeContext
 	*httptest.Server
 }
 
-func (f *FakeStorage) Close() error {
-	f.Server.Close()
-	return f.FakeContext.Close()
+func (s *localFileStorage) URL() string {
+	return s.Server.URL
+}
+
+func (s *localFileStorage) CtxDir() string {
+	return s.FakeContext.Dir
+}
+
+func (s *localFileStorage) Close() error {
+	defer s.Server.Close()
+	return s.FakeContext.Close()
 }
 
-func fakeStorage(files map[string]string) (*FakeStorage, error) {
+func newLocalFakeStorage(files map[string]string) (*localFileStorage, error) {
 	tmp, err := ioutil.TempDir("", "fake-storage")
 	if err != nil {
 		return nil, err
@@ -605,42 +630,101 @@ func fakeStorage(files map[string]string) (*FakeStorage, error) {
 	}
 	handler := http.FileServer(http.Dir(ctx.Dir))
 	server := httptest.NewServer(handler)
-	return &FakeStorage{
+	return &localFileStorage{
 		FakeContext: ctx,
 		Server:      server,
 	}, nil
 }
 
-func inspectField(name, field string) (string, error) {
-	format := fmt.Sprintf("{{.%s}}", field)
-	inspectCmd := exec.Command(dockerBinary, "inspect", "-f", format, name)
-	out, exitCode, err := runCommandWithOutput(inspectCmd)
-	if err != nil || exitCode != 0 {
-		return "", fmt.Errorf("failed to inspect %s: %s", name, out)
+// remoteFileServer is a containerized static file server started on the remote
+// testing machine to be used in URL-accepting docker build functionality.
+type remoteFileServer struct {
+	host      string // hostname/port web server is listening to on docker host e.g. 0.0.0.0:43712
+	container string
+	image     string
+	ctx       *FakeContext
+}
+
+func (f *remoteFileServer) URL() string {
+	u := url.URL{
+		Scheme: "http",
+		Host:   f.host}
+	return u.String()
+}
+
+func (f *remoteFileServer) CtxDir() string {
+	return f.ctx.Dir
+}
+
+func (f *remoteFileServer) Close() error {
+	defer func() {
+		if f.ctx != nil {
+			f.ctx.Close()
+		}
+		if f.image != "" {
+			deleteImages(f.image)
+		}
+	}()
+	if f.container == "" {
+		return nil
 	}
-	return strings.TrimSpace(out), nil
+	return deleteContainer(f.container)
 }
 
-func inspectFieldJSON(name, field string) (string, error) {
-	format := fmt.Sprintf("{{json .%s}}", field)
-	inspectCmd := exec.Command(dockerBinary, "inspect", "-f", format, name)
-	out, exitCode, err := runCommandWithOutput(inspectCmd)
-	if err != nil || exitCode != 0 {
-		return "", fmt.Errorf("failed to inspect %s: %s", name, out)
+func newRemoteFileServer(files map[string]string) (*remoteFileServer, error) {
+	var (
+		image     = fmt.Sprintf("fileserver-img-%s", strings.ToLower(makeRandomString(10)))
+		container = fmt.Sprintf("fileserver-cnt-%s", strings.ToLower(makeRandomString(10)))
+	)
+
+	// Build the image
+	ctx, err := fakeContext(`FROM httpserver
+COPY . /static`, files)
+	if _, err := buildImageFromContext(image, ctx, false); err != nil {
+		return nil, fmt.Errorf("failed building file storage container image: %v", err)
 	}
-	return strings.TrimSpace(out), nil
+
+	// Start the container
+	runCmd := exec.Command(dockerBinary, "run", "-d", "-P", "--name", container, image)
+	if out, ec, err := runCommandWithOutput(runCmd); err != nil {
+		return nil, fmt.Errorf("failed to start file storage container. ec=%v\nout=%s\nerr=%v", ec, out, err)
+	}
+
+	// Find out the system assigned port
+	out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "port", container, "80/tcp"))
+	if err != nil {
+		return nil, fmt.Errorf("failed to find container port: err=%v\nout=%s", err, out)
+	}
+
+	return &remoteFileServer{
+		container: container,
+		image:     image,
+		host:      strings.Trim(out, "\n"),
+		ctx:       ctx}, nil
 }
 
-func inspectFieldMap(name, path, field string) (string, error) {
-	format := fmt.Sprintf("{{index .%s %q}}", path, field)
+func inspectFilter(name, filter string) (string, error) {
+	format := fmt.Sprintf("{{%s}}", filter)
 	inspectCmd := exec.Command(dockerBinary, "inspect", "-f", format, name)
 	out, exitCode, err := runCommandWithOutput(inspectCmd)
 	if err != nil || exitCode != 0 {
-		return "", fmt.Errorf("failed to inspect %s: %s", name, out)
+		return "", fmt.Errorf("failed to inspect container %s: %s", name, out)
 	}
 	return strings.TrimSpace(out), nil
 }
 
+func inspectField(name, field string) (string, error) {
+	return inspectFilter(name, fmt.Sprintf(".%s", field))
+}
+
+func inspectFieldJSON(name, field string) (string, error) {
+	return inspectFilter(name, fmt.Sprintf("json .%s", field))
+}
+
+func inspectFieldMap(name, path, field string) (string, error) {
+	return inspectFilter(name, fmt.Sprintf("index .%s %q", path, field))
+}
+
 func getIDByName(name string) (string, error) {
 	return inspectField(name, "Id")
 }

project/make/.ensure-httpserver

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+
+# Build a Go static web server on top of busybox image
+# and compile it for target daemon
+
+dir="$DEST/httpserver"
+mkdir -p "$dir"
+(
+	cd "$dir"
+	GOOS=linux GOARCH=amd64 go build -o httpserver github.com/docker/docker/contrib/httpserver
+	cp ../../../../contrib/httpserver/Dockerfile .
+	docker build -qt httpserver . > /dev/null
+)
+rm -rf "$dir"

project/make/test-integration-cli

@@ -17,6 +17,7 @@ bundle_test_integration_cli() {
 	didFail=
 	if ! {
 		source "$(dirname "$BASH_SOURCE")/.ensure-busybox"
+		source "$(dirname "$BASH_SOURCE")/.ensure-httpserver"
 		source "$(dirname "$BASH_SOURCE")/.ensure-emptyfs"
 
 		bundle_test_integration_cli