testing infrastructure, issue #1800: Refactor docker testing using Docker in Docker

hack/infrastructure/docker-ci/buildbot/master.cfg

@@ -17,15 +17,12 @@ PORT_GITHUB = 8011      # Buildbot github hook port
 PORT_MASTER = 9989      # Port where buildbot master listen buildworkers
 TEST_USER = 'buildbot'  # Credential to authenticate build triggers
 TEST_PWD = 'docker'     # Credential to authenticate build triggers
-BUILDER_NAME = 'docker'
 GITHUB_DOCKER = 'github.com/dotcloud/docker'
 BUILDBOT_PATH = '/data/buildbot'
 DOCKER_PATH = '/go/src/github.com/dotcloud/docker'
 DOCKER_CI_PATH = '/docker-ci'
-BUILDER_PATH = '/data/buildbot/slave/{0}/build'.format(BUILDER_NAME)
-PULL_REQUEST_PATH = '/data/buildbot/slave/pullrequest/build'
 
-# Credentials set by setup.sh and Vagrantfile
+# Credentials set by setup.sh from deployment.py
 BUILDBOT_PWD = ''
 IRC_PWD = ''
 IRC_CHANNEL = ''
@@ -45,34 +42,35 @@ c['slavePortnum'] = PORT_MASTER
 
 
 # Schedulers
-c['schedulers'] = [ForceScheduler(name='trigger', builderNames=[BUILDER_NAME,
+c['schedulers'] = [ForceScheduler(name='trigger', builderNames=['docker',
     'index','registry','coverage','nightlyrelease'])]
-c['schedulers'] += [SingleBranchScheduler(name="all",
+c['schedulers'] += [SingleBranchScheduler(name="all", treeStableTimer=None,
-    change_filter=filter.ChangeFilter(branch='master'), treeStableTimer=None,
+    change_filter=filter.ChangeFilter(branch='master',
-    builderNames=[BUILDER_NAME])]
+    repository='https://github.com/dotcloud/docker'), builderNames=['docker'])]
 c['schedulers'] += [SingleBranchScheduler(name='pullrequest',
     change_filter=filter.ChangeFilter(category='github_pullrequest'), treeStableTimer=None,
     builderNames=['pullrequest'])]
-c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['nightlyrelease'],
+c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['nightlyrelease',
-    hour=7, minute=00)]
+    'coverage'], hour=7, minute=00)]
 c['schedulers'] += [Nightly(name='every4hrs', branch=None, builderNames=['registry','index'],
     hour=range(0,24,4), minute=15)]
 
 # Builders
 # Docker commit test
+test_cmd = ('docker run -privileged mzdaniel/test_docker hack/dind'
+    ' test_docker.sh %(src::revision)s')
 factory = BuildFactory()
 factory.addStep(ShellCommand(description='Docker', logEnviron=False,
-    usePTY=True, command=['sh', '-c', Interpolate(
+    usePTY=True, command=["sh", "-c", Interpolate(test_cmd)]))
-    '{0}/docker-test/test_docker.sh %(src::revision)s'.format(DOCKER_CI_PATH))]))
 c['builders'] = [BuilderConfig(name='docker',slavenames=['buildworker'],
     factory=factory)]
 
 # Docker pull request test
+test_cmd = ('docker run -privileged mzdaniel/test_docker hack/dind'
+    ' test_docker.sh %(src::revision)s %(src::repository)s %(src::branch)s')
 factory = BuildFactory()
 factory.addStep(ShellCommand(description='pull_request', logEnviron=False,
-    usePTY=True, command=['sh', '-c', Interpolate(
+    usePTY=True, command=["sh", "-c", Interpolate(test_cmd)]))
-    '{0}/docker-test/test_docker.sh %(src::revision)s %(src::repository)s'
-    ' %(src::branch)s'.format(DOCKER_CI_PATH))]))
 c['builders'] += [BuilderConfig(name='pullrequest',slavenames=['buildworker'],
     factory=factory)]
 
@@ -97,17 +95,16 @@ c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'],
 factory = BuildFactory()
 factory.addStep(ShellCommand(description='index', logEnviron=False,
     command='. {0}/master/credentials.cfg; '
-    '{1}/testing/functionaltests/test_index.py'.format(BUILDBOT_PATH,
+    '/docker-ci/functionaltests/test_index.py'.format(BUILDBOT_PATH),
-    DOCKER_PATH), usePTY=True))
+    usePTY=True))
 c['builders'] += [BuilderConfig(name='index',slavenames=['buildworker'],
     factory=factory)]
 
 # Docker nightly release
-nightlyrelease_cmd = ('docker run -i -t -privileged -lxc-conf=lxc.aa_profile=unconfined'
-    ' -e AWS_S3_BUCKET=test.docker.io dockerbuilder')
 factory = BuildFactory()
-factory.addStep(ShellCommand(description='NightlyRelease',logEnviron=False,usePTY=True,
+factory.addStep(ShellCommand(description='NightlyRelease', logEnviron=False,
-    command=nightlyrelease_cmd))
+    usePTY=True, command='docker run -privileged'
+    ' -e AWS_S3_BUCKET=test.docker.io dockerbuilder'))
 c['builders'] += [BuilderConfig(name='nightlyrelease',slavenames=['buildworker'],
     factory=factory)]
 

hack/infrastructure/docker-ci/deployment.py

@@ -140,11 +140,17 @@ sudo('stop docker')
 sudo('wget -q -O /usr/bin/docker http://test.docker.io/test/docker')
 sudo('start docker')
 
+# Preventively reboot docker-ci daily
+sudo('ln -s /sbin/reboot /etc/cron.daily')
+
 # Build docker-ci containers
 sudo('cd {}; docker build -t docker .'.format(DOCKER_PATH))
 sudo('cd {}/nightlyrelease; docker build -t dockerbuilder .'.format(
     DOCKER_CI_PATH))
 
+# Download docker-ci testing container
+sudo('docker pull mzdaniel/test_docker')
+
 # Setup buildbot
 sudo('mkdir /data')
 sudo('{0}/setup.sh root {0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}'

hack/infrastructure/docker-ci/docker-test/Dockerfile

@@ -0,0 +1,30 @@
+# VERSION:        0.3
+# DOCKER-VERSION  0.6.3
+# AUTHOR:         Daniel Mizyrycki <daniel@dotcloud.com>
+# DESCRIPTION:    Testing docker PRs and commits on top of master using
+# REFERENCES:     This code reuses the excellent implementation of
+#                 Docker in Docker made by Jerome Petazzoni.
+#                 https://github.com/jpetazzo/dind
+# COMMENTS:
+#   This Dockerfile adapts /Dockerfile to enable docker PRs and commits testing
+#   Optional arguments:
+#       [commit]  (default: 'HEAD')
+#       [repo]    (default: 'http://github.com/dotcloud/docker')
+#       [branch]  (default: 'master')
+# TO_BUILD:       docker build -t test_docker .
+# TO_RUN:         docker run -privileged test_docker hack/dind test_docker.sh [commit] [repo] [branch]
+
+from docker
+maintainer Daniel Mizyrycki <daniel@dotcloud.com>
+
+# Setup go environment. Extracted from /Dockerfile
+env     CGO_ENABLED 0
+env     GOROOT  /goroot
+env     PATH    $PATH:/goroot/bin
+env     GOPATH  /go:/go/src/github.com/dotcloud/docker/vendor
+volume  /var/lib/docker
+workdir /go/src/github.com/dotcloud/docker
+
+# Add test_docker.sh
+add test_docker.sh /usr/bin/test_docker.sh
+run chmod +x /usr/bin/test_docker.sh

hack/infrastructure/docker-ci/docker-test/test_docker.sh

@@ -5,17 +5,11 @@ COMMIT=${1-HEAD}
 REPO=${2-http://github.com/dotcloud/docker}
 BRANCH=${3-master}
 
-# Generate a random string of $1 characters
-function random {
-    cat /dev/urandom | tr -cd 'a-f0-9' | head -c $1
-}
-
 # Compute test paths
-BASE_PATH=`pwd`/test_docker_$(random 12)
+DOCKER_PATH=/go/src/github.com/dotcloud/docker
-DOCKER_PATH=$BASE_PATH/go/src/github.com/dotcloud/docker
-export GOPATH=$BASE_PATH/go:$DOCKER_PATH/vendor
 
 # Fetch latest master
+rm -rf /go
 mkdir -p $DOCKER_PATH
 cd $DOCKER_PATH
 git init .
@@ -23,12 +17,21 @@ git fetch -q http://github.com/dotcloud/docker master
 git reset --hard FETCH_HEAD
 
 # Merge commit
+#echo FIXME. Temporarily skip TestPrivilegedCanMount until DinD works reliable on AWS
+git pull -q https://github.com/mzdaniel/docker.git dind-aws || exit 1
+
+# Merge commit in top of master
 git fetch -q "$REPO" "$BRANCH"
 git merge --no-edit $COMMIT || exit 1
 
 # Test commit
 go test -v; exit_status=$?
 
+# Display load if test fails
+if [ $exit_status -eq 1 ] ; then
+    uptime; echo; free
+fi
+
 # Cleanup testing directory
 rm -rf $BASE_PATH
 

hack/infrastructure/docker-ci/nightlyrelease/dockerbuild

@@ -13,9 +13,8 @@ cd /
 git clone -q http://github.com/dotcloud/docker /go/src/github.com/dotcloud/docker
 cd /go/src/github.com/dotcloud/docker
 
-echo FIXME. Temporarily add Jerome changeset with proper apparmor handling
+echo FIXME. Temporarily skip TestPrivilegedCanMount until DinD works reliable on AWS
-git fetch  http://github.com/jpetazzo/docker escape-apparmor-confinement:escape-apparmor-confinement
+git pull -q https://github.com/mzdaniel/docker.git dind-aws || exit 1
-git rebase --onto master master escape-apparmor-confinement
 
 # Launch docker daemon using dind inside the container
 ./hack/dind /usr/bin/docker -d &
@@ -28,7 +27,17 @@ date > timestamp
 docker build -t docker .
 
 # Run Docker unittests binary and Ubuntu package
-docker run -privileged -lxc-conf=lxc.aa_profile=unconfined docker hack/make.sh || exit 1
+docker run -privileged -lxc-conf=lxc.aa_profile=unconfined docker hack/make.sh
+exit_status=$?
+
+# Display load if test fails
+if [ $exit_status -eq 1 ] ; then
+    uptime; echo; free
+    exit 1
+fi
+
+# Commit binary and ubuntu bundles for release
+docker commit -run '{"Env": ["PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin"], "WorkingDir": "/go/src/github.com/dotcloud/docker"}' $(docker ps -l -q) release
 
 # Turn debug off to load credentials from the environment
 set +x
@@ -36,7 +45,6 @@ eval $(cat /root/release_credentials.json  | python -c '
 import sys,json,base64;
 d=json.loads(base64.b64decode(sys.stdin.read()));
 exec("""for k in d: print "export {0}=\\"{1}\\"".format(k,d[k])""")')
-echo '{"https://index.docker.io/v1/":{"auth":"'$INDEX_AUTH'","email":"engineering@dotcloud.com"}}' > /.dockercfg
 set -x
 
 # Push docker nightly