diff --git a/hack/infrastructure/docker-ci/Dockerfile b/hack/infrastructure/docker-ci/Dockerfile index 3ac8d90d24..bb49944d1b 100644 --- a/hack/infrastructure/docker-ci/Dockerfile +++ b/hack/infrastructure/docker-ci/Dockerfile @@ -33,8 +33,10 @@ from ubuntu:12.04 -run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list -run apt-get update; apt-get install -y python2.7 python-dev python-pip ssh rsync less vim +run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' \ + > /etc/apt/sources.list +run apt-get update; apt-get install -y git python2.7 python-dev libevent-dev \ + python-pip ssh rsync less vim run pip install boto fabric # Add deployment code and set default container command diff --git a/hack/infrastructure/docker-ci/buildbot/master.cfg b/hack/infrastructure/docker-ci/buildbot/master.cfg index 52bf495df3..9ca5fc035a 100644 --- a/hack/infrastructure/docker-ci/buildbot/master.cfg +++ b/hack/infrastructure/docker-ci/buildbot/master.cfg @@ -43,7 +43,7 @@ c['slavePortnum'] = PORT_MASTER # Schedulers c['schedulers'] = [ForceScheduler(name='trigger', builderNames=['docker', - 'index','registry','coverage','nightlyrelease'])] + 'index','registry','docker-coverage','registry-coverage','nightlyrelease'])] c['schedulers'] += [SingleBranchScheduler(name="all", treeStableTimer=None, change_filter=filter.ChangeFilter(branch='master', repository='https://github.com/dotcloud/docker'), builderNames=['docker'])] @@ -51,7 +51,7 @@ c['schedulers'] += [SingleBranchScheduler(name='pullrequest', change_filter=filter.ChangeFilter(category='github_pullrequest'), treeStableTimer=None, builderNames=['pullrequest'])] c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['nightlyrelease', - 'coverage'], hour=7, minute=00)] + 'docker-coverage','registry-coverage'], hour=7, minute=00)] c['schedulers'] += [Nightly(name='every4hrs', branch=None, builderNames=['registry','index'], hour=range(0,24,4), minute=15)] @@ -76,17 +76,25 @@ c['builders'] += [BuilderConfig(name='pullrequest',slavenames=['buildworker'], # Docker coverage test factory = BuildFactory() -factory.addStep(ShellCommand(description='Coverage', logEnviron=False, +factory.addStep(ShellCommand(description='docker-coverage', logEnviron=False, usePTY=True, command='{0}/docker-coverage/coverage-docker.sh'.format( DOCKER_CI_PATH))) -c['builders'] += [BuilderConfig(name='coverage',slavenames=['buildworker'], +c['builders'] += [BuilderConfig(name='docker-coverage',slavenames=['buildworker'], + factory=factory)] + +# Docker registry coverage test +factory = BuildFactory() +factory.addStep(ShellCommand(description='registry-coverage', logEnviron=False, + usePTY=True, command='docker run registry_coverage'.format( + DOCKER_CI_PATH))) +c['builders'] += [BuilderConfig(name='registry-coverage',slavenames=['buildworker'], factory=factory)] # Registry functional test factory = BuildFactory() factory.addStep(ShellCommand(description='registry', logEnviron=False, command='. {0}/master/credentials.cfg; ' - '/docker-ci/functionaltests/test_registry.sh'.format(BUILDBOT_PATH), + '{1}/functionaltests/test_registry.sh'.format(BUILDBOT_PATH, DOCKER_CI_PATH), usePTY=True)) c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'], factory=factory)] @@ -95,16 +103,17 @@ c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'], factory = BuildFactory() factory.addStep(ShellCommand(description='index', logEnviron=False, command='. {0}/master/credentials.cfg; ' - '/docker-ci/functionaltests/test_index.py'.format(BUILDBOT_PATH), + '{1}/functionaltests/test_index.py'.format(BUILDBOT_PATH, DOCKER_CI_PATH), usePTY=True)) c['builders'] += [BuilderConfig(name='index',slavenames=['buildworker'], factory=factory)] # Docker nightly release +nightlyrelease_cmd = ('docker version; docker run -i -t -privileged -e AWS_S3_BUCKET=' + 'test.docker.io dockerbuilder hack/dind dockerbuild.sh') factory = BuildFactory() -factory.addStep(ShellCommand(description='NightlyRelease', logEnviron=False, - usePTY=True, command='docker run -privileged' - ' -e AWS_S3_BUCKET=test.docker.io dockerbuilder')) +factory.addStep(ShellCommand(description='NightlyRelease',logEnviron=False, + usePTY=True, command=nightlyrelease_cmd)) c['builders'] += [BuilderConfig(name='nightlyrelease',slavenames=['buildworker'], factory=factory)] diff --git a/hack/infrastructure/docker-ci/deployment.py b/hack/infrastructure/docker-ci/deployment.py index 453fad7590..1a389d2771 100755 --- a/hack/infrastructure/docker-ci/deployment.py +++ b/hack/infrastructure/docker-ci/deployment.py @@ -100,8 +100,7 @@ sudo("echo '{}' >> /root/.ssh/authorized_keys".format(env['DOCKER_CI_PUB'])) credentials = { 'AWS_ACCESS_KEY': env['PKG_ACCESS_KEY'], 'AWS_SECRET_KEY': env['PKG_SECRET_KEY'], - 'GPG_PASSPHRASE': env['PKG_GPG_PASSPHRASE'], - 'INDEX_AUTH': env['INDEX_AUTH']} + 'GPG_PASSPHRASE': env['PKG_GPG_PASSPHRASE']} open(DOCKER_CI_PATH + '/nightlyrelease/release_credentials.json', 'w').write( base64.b64encode(json.dumps(credentials))) @@ -143,8 +142,11 @@ sudo('ln -s /sbin/reboot /etc/cron.daily') # Build docker-ci containers sudo('cd {}; docker build -t docker .'.format(DOCKER_PATH)) +sudo('cd {}; docker build -t docker-ci .'.format(DOCKER_CI_PATH)) sudo('cd {}/nightlyrelease; docker build -t dockerbuilder .'.format( DOCKER_CI_PATH)) +sudo('cd {}/registry-coverage; docker build -t registry_coverage .'.format( + DOCKER_CI_PATH)) # Download docker-ci testing container sudo('docker pull mzdaniel/test_docker') diff --git a/hack/infrastructure/docker-ci/docker-test/test_docker.sh b/hack/infrastructure/docker-ci/docker-test/test_docker.sh index 895e4d9642..c8cfe147e6 100755 --- a/hack/infrastructure/docker-ci/docker-test/test_docker.sh +++ b/hack/infrastructure/docker-ci/docker-test/test_docker.sh @@ -9,30 +9,21 @@ BRANCH=${3-master} DOCKER_PATH=/go/src/github.com/dotcloud/docker # Fetch latest master +cd / rm -rf /go -mkdir -p $DOCKER_PATH +git clone -q -b master http://github.com/dotcloud/docker $DOCKER_PATH cd $DOCKER_PATH -git init . -git fetch -q http://github.com/dotcloud/docker master -git reset --hard FETCH_HEAD # Merge commit -#echo FIXME. Temporarily skip TestPrivilegedCanMount until DinD works reliable on AWS -git pull -q https://github.com/mzdaniel/docker.git dind-aws || exit 1 - -# Merge commit in top of master git fetch -q "$REPO" "$BRANCH" git merge --no-edit $COMMIT || exit 1 # Test commit -go test -v; exit_status=$? +./hack/make.sh test; exit_status=$? # Display load if test fails if [ $exit_status -eq 1 ] ; then uptime; echo; free fi -# Cleanup testing directory -rm -rf $BASE_PATH - exit $exit_status diff --git a/hack/infrastructure/docker-ci/functionaltests/test_registry.sh b/hack/infrastructure/docker-ci/functionaltests/test_registry.sh index 8bcd355c7c..d175f66d16 100755 --- a/hack/infrastructure/docker-ci/functionaltests/test_registry.sh +++ b/hack/infrastructure/docker-ci/functionaltests/test_registry.sh @@ -12,6 +12,7 @@ export DOCKER_REGISTRY_CONFIG=config_test.yml # Get latest docker registry git clone -q https://github.com/dotcloud/docker-registry.git cd docker-registry +sed -Ei "s#(boto_bucket: ).+#\1_env:S3_BUCKET#" config_test.yml # Get dependencies pip install -q -r requirements.txt diff --git a/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile b/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile index 541f3a9584..953d7c11cc 100644 --- a/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile +++ b/hack/infrastructure/docker-ci/nightlyrelease/Dockerfile @@ -1,5 +1,5 @@ -# VERSION: 1.2 -# DOCKER-VERSION 0.6.3 +# VERSION: 1.5 +# DOCKER-VERSION 0.6.4 # AUTHOR: Daniel Mizyrycki # DESCRIPTION: Build docker nightly release using Docker in Docker. # REFERENCES: This code reuses the excellent implementation of docker in docker @@ -7,11 +7,10 @@ # COMMENTS: # release_credentials.json is a base64 json encoded file containing: # { "AWS_ACCESS_KEY": "Test_docker_AWS_S3_bucket_id", -# "AWS_SECRET_KEY='Test_docker_AWS_S3_bucket_key' -# "GPG_PASSPHRASE='Test_docker_GPG_passphrase_signature' -# "INDEX_AUTH='Encripted_index_authentication' } +# "AWS_SECRET_KEY": "Test_docker_AWS_S3_bucket_key", +# "GPG_PASSPHRASE": "Test_docker_GPG_passphrase_signature" } # TO_BUILD: docker build -t dockerbuilder . -# TO_RELEASE: docker run -i -t -privileged -e AWS_S3_BUCKET="test.docker.io" dockerbuilder +# TO_RELEASE: docker run -i -t -privileged -e AWS_S3_BUCKET="test.docker.io" dockerbuilder hack/dind dockerbuild.sh from docker maintainer Daniel Mizyrycki @@ -24,11 +23,8 @@ run apt-get update; apt-get install -y -q wget python2.7 run wget -q -O /usr/bin/docker http://get.docker.io/builds/Linux/x86_64/docker-latest; chmod +x /usr/bin/docker # Add proto docker builder -add ./dockerbuild /usr/bin/dockerbuild -run chmod +x /usr/bin/dockerbuild +add ./dockerbuild.sh /usr/bin/dockerbuild.sh +run chmod +x /usr/bin/dockerbuild.sh # Add release credentials add ./release_credentials.json /root/release_credentials.json - -# Launch build process in a container -cmd dockerbuild diff --git a/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild b/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild deleted file mode 100644 index 83a7157a3e..0000000000 --- a/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild +++ /dev/null @@ -1,50 +0,0 @@ -#!/bin/bash - -# Variables AWS_ACCESS_KEY, AWS_SECRET_KEY, PG_PASSPHRASE and INDEX_AUTH -# are decoded from /root/release_credentials.json -# Variable AWS_S3_BUCKET is passed to the environment from docker run -e - -# Enable debugging -set -x - -# Fetch docker master branch -rm -rf /go/src/github.com/dotcloud/docker -cd / -git clone -q http://github.com/dotcloud/docker /go/src/github.com/dotcloud/docker -cd /go/src/github.com/dotcloud/docker - -# Launch docker daemon using dind inside the container -./hack/dind /usr/bin/docker -d & -sleep 5 - -# Add an uncommitted change to generate a timestamped release -date > timestamp - -# Build the docker package using /Dockerfile -docker build -t docker . - -# Run Docker unittests binary and Ubuntu package -docker run -privileged docker hack/make.sh -exit_status=$? - -# Display load if test fails -if [ $exit_status -eq 1 ] ; then - uptime; echo; free - exit 1 -fi - -# Commit binary and ubuntu bundles for release -docker commit -run '{"Env": ["PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin"], "WorkingDir": "/go/src/github.com/dotcloud/docker"}' $(docker ps -l -q) release - -# Turn debug off to load credentials from the environment -set +x -eval $(cat /root/release_credentials.json | python -c ' -import sys,json,base64; -d=json.loads(base64.b64decode(sys.stdin.read())); -exec("""for k in d: print "export {0}=\\"{1}\\"".format(k,d[k])""")') -set -x - -# Push docker nightly -echo docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=XXXXX -e AWS_SECRET_KEY=XXXXX -e GPG_PASSPHRASE=XXXXX release hack/release.sh -set +x -docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=$AWS_ACCESS_KEY -e AWS_SECRET_KEY=$AWS_SECRET_KEY -e GPG_PASSPHRASE=$GPG_PASSPHRASE release hack/release.sh diff --git a/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh b/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh new file mode 100644 index 0000000000..457db3f889 --- /dev/null +++ b/hack/infrastructure/docker-ci/nightlyrelease/dockerbuild.sh @@ -0,0 +1,40 @@ +#!/bin/bash + +# Variables AWS_ACCESS_KEY, AWS_SECRET_KEY and PG_PASSPHRASE are decoded +# from /root/release_credentials.json +# Variable AWS_S3_BUCKET is passed to the environment from docker run -e + +# Turn debug off to load credentials from the environment +set +x +eval $(cat /root/release_credentials.json | python -c ' +import sys,json,base64; +d=json.loads(base64.b64decode(sys.stdin.read())); +exec("""for k in d: print "export {0}=\\"{1}\\"".format(k,d[k])""")') + +# Fetch docker master branch +set -x +cd / +rm -rf /go +git clone -q -b master http://github.com/dotcloud/docker /go/src/github.com/dotcloud/docker +cd /go/src/github.com/dotcloud/docker + +# Launch docker daemon using dind inside the container +/usr/bin/docker version +/usr/bin/docker -d & +sleep 5 + +# Build Docker release container +docker build -t docker . + +# Test docker and if everything works well, release +echo docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=XXXXX -e AWS_SECRET_KEY=XXXXX -e GPG_PASSPHRASE=XXXXX docker hack/release.sh +set +x +docker run -privileged -i -t -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=$AWS_ACCESS_KEY -e AWS_SECRET_KEY=$AWS_SECRET_KEY -e GPG_PASSPHRASE=$GPG_PASSPHRASE docker hack/release.sh +exit_status=$? + +# Display load if test fails +set -x +if [ $exit_status -eq 1 ] ; then + uptime; echo; free + exit 1 +fi diff --git a/hack/infrastructure/docker-ci/nightlyrelease/release_credentials.json b/hack/infrastructure/docker-ci/nightlyrelease/release_credentials.json deleted file mode 100644 index ed6d53ecd1..0000000000 --- a/hack/infrastructure/docker-ci/nightlyrelease/release_credentials.json +++ /dev/null @@ -1 +0,0 @@ -eyAiQVdTX0FDQ0VTU19LRVkiOiAiIiwKICAiQVdTX1NFQ1JFVF9LRVkiOiAiIiwKICAiR1BHX1BBU1NQSFJBU0UiOiAiIiwKICAiSU5ERVhfQVVUSCI6ICIiIH0= diff --git a/hack/infrastructure/docker-ci/registry-coverage/Dockerfile b/hack/infrastructure/docker-ci/registry-coverage/Dockerfile new file mode 100644 index 0000000000..59c914fb2b --- /dev/null +++ b/hack/infrastructure/docker-ci/registry-coverage/Dockerfile @@ -0,0 +1,18 @@ +# VERSION: 0.1 +# DOCKER-VERSION 0.6.4 +# AUTHOR: Daniel Mizyrycki +# DESCRIPTION: Docker registry coverage +# COMMENTS: Add registry coverage into the docker-ci image +# TO_BUILD: docker build -t registry_coverage . +# TO_RUN: docker run registry_coverage + +from docker-ci +maintainer Daniel Mizyrycki + +# Add registry_coverager.sh and dependencies +run pip install coverage flask pyyaml requests simplejson python-glanceclient \ + blinker redis gevent +add registry_coverage.sh /usr/bin/registry_coverage.sh +run chmod +x /usr/bin/registry_coverage.sh + +cmd "/usr/bin/registry_coverage.sh" diff --git a/hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh b/hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh new file mode 100755 index 0000000000..e9f0172651 --- /dev/null +++ b/hack/infrastructure/docker-ci/registry-coverage/registry_coverage.sh @@ -0,0 +1,18 @@ +#!/bin/bash + +set -x + +# Compute test paths +REGISTRY_PATH=/data/docker-registry + +# Fetch latest docker-registry master +rm -rf $REGISTRY_PATH +git clone https://github.com/dotcloud/docker-registry -b master $REGISTRY_PATH +cd $REGISTRY_PATH + +# Generate coverage +export SETTINGS_FLAVOR=test +export DOCKER_REGISTRY_CONFIG=config_test.yml + +coverage run -m unittest discover test || exit 1 +coverage report --include='./*' --omit='./test/*'