testing infrastructure, issue #1766: Dockerize deployment of docker-ci
This commit is contained in:
Daniel Mizyrycki
parent
6e3068700f
commit
d542318d63
24 changed files with 367 additions and 266 deletions
43
hack/infrastructure/docker-ci/Dockerfile
Normal file
43
hack/infrastructure/docker-ci/Dockerfile
Normal file
|
|
@ -0,0 +1,43 @@
|
|||
# VERSION: 0.22
|
||||
# DOCKER-VERSION 0.6.3
|
||||
# AUTHOR: Daniel Mizyrycki <daniel@dotcloud.com>
|
||||
# DESCRIPTION: Deploy docker-ci on Amazon EC2
|
||||
# COMMENTS:
|
||||
# CONFIG_JSON is an environment variable json string loaded as:
|
||||
#
|
||||
# export CONFIG_JSON='
|
||||
# { "AWS_TAG": "EC2_instance_name",
|
||||
# "AWS_ACCESS_KEY": "EC2_access_key",
|
||||
# "AWS_SECRET_KEY": "EC2_secret_key",
|
||||
# "DOCKER_CI_PUB": "$(cat docker-ci_ssh_public_key.pub)",
|
||||
# "DOCKER_CI_KEY": "$(cat docker-ci_ssh_private_key.key)",
|
||||
# "BUILDBOT_PWD": "Buildbot_server_password",
|
||||
# "IRC_PWD": "Buildbot_IRC_password",
|
||||
# "SMTP_USER": "SMTP_server_user",
|
||||
# "SMTP_PWD": "SMTP_server_password",
|
||||
# "PKG_ACCESS_KEY": "Docker_release_S3_bucket_access_key",
|
||||
# "PKG_SECRET_KEY": "Docker_release_S3_bucket_secret_key",
|
||||
# "PKG_GPG_PASSPHRASE": "Docker_release_gpg_passphrase",
|
||||
# "INDEX_AUTH": "Index_encripted_user_password",
|
||||
# "REGISTRY_USER": "Registry_test_user",
|
||||
# "REGISTRY_PWD": "Registry_test_password",
|
||||
# "REGISTRY_BUCKET": "Registry_S3_bucket_name",
|
||||
# "REGISTRY_ACCESS_KEY": "Registry_S3_bucket_access_key",
|
||||
# "REGISTRY_SECRET_KEY": "Registry_S3_bucket_secret_key",
|
||||
# "IRC_CHANNEL": "Buildbot_IRC_channel",
|
||||
# "EMAIL_RCP": "Buildbot_mailing_receipient" }'
|
||||
#
|
||||
#
|
||||
# TO_BUILD: docker build -t docker-ci .
|
||||
# TO_DEPLOY: docker run -e CONFIG_JSON="${CONFIG_JSON}" docker-ci
|
||||
|
||||
from ubuntu:12.04
|
||||
|
||||
run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list
|
||||
run apt-get update; apt-get install -y python2.7 python-dev python-pip ssh rsync less vim
|
||||
run pip install boto fabric
|
||||
|
||||
# Add deployment code and set default container command
|
||||
add . /docker-ci
|
||||
cmd "/docker-ci/deployment.py"
|
||||
|
||||
26
hack/infrastructure/docker-ci/README.rst
Normal file
26
hack/infrastructure/docker-ci/README.rst
Normal file
|
|
@ -0,0 +1,26 @@
|
|||
=======
|
||||
testing
|
||||
=======
|
||||
|
||||
This directory contains docker-ci testing related files.
|
||||
|
||||
|
||||
Buildbot
|
||||
========
|
||||
|
||||
Buildbot is a continuous integration system designed to automate the
|
||||
build/test cycle. By automatically rebuilding and testing the tree each time
|
||||
something has changed, build problems are pinpointed quickly, before other
|
||||
developers are inconvenienced by the failure.
|
||||
|
||||
We are running buildbot in Amazon's EC2 to verify docker passes all
|
||||
tests when commits get pushed to the master branch and building
|
||||
nightly releases using Docker in Docker awesome implementation made
|
||||
by Jerome Petazzoni.
|
||||
|
||||
https://github.com/jpetazzo/dind
|
||||
|
||||
Docker's buildbot instance is at http://docker-ci.dotcloud.com/waterfall
|
||||
|
||||
For deployment instructions, please take a look at
|
||||
hack/infrastructure/docker-ci/Dockerfile
|
||||
1
hack/infrastructure/docker-ci/buildbot/README.rst
Normal file
1
hack/infrastructure/docker-ci/buildbot/README.rst
Normal file
|
|
@ -0,0 +1 @@
|
|||
Buildbot configuration and setup files
|
||||
|
|
@ -86,10 +86,14 @@ def getChanges(request, options = None):
|
|||
the http request object
|
||||
"""
|
||||
payload = json.loads(request.args['payload'][0])
|
||||
import urllib,datetime
|
||||
fname = str(datetime.datetime.now()).replace(' ','_').replace(':','-')[:19]
|
||||
open('github_{0}.json'.format(fname),'w').write(json.dumps(json.loads(urllib.unquote(request.args['payload'][0])), sort_keys = True, indent = 2))
|
||||
|
||||
if 'pull_request' in payload:
|
||||
user = payload['repository']['owner']['login']
|
||||
repo = payload['repository']['name']
|
||||
repo_url = payload['repository']['html_url']
|
||||
user = payload['pull_request']['user']['login']
|
||||
repo = payload['pull_request']['head']['repo']['name']
|
||||
repo_url = payload['pull_request']['head']['repo']['html_url']
|
||||
else:
|
||||
user = payload['repository']['owner']['name']
|
||||
repo = payload['repository']['name']
|
||||
|
|
@ -133,7 +137,7 @@ def process_change(payload, user, repo, repo_url, project):
|
|||
if 'pull_request' in payload:
|
||||
changes = [{
|
||||
'category' : 'github_pullrequest',
|
||||
'who' : user,
|
||||
'who' : '{0} - PR#{1}'.format(user,payload['number']),
|
||||
'files' : [],
|
||||
'comments' : payload['pull_request']['title'],
|
||||
'revision' : newrev,
|
||||
|
|
@ -21,6 +21,7 @@ BUILDER_NAME = 'docker'
|
|||
GITHUB_DOCKER = 'github.com/dotcloud/docker'
|
||||
BUILDBOT_PATH = '/data/buildbot'
|
||||
DOCKER_PATH = '/go/src/github.com/dotcloud/docker'
|
||||
DOCKER_CI_PATH = '/docker-ci'
|
||||
BUILDER_PATH = '/data/buildbot/slave/{0}/build'.format(BUILDER_NAME)
|
||||
PULL_REQUEST_PATH = '/data/buildbot/slave/pullrequest/build'
|
||||
|
||||
|
|
@ -60,34 +61,26 @@ c['schedulers'] += [Nightly(name='every4hrs', branch=None, builderNames=['regist
|
|||
# Builders
|
||||
# Docker commit test
|
||||
factory = BuildFactory()
|
||||
factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True,
|
||||
command=["sh", "-c", Interpolate("cd ..; rm -rf build; mkdir build; "
|
||||
"cp -r {2}-dependencies/src {0}; export GOPATH={0}; go get {3}; cd {1}; "
|
||||
"git reset --hard %(src::revision)s; go test -v".format(
|
||||
BUILDER_PATH, BUILDER_PATH+'/src/'+GITHUB_DOCKER, DOCKER_PATH, GITHUB_DOCKER))]))
|
||||
c['builders'] = [BuilderConfig(name=BUILDER_NAME,slavenames=['buildworker'],
|
||||
factory.addStep(ShellCommand(description='Docker', logEnviron=False,
|
||||
usePTY=True, command=['sh', '-c', Interpolate(
|
||||
'{0}/docker-test/test_docker.sh %(src::revision)s'.format(DOCKER_CI_PATH))]))
|
||||
c['builders'] = [BuilderConfig(name='docker',slavenames=['buildworker'],
|
||||
factory=factory)]
|
||||
|
||||
# Docker pull request test
|
||||
factory = BuildFactory()
|
||||
factory.addStep(ShellCommand(description='pull_request',logEnviron=False,usePTY=True,
|
||||
command=["sh", "-c", Interpolate("cd ..; rm -rf build; mkdir build; "
|
||||
"cp -r {2}-dependencies/src {0}; export GOPATH={0}; go get {3}; cd {1}; "
|
||||
"git fetch %(src::repository)s %(src::branch)s:PR-%(src::branch)s; "
|
||||
"git checkout %(src::revision)s; git rebase master; go test -v".format(
|
||||
PULL_REQUEST_PATH, PULL_REQUEST_PATH+'/src/'+GITHUB_DOCKER, DOCKER_PATH, GITHUB_DOCKER))]))
|
||||
factory.addStep(ShellCommand(description='pull_request', logEnviron=False,
|
||||
usePTY=True, command=['sh', '-c', Interpolate(
|
||||
'{0}/docker-test/test_docker.sh %(src::revision)s %(src::repository)s'
|
||||
' %(src::branch)s'.format(DOCKER_CI_PATH))]))
|
||||
c['builders'] += [BuilderConfig(name='pullrequest',slavenames=['buildworker'],
|
||||
factory=factory)]
|
||||
|
||||
# Docker coverage test
|
||||
coverage_cmd = ('GOPATH=`pwd` go get -d github.com/dotcloud/docker\n'
|
||||
'GOPATH=`pwd` go get github.com/axw/gocov/gocov\n'
|
||||
'sudo -E GOPATH=`pwd` ./bin/gocov test -deps -exclude-goroot -v'
|
||||
' -exclude github.com/gorilla/context,github.com/gorilla/mux,github.com/kr/pty,'
|
||||
'code.google.com/p/go.net/websocket github.com/dotcloud/docker | ./bin/gocov report')
|
||||
factory = BuildFactory()
|
||||
factory.addStep(ShellCommand(description='Coverage',logEnviron=False,usePTY=True,
|
||||
command=coverage_cmd))
|
||||
factory.addStep(ShellCommand(description='Coverage', logEnviron=False,
|
||||
usePTY=True, command='{0}/docker-coverage/coverage-docker.sh'.format(
|
||||
DOCKER_CI_PATH)))
|
||||
c['builders'] += [BuilderConfig(name='coverage',slavenames=['buildworker'],
|
||||
factory=factory)]
|
||||
|
||||
|
|
@ -95,8 +88,8 @@ c['builders'] += [BuilderConfig(name='coverage',slavenames=['buildworker'],
|
|||
factory = BuildFactory()
|
||||
factory.addStep(ShellCommand(description='registry', logEnviron=False,
|
||||
command='. {0}/master/credentials.cfg; '
|
||||
'{1}/testing/functionaltests/test_registry.sh'.format(BUILDBOT_PATH,
|
||||
DOCKER_PATH), usePTY=True))
|
||||
'/docker-ci/functionaltests/test_registry.sh'.format(BUILDBOT_PATH),
|
||||
usePTY=True))
|
||||
c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'],
|
||||
factory=factory)]
|
||||
|
||||
|
|
@ -6,15 +6,22 @@
|
|||
|
||||
USER=$1
|
||||
CFG_PATH=$2
|
||||
BUILDBOT_PWD=$3
|
||||
IRC_PWD=$4
|
||||
IRC_CHANNEL=$5
|
||||
SMTP_USER=$6
|
||||
SMTP_PWD=$7
|
||||
EMAIL_RCP=$8
|
||||
DOCKER_PATH=$3
|
||||
BUILDBOT_PWD=$4
|
||||
IRC_PWD=$5
|
||||
IRC_CHANNEL=$6
|
||||
SMTP_USER=$7
|
||||
SMTP_PWD=$8
|
||||
EMAIL_RCP=$9
|
||||
REGISTRY_USER=${10}
|
||||
REGISTRY_PWD=${11}
|
||||
REGISTRY_BUCKET=${12}
|
||||
REGISTRY_ACCESS_KEY=${13}
|
||||
REGISTRY_SECRET_KEY=${14}
|
||||
BUILDBOT_PATH="/data/buildbot"
|
||||
SLAVE_NAME="buildworker"
|
||||
SLAVE_SOCKET="localhost:9989"
|
||||
|
||||
export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin"
|
||||
|
||||
function run { su $USER -c "$1"; }
|
||||
|
|
@ -34,6 +41,10 @@ run "sed -i -E 's#(SMTP_USER = ).+#\1\"$SMTP_USER\"#' master/master.cfg"
|
|||
run "sed -i -E 's#(SMTP_PWD = ).+#\1\"$SMTP_PWD\"#' master/master.cfg"
|
||||
run "sed -i -E 's#(EMAIL_RCP = ).+#\1\"$EMAIL_RCP\"#' master/master.cfg"
|
||||
run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD"
|
||||
run "echo 'export DOCKER_CREDS=\"$REGISTRY_USER:$REGISTRY_PWD\"' > $BUILDBOT_PATH/master/credentials.cfg"
|
||||
run "echo 'export S3_BUCKET=\"$REGISTRY_BUCKET\"' >> $BUILDBOT_PATH/master/credentials.cfg"
|
||||
run "echo 'export S3_ACCESS_KEY=\"$REGISTRY_ACCESS_KEY\"' >> $BUILDBOT_PATH/master/credentials.cfg"
|
||||
run "echo 'export S3_SECRET_KEY=\"$REGISTRY_SECRET_KEY\"' >> $BUILDBOT_PATH/master/credentials.cfg"
|
||||
|
||||
# Patch github webstatus to capture pull requests
|
||||
cp $CFG_PATH/github.py /usr/local/lib/python2.7/dist-packages/buildbot/status/web/hooks
|
||||
155
hack/infrastructure/docker-ci/deployment.py
Executable file
155
hack/infrastructure/docker-ci/deployment.py
Executable file
|
|
@ -0,0 +1,155 @@
|
|||
#!/usr/bin/env python
|
||||
|
||||
import os, sys, re, json, base64
|
||||
from boto.ec2.connection import EC2Connection
|
||||
from subprocess import call
|
||||
from fabric import api
|
||||
from fabric.api import cd, run, put, sudo
|
||||
from os import environ as env
|
||||
from time import sleep
|
||||
|
||||
# Remove SSH private key as it needs more processing
|
||||
CONFIG = json.loads(re.sub(r'("DOCKER_CI_KEY".+?"(.+?)",)','',
|
||||
env['CONFIG_JSON'], flags=re.DOTALL))
|
||||
|
||||
# Populate environment variables
|
||||
for key in CONFIG:
|
||||
env[key] = CONFIG[key]
|
||||
|
||||
# Load SSH private key
|
||||
env['DOCKER_CI_KEY'] = re.sub('^.+"DOCKER_CI_KEY".+?"(.+?)".+','\\1',
|
||||
env['CONFIG_JSON'],flags=re.DOTALL)
|
||||
|
||||
|
||||
AWS_TAG = env.get('AWS_TAG','docker-ci')
|
||||
AWS_KEY_NAME = 'dotcloud-dev' # Same as CONFIG_JSON['DOCKER_CI_PUB']
|
||||
AWS_AMI = 'ami-d582d6bc' # Ubuntu 13.04
|
||||
AWS_REGION = 'us-east-1'
|
||||
AWS_TYPE = 'm1.small'
|
||||
AWS_SEC_GROUPS = 'gateway'
|
||||
AWS_IMAGE_USER = 'ubuntu'
|
||||
DOCKER_PATH = '/go/src/github.com/dotcloud/docker'
|
||||
DOCKER_CI_PATH = '/docker-ci'
|
||||
CFG_PATH = '{}/buildbot'.format(DOCKER_CI_PATH)
|
||||
|
||||
|
||||
class AWS_EC2:
|
||||
'''Amazon EC2'''
|
||||
def __init__(self, access_key, secret_key):
|
||||
'''Set default API parameters'''
|
||||
self.handler = EC2Connection(access_key, secret_key)
|
||||
def create_instance(self, tag, instance_type):
|
||||
reservation = self.handler.run_instances(**instance_type)
|
||||
instance = reservation.instances[0]
|
||||
sleep(10)
|
||||
while instance.state != 'running':
|
||||
sleep(5)
|
||||
instance.update()
|
||||
print "Instance state: %s" % (instance.state)
|
||||
instance.add_tag("Name",tag)
|
||||
print "instance %s done!" % (instance.id)
|
||||
return instance.ip_address
|
||||
def get_instances(self):
|
||||
return self.handler.get_all_instances()
|
||||
def get_tags(self):
|
||||
return dict([(i.instances[0].id, i.instances[0].tags['Name'])
|
||||
for i in self.handler.get_all_instances() if i.instances[0].tags])
|
||||
def del_instance(self, instance_id):
|
||||
self.handler.terminate_instances(instance_ids=[instance_id])
|
||||
|
||||
|
||||
def json_fmt(data):
|
||||
'''Format json output'''
|
||||
return json.dumps(data, sort_keys = True, indent = 2)
|
||||
|
||||
|
||||
# Create EC2 API handler
|
||||
ec2 = AWS_EC2(env['AWS_ACCESS_KEY'], env['AWS_SECRET_KEY'])
|
||||
|
||||
# Stop processing if AWS_TAG exists on EC2
|
||||
if AWS_TAG in ec2.get_tags().values():
|
||||
print ('Instance: {} already deployed. Not further processing.'
|
||||
.format(AWS_TAG))
|
||||
exit(1)
|
||||
|
||||
ip = ec2.create_instance(AWS_TAG, {'image_id':AWS_AMI, 'instance_type':AWS_TYPE,
|
||||
'security_groups':[AWS_SEC_GROUPS], 'key_name':AWS_KEY_NAME})
|
||||
|
||||
# Wait 30 seconds for the machine to boot
|
||||
sleep(30)
|
||||
|
||||
# Create docker-ci ssh private key so docker-ci docker container can communicate
|
||||
# with its EC2 instance
|
||||
os.makedirs('/root/.ssh')
|
||||
open('/root/.ssh/id_rsa','w').write(env['DOCKER_CI_KEY'])
|
||||
os.chmod('/root/.ssh/id_rsa',0600)
|
||||
open('/root/.ssh/config','w').write('StrictHostKeyChecking no\n')
|
||||
|
||||
api.env.host_string = ip
|
||||
api.env.user = AWS_IMAGE_USER
|
||||
api.env.key_filename = '/root/.ssh/id_rsa'
|
||||
|
||||
# Correct timezone
|
||||
sudo('echo "America/Los_Angeles" >/etc/timezone')
|
||||
sudo('dpkg-reconfigure --frontend noninteractive tzdata')
|
||||
|
||||
# Load public docker-ci key
|
||||
sudo("echo '{}' >> /root/.ssh/authorized_keys".format(env['DOCKER_CI_PUB']))
|
||||
|
||||
# Create docker nightly release credentials file
|
||||
credentials = {
|
||||
'AWS_ACCESS_KEY': env['PKG_ACCESS_KEY'],
|
||||
'AWS_SECRET_KEY': env['PKG_SECRET_KEY'],
|
||||
'GPG_PASSPHRASE': env['PKG_GPG_PASSPHRASE'],
|
||||
'INDEX_AUTH': env['INDEX_AUTH']}
|
||||
open(DOCKER_CI_PATH + '/nightlyrelease/release_credentials.json', 'w').write(
|
||||
base64.b64encode(json.dumps(credentials)))
|
||||
|
||||
# Transfer docker
|
||||
sudo('mkdir -p ' + DOCKER_CI_PATH)
|
||||
sudo('chown {}.{} {}'.format(AWS_IMAGE_USER, AWS_IMAGE_USER, DOCKER_CI_PATH))
|
||||
call('/usr/bin/rsync -aH {} {}@{}:{}'.format(DOCKER_CI_PATH, AWS_IMAGE_USER, ip,
|
||||
os.path.dirname(DOCKER_CI_PATH)), shell=True)
|
||||
|
||||
# Install Docker and Buildbot dependencies
|
||||
sudo('addgroup docker')
|
||||
sudo('usermod -a -G docker ubuntu')
|
||||
sudo('mkdir /mnt/docker; ln -s /mnt/docker /var/lib/docker')
|
||||
sudo('wget -q -O - https://get.docker.io/gpg | apt-key add -')
|
||||
sudo('echo deb https://get.docker.io/ubuntu docker main >'
|
||||
' /etc/apt/sources.list.d/docker.list')
|
||||
sudo('echo -e "deb http://archive.ubuntu.com/ubuntu raring main universe\n'
|
||||
'deb http://us.archive.ubuntu.com/ubuntu/ raring-security main universe\n"'
|
||||
' > /etc/apt/sources.list; apt-get update')
|
||||
sudo('DEBIAN_FRONTEND=noninteractive apt-get install -q -y wget python-dev'
|
||||
' python-pip supervisor git mercurial linux-image-extra-$(uname -r)'
|
||||
' aufs-tools make libfontconfig libevent-dev')
|
||||
sudo('wget -O - https://go.googlecode.com/files/go1.1.2.linux-amd64.tar.gz | '
|
||||
'tar -v -C /usr/local -xz; ln -s /usr/local/go/bin/go /usr/bin/go')
|
||||
sudo('GOPATH=/go go get -d github.com/dotcloud/docker')
|
||||
sudo('pip install -r {}/requirements.txt'.format(CFG_PATH))
|
||||
|
||||
# Install docker and testing dependencies
|
||||
sudo('apt-get install -y -q lxc-docker')
|
||||
sudo('curl -s https://phantomjs.googlecode.com/files/'
|
||||
'phantomjs-1.9.1-linux-x86_64.tar.bz2 | tar jx -C /usr/bin'
|
||||
' --strip-components=2 phantomjs-1.9.1-linux-x86_64/bin/phantomjs')
|
||||
|
||||
#### FIXME. Temporarily install docker with proper apparmor handling
|
||||
sudo('stop docker')
|
||||
sudo('wget -q -O /usr/bin/docker http://test.docker.io/test/docker')
|
||||
sudo('start docker')
|
||||
|
||||
# Build docker-ci containers
|
||||
sudo('cd {}; docker build -t docker .'.format(DOCKER_PATH))
|
||||
sudo('cd {}/nightlyrelease; docker build -t dockerbuilder .'.format(
|
||||
DOCKER_CI_PATH))
|
||||
|
||||
# Setup buildbot
|
||||
sudo('mkdir /data')
|
||||
sudo('{0}/setup.sh root {0} {1} {2} {3} {4} {5} {6} {7} {8} {9} {10}'
|
||||
' {11} {12}'.format(CFG_PATH, DOCKER_PATH, env['BUILDBOT_PWD'],
|
||||
env['IRC_PWD'], env['IRC_CHANNEL'], env['SMTP_USER'],
|
||||
env['SMTP_PWD'], env['EMAIL_RCP'], env['REGISTRY_USER'],
|
||||
env['REGISTRY_PWD'], env['REGISTRY_BUCKET'], env['REGISTRY_ACCESS_KEY'],
|
||||
env['REGISTRY_SECRET_KEY']))
|
||||
32
hack/infrastructure/docker-ci/docker-coverage/coverage-docker.sh
Executable file
32
hack/infrastructure/docker-ci/docker-coverage/coverage-docker.sh
Executable file
|
|
@ -0,0 +1,32 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -x
|
||||
# Generate a random string of $1 characters
|
||||
function random {
|
||||
cat /dev/urandom | tr -cd 'a-f0-9' | head -c $1
|
||||
}
|
||||
|
||||
# Compute test paths
|
||||
BASE_PATH=`pwd`/test_docker_$(random 12)
|
||||
DOCKER_PATH=$BASE_PATH/go/src/github.com/dotcloud/docker
|
||||
export GOPATH=$BASE_PATH/go:$DOCKER_PATH/vendor
|
||||
|
||||
# Fetch latest master
|
||||
mkdir -p $DOCKER_PATH
|
||||
cd $DOCKER_PATH
|
||||
git init .
|
||||
git fetch -q http://github.com/dotcloud/docker master
|
||||
git reset --hard FETCH_HEAD
|
||||
|
||||
# Fetch go coverage
|
||||
cd $BASE_PATH/go
|
||||
GOPATH=$BASE_PATH/go go get github.com/axw/gocov/gocov
|
||||
sudo -E GOPATH=$GOPATH ./bin/gocov test -deps -exclude-goroot -v\
|
||||
-exclude github.com/gorilla/context,github.com/gorilla/mux,github.com/kr/pty,\
|
||||
code.google.com/p/go.net/websocket,github.com/dotcloud/tar\
|
||||
github.com/dotcloud/docker | ./bin/gocov report; exit_status=$?
|
||||
|
||||
# Cleanup testing directory
|
||||
rm -rf $BASE_PATH
|
||||
|
||||
exit $exit_status
|
||||
35
hack/infrastructure/docker-ci/docker-test/test_docker.sh
Executable file
35
hack/infrastructure/docker-ci/docker-test/test_docker.sh
Executable file
|
|
@ -0,0 +1,35 @@
|
|||
#!/bin/bash
|
||||
|
||||
set -x
|
||||
COMMIT=${1-HEAD}
|
||||
REPO=${2-http://github.com/dotcloud/docker}
|
||||
BRANCH=${3-master}
|
||||
|
||||
# Generate a random string of $1 characters
|
||||
function random {
|
||||
cat /dev/urandom | tr -cd 'a-f0-9' | head -c $1
|
||||
}
|
||||
|
||||
# Compute test paths
|
||||
BASE_PATH=`pwd`/test_docker_$(random 12)
|
||||
DOCKER_PATH=$BASE_PATH/go/src/github.com/dotcloud/docker
|
||||
export GOPATH=$BASE_PATH/go:$DOCKER_PATH/vendor
|
||||
|
||||
# Fetch latest master
|
||||
mkdir -p $DOCKER_PATH
|
||||
cd $DOCKER_PATH
|
||||
git init .
|
||||
git fetch -q http://github.com/dotcloud/docker master
|
||||
git reset --hard FETCH_HEAD
|
||||
|
||||
# Merge commit
|
||||
git fetch -q "$REPO" "$BRANCH"
|
||||
git merge --no-edit $COMMIT || exit 1
|
||||
|
||||
# Test commit
|
||||
go test -v; exit_status=$?
|
||||
|
||||
# Cleanup testing directory
|
||||
rm -rf $BASE_PATH
|
||||
|
||||
exit $exit_status
|
||||
26
hack/infrastructure/docker-ci/functionaltests/test_registry.sh
Executable file
26
hack/infrastructure/docker-ci/functionaltests/test_registry.sh
Executable file
|
|
@ -0,0 +1,26 @@
|
|||
#!/bin/sh
|
||||
|
||||
set -x
|
||||
|
||||
# Cleanup
|
||||
rm -rf docker-registry
|
||||
|
||||
# Setup the environment
|
||||
export SETTINGS_FLAVOR=test
|
||||
export DOCKER_REGISTRY_CONFIG=config_test.yml
|
||||
|
||||
# Get latest docker registry
|
||||
git clone -q https://github.com/dotcloud/docker-registry.git
|
||||
cd docker-registry
|
||||
|
||||
# Get dependencies
|
||||
pip install -q -r requirements.txt
|
||||
pip install -q -r test-requirements.txt
|
||||
pip install -q tox
|
||||
|
||||
# Run registry tests
|
||||
tox || exit 1
|
||||
export PYTHONPATH=$(pwd)/docker-registry
|
||||
python -m unittest discover -p s3.py -s test || exit 1
|
||||
python -m unittest discover -p workflow.py -s test
|
||||
|
||||
|
|
@ -1,5 +1,5 @@
|
|||
# VERSION: 1.1
|
||||
# DOCKER-VERSION 0.6.2
|
||||
# VERSION: 1.2
|
||||
# DOCKER-VERSION 0.6.3
|
||||
# AUTHOR: Daniel Mizyrycki <daniel@dotcloud.com>
|
||||
# DESCRIPTION: Build docker nightly release using Docker in Docker.
|
||||
# REFERENCES: This code reuses the excellent implementation of docker in docker
|
||||
|
|
@ -13,12 +13,12 @@
|
|||
# TO_BUILD: docker build -t dockerbuilder .
|
||||
# TO_RELEASE: docker run -i -t -privileged -lxc-conf="lxc.aa_profile = unconfined" -e AWS_S3_BUCKET="test.docker.io" dockerbuilder
|
||||
|
||||
from ubuntu:12.04
|
||||
from docker
|
||||
maintainer Daniel Mizyrycki <daniel@dotcloud.com>
|
||||
|
||||
# Add docker dependencies and downloading packages
|
||||
run echo 'deb http://archive.ubuntu.com/ubuntu precise main universe' > /etc/apt/sources.list
|
||||
run apt-get update; apt-get install -y -q lxc bzip2 iptables ca-certificates git wget python2.7
|
||||
run apt-get update; apt-get install -y -q wget python2.7
|
||||
|
||||
# Add production docker binary
|
||||
run wget -q -O /usr/bin/docker http://get.docker.io/builds/Linux/x86_64/docker-latest; chmod +x /usr/bin/docker
|
||||
|
|
@ -33,9 +33,5 @@ run chmod +x /usr/bin/dockerbuild
|
|||
# Add release credentials
|
||||
add ./release_credentials.json /root/release_credentials.json
|
||||
|
||||
# Make /var/lib/docker inside the container addressable by other containers.
|
||||
# This is done to ensure /var/lib/docker has AUFS support needed by the inner docker server
|
||||
volume /var/lib/docker
|
||||
|
||||
# Launch build process in a container
|
||||
cmd dockerbuild
|
||||
|
|
@ -8,6 +8,8 @@
|
|||
set -x
|
||||
|
||||
# Fetch docker master branch
|
||||
rm -rf /go/src/github.com/dotcloud/docker
|
||||
cd /
|
||||
git clone -q http://github.com/dotcloud/docker /go/src/github.com/dotcloud/docker
|
||||
cd /go/src/github.com/dotcloud/docker
|
||||
|
||||
|
|
@ -16,7 +18,7 @@ git fetch http://github.com/jpetazzo/docker escape-apparmor-confinement:escape-
|
|||
git rebase --onto master master escape-apparmor-confinement
|
||||
|
||||
# Launch docker daemon using dind inside the container
|
||||
./hack/dind /usr/bin/docker -dns=8.8.8.8 -d &
|
||||
./hack/dind /usr/bin/docker -d &
|
||||
sleep 5
|
||||
|
||||
# Add an uncommitted change to generate a timestamped release
|
||||
|
|
@ -25,17 +27,10 @@ date > timestamp
|
|||
# Build the docker package using /Dockerfile
|
||||
docker build -t docker .
|
||||
|
||||
# Run Docker unittests
|
||||
docker run -privileged docker go test -v || exit 1
|
||||
# Run Docker unittests binary and Ubuntu package
|
||||
docker run -privileged -lxc-conf=lxc.aa_profile=unconfined docker hack/make.sh || exit 1
|
||||
|
||||
# Create Docker binary and Ubuntu package
|
||||
docker run -privileged docker hack/make.sh binary ubuntu
|
||||
|
||||
# Freeze the container to upload the release
|
||||
docker commit -run '{"Env": ["PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin"], "WorkingDir": "/go/src/github.com/dotcloud/docker"}' $(docker ps -l -q) release
|
||||
|
||||
# Turn debug off to load credentials in the environment and
|
||||
# to authenticate to the index
|
||||
# Turn debug off to load credentials from the environment
|
||||
set +x
|
||||
eval $(cat /root/release_credentials.json | python -c '
|
||||
import sys,json,base64;
|
||||
|
|
@ -44,34 +39,6 @@ exec("""for k in d: print "export {0}=\\"{1}\\"".format(k,d[k])""")')
|
|||
echo '{"https://index.docker.io/v1/":{"auth":"'$INDEX_AUTH'","email":"engineering@dotcloud.com"}}' > /.dockercfg
|
||||
set -x
|
||||
|
||||
# Extract docker binary
|
||||
docker cp $(docker ps -l -q):/go/src/github.com/dotcloud/docker/bundles /tmp
|
||||
|
||||
# Swap docker production daemon with new docker binary for testing
|
||||
kill $(pgrep '^docker$')
|
||||
sleep 15
|
||||
cp /tmp/bundles/*/binary/* /usr/bin/docker
|
||||
./hack/dind /usr/bin/docker -dns=8.8.8.8 -d &
|
||||
sleep 15
|
||||
|
||||
# Run Docker functional tests
|
||||
# Generate unique image name
|
||||
export DIMAGE=testimage`date +'%Y%m%d%H%M%S'`
|
||||
|
||||
# Simple docker version test
|
||||
docker version || exit 1
|
||||
|
||||
# Containerized hello world
|
||||
docker run -cidfile=hello.cid busybox echo 'Hello world' | grep -q 'Hello world' || exit 1
|
||||
|
||||
# Create an image based on the busybox container and test pushing it to the index
|
||||
docker commit `cat hello.cid` test/$DIMAGE
|
||||
docker images | grep -q test/$DIMAGE || exit 1
|
||||
docker push test/$DIMAGE
|
||||
|
||||
# Verify the image was properly pushed to the index
|
||||
docker search $DIMAGE | grep -q $DIMAGE || exit 1
|
||||
|
||||
# Push docker nightly
|
||||
echo docker run -i -t -privileged -e AWS_S3_BUCKET=$AWS_S3_BUCKET -e AWS_ACCESS_KEY=XXXXX -e AWS_SECRET_KEY=XXXXX -e GPG_PASSPHRASE=XXXXX release hack/release.sh
|
||||
set +x
|
||||
|
|
@ -1,58 +0,0 @@
|
|||
=======
|
||||
testing
|
||||
=======
|
||||
|
||||
This directory contains testing related files.
|
||||
|
||||
|
||||
Buildbot
|
||||
========
|
||||
|
||||
Buildbot is a continuous integration system designed to automate the
|
||||
build/test cycle. By automatically rebuilding and testing the tree each time
|
||||
something has changed, build problems are pinpointed quickly, before other
|
||||
developers are inconvenienced by the failure.
|
||||
|
||||
We are running buildbot in an AWS instance to verify docker passes all tests
|
||||
when commits get pushed to the master branch.
|
||||
|
||||
You can check docker's buildbot instance at http://docker-ci.dotcloud.com/waterfall
|
||||
|
||||
|
||||
Deployment
|
||||
~~~~~~~~~~
|
||||
|
||||
::
|
||||
|
||||
# Define AWS credential environment variables
|
||||
export AWS_ACCESS_KEY_ID=xxxxxxxxxxxx
|
||||
export AWS_SECRET_ACCESS_KEY=xxxxxxxxxxxx
|
||||
export AWS_KEYPAIR_NAME=xxxxxxxxxxxx
|
||||
export AWS_SSH_PRIVKEY=xxxxxxxxxxxx
|
||||
|
||||
# Define email recipient and IRC channel
|
||||
export EMAIL_RCP=xxxxxx@domain.com
|
||||
export IRC_CHANNEL=docker
|
||||
|
||||
# Define buildbot credentials
|
||||
export BUILDBOT_PWD=xxxxxxxxxxxx
|
||||
export IRC_PWD=xxxxxxxxxxxx
|
||||
export SMTP_USER=xxxxxxxxxxxx
|
||||
export SMTP_PWD=xxxxxxxxxxxx
|
||||
|
||||
# Define docker registry functional test credentials
|
||||
export REGISTRY_USER=xxxxxxxxxxxx
|
||||
export REGISTRY_PWD=xxxxxxxxxxxx
|
||||
|
||||
# Checkout docker
|
||||
git clone git://github.com/dotcloud/docker.git
|
||||
|
||||
# Deploy docker on AWS
|
||||
cd docker/testing
|
||||
vagrant up --provider=aws
|
||||
|
||||
|
||||
Buildbot AWS dependencies
|
||||
-------------------------
|
||||
|
||||
vagrant, virtualbox packages and vagrant aws plugin
|
||||
87
testing/Vagrantfile
vendored
87
testing/Vagrantfile
vendored
|
|
@ -1,87 +0,0 @@
|
|||
# -*- mode: ruby -*-
|
||||
# vi: set ft=ruby :
|
||||
|
||||
BOX_NAME = "docker-ci"
|
||||
BOX_URI = "http://files.vagrantup.com/precise64.box"
|
||||
AWS_AMI = "ami-d0f89fb9"
|
||||
DOCKER_PATH = "/go/src/github.com/dotcloud/docker"
|
||||
CFG_PATH = "#{DOCKER_PATH}/testing/buildbot"
|
||||
on_vbox = File.file?("#{File.dirname(__FILE__)}/.vagrant/machines/default/virtualbox/id") | \
|
||||
Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty? & \
|
||||
(on_vbox=true; ARGV.each do |arg| on_vbox &&= !arg.downcase.start_with?("--provider") end; on_vbox)
|
||||
USER = on_vbox ? "vagrant": "ubuntu"
|
||||
|
||||
Vagrant::Config.run do |config|
|
||||
# Setup virtual machine box. This VM configuration code is always executed.
|
||||
config.vm.box = BOX_NAME
|
||||
config.vm.box_url = BOX_URI
|
||||
config.vm.forward_port 8010, 8010
|
||||
config.vm.share_folder "v-data", DOCKER_PATH, "#{File.dirname(__FILE__)}/.."
|
||||
|
||||
# Deploy buildbot and its dependencies if it was not done
|
||||
if Dir.glob("#{File.dirname(__FILE__)}/.vagrant/machines/default/*/id").empty?
|
||||
|
||||
# Add memory limitation capabilities
|
||||
pkg_cmd = 'sed -Ei \'s/^(GRUB_CMDLINE_LINUX_DEFAULT)=.+/\\1="cgroup_enable=memory swapaccount=1 quiet"/\' /etc/default/grub; '
|
||||
# Install new kernel
|
||||
pkg_cmd << "apt-get update -qq; apt-get install -q -y linux-image-generic-lts-raring; "
|
||||
|
||||
# Deploy buildbot CI
|
||||
pkg_cmd << "echo 'America/Los_Angeles' > /etc/timezone; " \
|
||||
"dpkg-reconfigure --frontend noninteractive tzdata; " \
|
||||
"apt-get install -q -y python-dev python-pip supervisor; " \
|
||||
"pip install -r #{CFG_PATH}/requirements.txt; " \
|
||||
"mkdir /data; chown #{USER}.#{USER} /data; cd /data; " \
|
||||
"#{CFG_PATH}/setup.sh #{USER} #{CFG_PATH} #{ENV['BUILDBOT_PWD']} " \
|
||||
"#{ENV['IRC_PWD']} #{ENV['IRC_CHANNEL']} #{ENV['SMTP_USER']} " \
|
||||
"#{ENV['SMTP_PWD']} #{ENV['EMAIL_RCP']}; " \
|
||||
"#{CFG_PATH}/setup_credentials.sh #{USER} " \
|
||||
"#{ENV['REGISTRY_USER']} #{ENV['REGISTRY_PWD']}; "
|
||||
|
||||
# Install docker
|
||||
pkg_cmd << "mkdir /mnt/docker; ln -s /mnt/docker /var/lib/docker; " \
|
||||
"wget -q -O - https://get.docker.io/gpg | apt-key add -; " \
|
||||
"echo deb https://get.docker.io/ubuntu docker main > /etc/apt/sources.list.d/docker.list; " \
|
||||
"apt-get update -qq; apt-get install -yq lxc-docker; "
|
||||
|
||||
#### FIXME. Temporarily install docker with proper apparmor handling
|
||||
pkg_cmd << "stop docker; wget -q -O /usr/bin/docker http://test.docker.io/test/docker; start docker; " \
|
||||
|
||||
# Install testing dependencies
|
||||
pkg_cmd << "wget -O go.tgz http://go.googlecode.com/files/go1.1.2.linux-amd64.tar.gz; " \
|
||||
" tar -C /usr/local -vxzf go.tgz; ln -s /usr/local/go/bin/go /usr/bin/go; " \
|
||||
"curl -s https://phantomjs.googlecode.com/files/phantomjs-1.9.1-linux-x86_64.tar.bz2 | " \
|
||||
" tar jx -C /usr/bin --strip-components=2 phantomjs-1.9.1-linux-x86_64/bin/phantomjs; " \
|
||||
"DEBIAN_FRONTEND=noninteractive apt-get install -qy lxc git mercurial aufs-tools " \
|
||||
" make libfontconfig libevent-dev; " \
|
||||
"export GOPATH=/data/docker-dependencies; go get -d github.com/dotcloud/docker; " \
|
||||
"rm -rf ${GOPATH}/src/github.com/dotcloud/docker; "
|
||||
|
||||
# After reboot, build containers and prepare docker to run on dind
|
||||
pkg_cmd << "cp #{DOCKER_PATH}/testing/buildbot/dockerci_local.sh /etc/rc.local; " \
|
||||
|
||||
# Activate new kernel
|
||||
pkg_cmd << "shutdown -r +1; "
|
||||
config.vm.provision :shell, :inline => pkg_cmd
|
||||
end
|
||||
end
|
||||
|
||||
# Providers were added on Vagrant >= 1.1.0
|
||||
Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config|
|
||||
config.vm.box = "dummy"
|
||||
config.vm.provider :aws do |aws, override|
|
||||
aws.tags = { 'Name' => 'docker-ci' }
|
||||
aws.access_key_id = ENV["AWS_ACCESS_KEY_ID"]
|
||||
aws.secret_access_key = ENV["AWS_SECRET_ACCESS_KEY"]
|
||||
aws.keypair_name = ENV["AWS_KEYPAIR_NAME"]
|
||||
override.ssh.private_key_path = ENV["AWS_SSH_PRIVKEY"]
|
||||
override.ssh.username = USER
|
||||
aws.ami = AWS_AMI
|
||||
aws.region = "us-east-1"
|
||||
aws.instance_type = "m1.small"
|
||||
aws.security_groups = "gateway"
|
||||
end
|
||||
|
||||
config.vm.provider :virtualbox do |vb|
|
||||
end
|
||||
end
|
||||
|
|
@ -1 +0,0 @@
|
|||
Buildbot configuration and setup files (except Vagrantfile located on ..)
|
||||
|
|
@ -1,5 +0,0 @@
|
|||
# Credentials for tests. Buildbot source this file on tests
|
||||
# when needed.
|
||||
|
||||
# Docker registry credentials. Format: 'username:password'
|
||||
export DOCKER_CREDS=''
|
||||
|
|
@ -1,9 +0,0 @@
|
|||
#!/bin/sh -e
|
||||
# This is a one time script to prepare docker-ci
|
||||
|
||||
# Build docker nightly release container
|
||||
cd /go/src/github.com/dotcloud/docker/testing/nightlyrelease; docker build -t dockerbuilder .
|
||||
|
||||
# Self removing
|
||||
echo -e '#!/bin/sh -e\nexit 0\n' > /etc/rc.local
|
||||
exit 0
|
||||
|
|
@ -1,17 +0,0 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Setup of test credentials. Called by Vagrantfile
|
||||
export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin"
|
||||
|
||||
USER=$1
|
||||
REGISTRY_USER=$2
|
||||
REGISTRY_PWD=$3
|
||||
|
||||
BUILDBOT_PATH="/data/buildbot"
|
||||
DOCKER_PATH="/go/src/github.com/dotcloud/docker"
|
||||
|
||||
function run { su $USER -c "$1"; }
|
||||
|
||||
run "cp $DOCKER_PATH/testing/buildbot/credentials.cfg $BUILDBOT_PATH/master"
|
||||
cd $BUILDBOT_PATH/master
|
||||
run "sed -i -E 's#(export DOCKER_CREDS=).+#\1\"$REGISTRY_USER:$REGISTRY_PWD\"#' credentials.cfg"
|
||||
|
|
@ -1,11 +0,0 @@
|
|||
#!/bin/sh
|
||||
|
||||
# Cleanup
|
||||
rm -rf docker-registry
|
||||
|
||||
# Get latest docker registry
|
||||
git clone https://github.com/dotcloud/docker-registry.git
|
||||
|
||||
# Configure and run registry tests
|
||||
cd docker-registry; cp config_sample.yml config.yml
|
||||
cd test; python -m unittest workflow
|
||||
Loading…
Reference in a new issue