|
@@ -1,4 +1,4 @@
|
|
|
-import os
|
|
|
+import os, re
|
|
|
from buildbot.buildslave import BuildSlave
|
|
|
from buildbot.schedulers.forcesched import ForceScheduler
|
|
|
from buildbot.schedulers.basic import SingleBranchScheduler
|
|
@@ -6,127 +6,156 @@ from buildbot.schedulers.timed import Nightly
|
|
|
from buildbot.changes import filter
|
|
|
from buildbot.config import BuilderConfig
|
|
|
from buildbot.process.factory import BuildFactory
|
|
|
-from buildbot.process.properties import Interpolate
|
|
|
+from buildbot.process.properties import Property
|
|
|
from buildbot.steps.shell import ShellCommand
|
|
|
from buildbot.status import html, words
|
|
|
from buildbot.status.web import authz, auth
|
|
|
from buildbot.status.mail import MailNotifier
|
|
|
|
|
|
-PORT_WEB = 80 # Buildbot webserver port
|
|
|
-PORT_GITHUB = 8011 # Buildbot github hook port
|
|
|
-PORT_MASTER = 9989 # Port where buildbot master listen buildworkers
|
|
|
-TEST_USER = 'buildbot' # Credential to authenticate build triggers
|
|
|
-TEST_PWD = 'docker' # Credential to authenticate build triggers
|
|
|
-GITHUB_DOCKER = 'github.com/dotcloud/docker'
|
|
|
-BUILDBOT_PATH = '/data/buildbot'
|
|
|
-DOCKER_PATH = '/go/src/github.com/dotcloud/docker'
|
|
|
-DOCKER_CI_PATH = '/docker-ci'
|
|
|
+
|
|
|
+def ENV(x):
|
|
|
+ '''Promote an environment variable for global use returning its value'''
|
|
|
+ retval = os.environ.get(x, '')
|
|
|
+ globals()[x] = retval
|
|
|
+ return retval
|
|
|
+
|
|
|
+
|
|
|
+class TestCommand(ShellCommand):
|
|
|
+ '''Extend ShellCommand with optional summary logs'''
|
|
|
+ def __init__(self, *args, **kwargs):
|
|
|
+ super(TestCommand, self).__init__(*args, **kwargs)
|
|
|
+
|
|
|
+ def createSummary(self, log):
|
|
|
+ exit_status = re.sub(r'.+\n\+ exit (\d+).+',
|
|
|
+ r'\1', log.getText()[-100:], flags=re.DOTALL)
|
|
|
+ if exit_status != '0':
|
|
|
+ return
|
|
|
+ # Infer coverage path from log
|
|
|
+ if '+ COVERAGE_PATH' in log.getText():
|
|
|
+ path = re.sub(r'.+\+ COVERAGE_PATH=((.+?)-\d+).+',
|
|
|
+ r'\2/\1', log.getText(), flags=re.DOTALL)
|
|
|
+ url = '{}coverage/{}/index.html'.format(c['buildbotURL'], path)
|
|
|
+ self.addURL('coverage', url)
|
|
|
+ elif 'COVERAGE_FILE' in log.getText():
|
|
|
+ path = re.sub(r'.+\+ COVERAGE_FILE=((.+?)-\d+).+',
|
|
|
+ r'\2/\1', log.getText(), flags=re.DOTALL)
|
|
|
+ url = '{}coverage/{}/index.html'.format(c['buildbotURL'], path)
|
|
|
+ self.addURL('coverage', url)
|
|
|
+
|
|
|
+
|
|
|
+PORT_WEB = 8000 # Buildbot webserver port
|
|
|
+PORT_GITHUB = 8011 # Buildbot github hook port
|
|
|
+PORT_MASTER = 9989 # Port where buildbot master listen buildworkers
|
|
|
+
|
|
|
+BUILDBOT_URL = '//localhost:{}/'.format(PORT_WEB)
|
|
|
+DOCKER_REPO = 'https://github.com/docker-test/docker'
|
|
|
+DOCKER_TEST_ARGV = 'HEAD {}'.format(DOCKER_REPO)
|
|
|
+REGISTRY_REPO = 'https://github.com/docker-test/docker-registry'
|
|
|
+REGISTRY_TEST_ARGV = 'HEAD {}'.format(REGISTRY_REPO)
|
|
|
+if ENV('DEPLOYMENT') == 'staging':
|
|
|
+ BUILDBOT_URL = "//docker-ci-stage.docker.io/"
|
|
|
+if ENV('DEPLOYMENT') == 'production':
|
|
|
+ BUILDBOT_URL = '//docker-ci.docker.io/'
|
|
|
+ DOCKER_REPO = 'https://github.com/dotcloud/docker'
|
|
|
+ DOCKER_TEST_ARGV = ''
|
|
|
+ REGISTRY_REPO = 'https://github.com/dotcloud/docker-registry'
|
|
|
+ REGISTRY_TEST_ARGV = ''
|
|
|
|
|
|
# Credentials set by setup.sh from deployment.py
|
|
|
-BUILDBOT_PWD = ''
|
|
|
-IRC_PWD = ''
|
|
|
-IRC_CHANNEL = ''
|
|
|
-SMTP_USER = ''
|
|
|
-SMTP_PWD = ''
|
|
|
-EMAIL_RCP = ''
|
|
|
+ENV('WEB_USER')
|
|
|
+ENV('WEB_IRC_PWD')
|
|
|
+ENV('BUILDBOT_PWD')
|
|
|
+ENV('SMTP_USER')
|
|
|
+ENV('SMTP_PWD')
|
|
|
+ENV('EMAIL_RCP')
|
|
|
+ENV('IRC_CHANNEL')
|
|
|
|
|
|
|
|
|
c = BuildmasterConfig = {}
|
|
|
|
|
|
-c['title'] = "Docker"
|
|
|
+c['title'] = "docker-ci"
|
|
|
c['titleURL'] = "waterfall"
|
|
|
-c['buildbotURL'] = "http://docker-ci.dotcloud.com/"
|
|
|
+c['buildbotURL'] = BUILDBOT_URL
|
|
|
c['db'] = {'db_url':"sqlite:///state.sqlite"}
|
|
|
c['slaves'] = [BuildSlave('buildworker', BUILDBOT_PWD)]
|
|
|
c['slavePortnum'] = PORT_MASTER
|
|
|
|
|
|
|
|
|
# Schedulers
|
|
|
-c['schedulers'] = [ForceScheduler(name='trigger', builderNames=['docker',
|
|
|
- 'index','registry','docker-coverage','registry-coverage','nightlyrelease'])]
|
|
|
-c['schedulers'] += [SingleBranchScheduler(name="all", treeStableTimer=None,
|
|
|
+c['schedulers'] = [ForceScheduler(name='trigger', builderNames=[
|
|
|
+ 'docker', 'docker-registry', 'nightlyrelease', 'backup'])]
|
|
|
+c['schedulers'] += [SingleBranchScheduler(name="docker", treeStableTimer=None,
|
|
|
change_filter=filter.ChangeFilter(branch='master',
|
|
|
- repository='https://github.com/dotcloud/docker'), builderNames=['docker'])]
|
|
|
-c['schedulers'] += [SingleBranchScheduler(name='pullrequest',
|
|
|
- change_filter=filter.ChangeFilter(category='github_pullrequest'), treeStableTimer=None,
|
|
|
- builderNames=['pullrequest'])]
|
|
|
-c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['nightlyrelease',
|
|
|
- 'docker-coverage','registry-coverage'], hour=7, minute=00)]
|
|
|
-c['schedulers'] += [Nightly(name='every4hrs', branch=None, builderNames=['registry','index'],
|
|
|
- hour=range(0,24,4), minute=15)]
|
|
|
+ repository=DOCKER_REPO), builderNames=['docker'])]
|
|
|
+c['schedulers'] += [SingleBranchScheduler(name="registry", treeStableTimer=None,
|
|
|
+ change_filter=filter.ChangeFilter(branch='master',
|
|
|
+ repository=REGISTRY_REPO), builderNames=['docker-registry'])]
|
|
|
+c['schedulers'] += [SingleBranchScheduler(name='docker-pr', treeStableTimer=None,
|
|
|
+ change_filter=filter.ChangeFilter(category='github_pullrequest',
|
|
|
+ project='docker'), builderNames=['docker-pr'])]
|
|
|
+c['schedulers'] += [SingleBranchScheduler(name='docker-registry-pr', treeStableTimer=None,
|
|
|
+ change_filter=filter.ChangeFilter(category='github_pullrequest',
|
|
|
+ project='docker-registry'), builderNames=['docker-registry-pr'])]
|
|
|
+c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=[
|
|
|
+ 'nightlyrelease', 'backup'], hour=7, minute=00)]
|
|
|
+
|
|
|
|
|
|
# Builders
|
|
|
-# Docker commit test
|
|
|
-test_cmd = ('docker run -privileged mzdaniel/test_docker hack/dind'
|
|
|
- ' test_docker.sh %(src::revision)s')
|
|
|
-factory = BuildFactory()
|
|
|
-factory.addStep(ShellCommand(description='Docker', logEnviron=False,
|
|
|
- usePTY=True, command=["sh", "-c", Interpolate(test_cmd)]))
|
|
|
-c['builders'] = [BuilderConfig(name='docker',slavenames=['buildworker'],
|
|
|
- factory=factory)]
|
|
|
|
|
|
-# Docker pull request test
|
|
|
-test_cmd = ('docker run -privileged mzdaniel/test_docker hack/dind'
|
|
|
- ' test_docker.sh %(src::revision)s %(src::repository)s %(src::branch)s')
|
|
|
+# Backup
|
|
|
factory = BuildFactory()
|
|
|
-factory.addStep(ShellCommand(description='pull_request', logEnviron=False,
|
|
|
- usePTY=True, command=["sh", "-c", Interpolate(test_cmd)]))
|
|
|
-c['builders'] += [BuilderConfig(name='pullrequest',slavenames=['buildworker'],
|
|
|
+factory.addStep(TestCommand(description='backup', logEnviron=False,
|
|
|
+ usePTY=True, command='/docker-ci/tool/backup.py'))
|
|
|
+c['builders'] = [BuilderConfig(name='backup',slavenames=['buildworker'],
|
|
|
factory=factory)]
|
|
|
|
|
|
-# Docker coverage test
|
|
|
+# Docker test
|
|
|
factory = BuildFactory()
|
|
|
-factory.addStep(ShellCommand(description='docker-coverage', logEnviron=False,
|
|
|
- usePTY=True, command='{0}/docker-coverage/coverage-docker.sh'.format(
|
|
|
- DOCKER_CI_PATH)))
|
|
|
-c['builders'] += [BuilderConfig(name='docker-coverage',slavenames=['buildworker'],
|
|
|
+factory.addStep(TestCommand(description='docker', logEnviron=False,
|
|
|
+ usePTY=True, command='/docker-ci/dockertest/docker {}'.format(DOCKER_TEST_ARGV)))
|
|
|
+c['builders'] += [BuilderConfig(name='docker',slavenames=['buildworker'],
|
|
|
factory=factory)]
|
|
|
|
|
|
-# Docker registry coverage test
|
|
|
+# Docker pull request test
|
|
|
factory = BuildFactory()
|
|
|
-factory.addStep(ShellCommand(description='registry-coverage', logEnviron=False,
|
|
|
- usePTY=True, command='docker run registry_coverage'.format(
|
|
|
- DOCKER_CI_PATH)))
|
|
|
-c['builders'] += [BuilderConfig(name='registry-coverage',slavenames=['buildworker'],
|
|
|
+factory.addStep(TestCommand(description='docker-pr', logEnviron=False,
|
|
|
+ usePTY=True, command=['/docker-ci/dockertest/docker',
|
|
|
+ Property('revision'), Property('repository'), Property('branch')]))
|
|
|
+c['builders'] += [BuilderConfig(name='docker-pr',slavenames=['buildworker'],
|
|
|
factory=factory)]
|
|
|
|
|
|
-# Registry functional test
|
|
|
+# docker-registry test
|
|
|
factory = BuildFactory()
|
|
|
-factory.addStep(ShellCommand(description='registry', logEnviron=False,
|
|
|
- command='. {0}/master/credentials.cfg; '
|
|
|
- '{1}/functionaltests/test_registry.sh'.format(BUILDBOT_PATH, DOCKER_CI_PATH),
|
|
|
- usePTY=True))
|
|
|
-c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'],
|
|
|
+factory.addStep(TestCommand(description='docker-registry', logEnviron=False,
|
|
|
+ usePTY=True, command='/docker-ci/dockertest/docker-registry {}'.format(REGISTRY_TEST_ARGV)))
|
|
|
+c['builders'] += [BuilderConfig(name='docker-registry',slavenames=['buildworker'],
|
|
|
factory=factory)]
|
|
|
|
|
|
-# Index functional test
|
|
|
+# Docker registry pull request test
|
|
|
factory = BuildFactory()
|
|
|
-factory.addStep(ShellCommand(description='index', logEnviron=False,
|
|
|
- command='. {0}/master/credentials.cfg; '
|
|
|
- '{1}/functionaltests/test_index.py'.format(BUILDBOT_PATH, DOCKER_CI_PATH),
|
|
|
- usePTY=True))
|
|
|
-c['builders'] += [BuilderConfig(name='index',slavenames=['buildworker'],
|
|
|
+factory.addStep(TestCommand(description='docker-registry-pr', logEnviron=False,
|
|
|
+ usePTY=True, command=['/docker-ci/dockertest/docker-registry',
|
|
|
+ Property('revision'), Property('repository'), Property('branch')]))
|
|
|
+c['builders'] += [BuilderConfig(name='docker-registry-pr',slavenames=['buildworker'],
|
|
|
factory=factory)]
|
|
|
|
|
|
# Docker nightly release
|
|
|
-nightlyrelease_cmd = ('docker version; docker run -i -t -privileged -e AWS_S3_BUCKET='
|
|
|
- 'test.docker.io dockerbuilder hack/dind dockerbuild.sh')
|
|
|
factory = BuildFactory()
|
|
|
factory.addStep(ShellCommand(description='NightlyRelease',logEnviron=False,
|
|
|
- usePTY=True, command=nightlyrelease_cmd))
|
|
|
+ usePTY=True, command=['/docker-ci/dockertest/nightlyrelease']))
|
|
|
c['builders'] += [BuilderConfig(name='nightlyrelease',slavenames=['buildworker'],
|
|
|
factory=factory)]
|
|
|
|
|
|
# Status
|
|
|
-authz_cfg = authz.Authz(auth=auth.BasicAuth([(TEST_USER, TEST_PWD)]),
|
|
|
+authz_cfg = authz.Authz(auth=auth.BasicAuth([(WEB_USER, WEB_IRC_PWD)]),
|
|
|
forceBuild='auth')
|
|
|
c['status'] = [html.WebStatus(http_port=PORT_WEB, authz=authz_cfg)]
|
|
|
c['status'].append(html.WebStatus(http_port=PORT_GITHUB, allowForce=True,
|
|
|
change_hook_dialects={ 'github': True }))
|
|
|
-c['status'].append(MailNotifier(fromaddr='buildbot@docker.io',
|
|
|
+c['status'].append(MailNotifier(fromaddr='docker-test@docker.io',
|
|
|
sendToInterestedUsers=False, extraRecipients=[EMAIL_RCP],
|
|
|
mode='failing', relayhost='smtp.mailgun.org', smtpPort=587, useTls=True,
|
|
|
smtpUser=SMTP_USER, smtpPassword=SMTP_PWD))
|
|
|
c['status'].append(words.IRC("irc.freenode.net", "dockerqabot",
|
|
|
- channels=[IRC_CHANNEL], password=IRC_PWD, allowForce=True,
|
|
|
+ channels=[IRC_CHANNEL], password=WEB_IRC_PWD, allowForce=True,
|
|
|
notify_events={'exception':1, 'successToFailure':1, 'failureToSuccess':1}))
|