Merge remote-tracking branch 'origin/buildbot'

Makefile

@@ -52,3 +52,6 @@ test: all
 
 fmt:
 	@gofmt -s -l -w .
+
+hack:
+	cd $(CURDIR)/buildbot && vagrant up

buildbot/README.rst

@@ -0,0 +1,20 @@
+Buildbot
+========
+
+Buildbot is a continuous integration system designed to automate the
+build/test cycle. By automatically rebuilding and testing the tree each time
+something has changed, build problems are pinpointed quickly, before other
+developers are inconvenienced by the failure.
+
+When running 'make hack' at the docker root directory, it spawns a virtual
+machine in the background running a buildbot instance and adds a git
+post-commit hook that automatically run docker tests for you.
+
+You can check your buildbot instance at http://192.168.33.21:8010/waterfall
+
+
+Buildbot dependencies
+---------------------
+
+vagrant, virtualbox packages and python package requests
+

buildbot/Vagrantfile

@@ -0,0 +1,28 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+$BUILDBOT_IP = '192.168.33.21'
+
+def v10(config)
+  config.vm.box = "quantal64_3.5.0-25"
+  config.vm.box_url = "http://get.docker.io/vbox/ubuntu/12.10/quantal64_3.5.0-25.box"
+  config.vm.share_folder 'v-data', '/data/docker', File.dirname(__FILE__) + '/..'
+  config.vm.network :hostonly, $BUILDBOT_IP
+
+  # Ensure puppet is installed on the instance
+  config.vm.provision :shell, :inline => 'apt-get -qq update; apt-get install -y puppet'
+
+  config.vm.provision :puppet do |puppet|
+    puppet.manifests_path = '.'
+    puppet.manifest_file  = 'buildbot.pp'
+    puppet.options = ['--templatedir','.']
+  end
+end
+
+Vagrant::VERSION < '1.1.0' and Vagrant::Config.run do |config|
+  v10(config)
+end
+
+Vagrant::VERSION >= '1.1.0' and Vagrant.configure('1') do |config|
+  v10(config)
+end

buildbot/buildbot-cfg/buildbot-cfg.sh

@@ -0,0 +1,43 @@
+#!/bin/bash
+
+# Auto setup of buildbot configuration. Package installation is being done
+# on buildbot.pp
+# Dependencies: buildbot, buildbot-slave, supervisor
+
+SLAVE_NAME='buildworker'
+SLAVE_SOCKET='localhost:9989'
+BUILDBOT_PWD='pass-docker'
+USER='vagrant'
+ROOT_PATH='/data/buildbot'
+DOCKER_PATH='/data/docker'
+BUILDBOT_CFG="$DOCKER_PATH/buildbot/buildbot-cfg"
+IP=$(grep BUILDBOT_IP /data/docker/buildbot/Vagrantfile | awk -F "'" '{ print $2; }')
+
+function run { su $USER -c "$1"; }
+
+export PATH=/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin
+
+# Exit if buildbot has already been installed
+[ -d "$ROOT_PATH" ] && exit 0
+
+# Setup buildbot
+run "mkdir -p ${ROOT_PATH}"
+cd ${ROOT_PATH}
+run "buildbot create-master master"
+run "cp $BUILDBOT_CFG/master.cfg master"
+run "sed -i 's/localhost/$IP/' master/master.cfg"
+run "buildslave create-slave slave $SLAVE_SOCKET $SLAVE_NAME $BUILDBOT_PWD"
+
+# Allow buildbot subprocesses (docker tests) to properly run in containers,
+# in particular with docker -u
+run "sed -i 's/^umask = None/umask = 000/' ${ROOT_PATH}/slave/buildbot.tac"
+
+# Setup supervisor
+cp $BUILDBOT_CFG/buildbot.conf /etc/supervisor/conf.d/buildbot.conf
+sed -i "s/^chmod=0700.*0700./chmod=0770\nchown=root:$USER/" /etc/supervisor/supervisord.conf
+kill -HUP `pgrep -f "/usr/bin/python /usr/bin/supervisord"`
+
+# Add git hook
+cp $BUILDBOT_CFG/post-commit $DOCKER_PATH/.git/hooks
+sed -i "s/localhost/$IP/" $DOCKER_PATH/.git/hooks/post-commit
+

buildbot/buildbot-cfg/buildbot.conf

@@ -0,0 +1,18 @@
+[program:buildmaster]
+command=su vagrant -c "buildbot start master"
+directory=/data/buildbot
+chown= root:root
+redirect_stderr=true
+stdout_logfile=/var/log/supervisor/buildbot-master.log
+stderr_logfile=/var/log/supervisor/buildbot-master.log
+
+[program:buildworker]
+command=buildslave start slave
+directory=/data/buildbot
+chown= root:root
+redirect_stderr=true
+stdout_logfile=/var/log/supervisor/buildbot-slave.log
+stderr_logfile=/var/log/supervisor/buildbot-slave.log
+
+[group:buildbot]
+programs=buildmaster,buildworker

buildbot/buildbot-cfg/master.cfg

@@ -0,0 +1,46 @@
+import os
+from buildbot.buildslave import BuildSlave
+from buildbot.schedulers.forcesched import ForceScheduler
+from buildbot.config import BuilderConfig
+from buildbot.process.factory import BuildFactory
+from buildbot.steps.shell import ShellCommand
+from buildbot.status import html
+from buildbot.status.web import authz, auth
+
+PORT_WEB = 8010         # Buildbot webserver port
+PORT_MASTER = 9989      # Port where buildbot master listen buildworkers
+TEST_USER = 'buildbot'  # Credential to authenticate build triggers
+TEST_PWD = 'docker'     # Credential to authenticate build triggers
+BUILDER_NAME = 'docker'
+BUILDPASSWORD = 'pass-docker'  # Credential to authenticate buildworkers
+DOCKER_PATH = '/data/docker'
+
+
+c = BuildmasterConfig = {}
+
+c['title'] = "Docker"
+c['titleURL'] = "waterfall"
+c['buildbotURL'] = "http://localhost:{0}/".format(PORT_WEB)
+c['db'] = {'db_url':"sqlite:///state.sqlite"}
+c['slaves'] = [BuildSlave('buildworker', BUILDPASSWORD)]
+c['slavePortnum'] = PORT_MASTER
+
+c['schedulers'] = [ForceScheduler(name='trigger',builderNames=[BUILDER_NAME])]
+
+# Docker test command
+test_cmd = """(
+    cd {0}/..; rm -rf docker-tmp; git clone docker docker-tmp;
+    cd docker-tmp; make test; exit_status=$?;
+    cd ..; rm -rf docker-tmp; exit $exit_status)""".format(DOCKER_PATH)
+
+# Builder
+factory = BuildFactory()
+factory.addStep(ShellCommand(description='Docker',logEnviron=False,
+    usePTY=True,command=test_cmd))
+c['builders'] = [BuilderConfig(name=BUILDER_NAME,slavenames=['buildworker'],
+    factory=factory)]
+
+# Status
+authz_cfg=authz.Authz(auth=auth.BasicAuth([(TEST_USER,TEST_PWD)]),
+    forceBuild='auth')
+c['status'] = [html.WebStatus(http_port=PORT_WEB, authz=authz_cfg)]

buildbot/buildbot-cfg/post-commit

@@ -0,0 +1,21 @@
+#!/usr/bin/env python
+
+'''Trigger buildbot docker test build
+
+   post-commit git hook designed to automatically trigger buildbot on
+   the provided vagrant docker VM.'''
+
+import requests
+
+USERNAME = 'buildbot'
+PASSWORD = 'docker'
+BASE_URL = 'http://localhost:8010'
+path = lambda s: BASE_URL + '/' + s
+
+try:
+    session = requests.session()
+    session.post(path('login'),data={'username':USERNAME,'passwd':PASSWORD})
+    session.post(path('builders/docker/force'),
+        data={'forcescheduler':'trigger','reason':'Test commit'})
+except:
+    pass

buildbot/buildbot.pp

@@ -0,0 +1,32 @@
+node default {
+    $USER = 'vagrant'
+    $ROOT_PATH = '/data/buildbot'
+    $DOCKER_PATH = '/data/docker'
+
+    exec {'apt_update': command => '/usr/bin/apt-get update' }
+    Package { require => Exec['apt_update'] }
+    group {'puppet': ensure => 'present'}
+
+    # Install dependencies
+    Package { ensure => 'installed' }
+    package { ['python-dev','python-pip','supervisor','lxc','bsdtar','git','golang']: }
+
+    file{[ '/data' ]:
+        owner => $USER, group => $USER, ensure => 'directory' }
+
+    file {'/var/tmp/requirements.txt':
+        content => template('requirements.txt') }
+
+    exec {'requirements':
+        require => [ Package['python-dev'], Package['python-pip'],
+            File['/var/tmp/requirements.txt'] ],
+        cwd     => '/var/tmp',
+        command => "/bin/sh -c '(/usr/bin/pip install -r requirements.txt;
+            rm /var/tmp/requirements.txt)'" }
+
+    exec {'buildbot-cfg-sh':
+        require => [ Package['supervisor'], Exec['requirements']],
+        path    => '/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin',
+        cwd     => '/data',
+        command => "$DOCKER_PATH/buildbot/buildbot-cfg/buildbot-cfg.sh" }
+}

buildbot/requirements.txt

@@ -0,0 +1,6 @@
+sqlalchemy<=0.7.9
+sqlalchemy-migrate>=0.7.2
+buildbot==0.8.7p1
+buildbot_slave==0.8.7p1
+nose==1.2.1
+requests==1.1.0