diff --git a/contrib/brew/.gitignore b/contrib/brew/.gitignore new file mode 100644 index 0000000000..0d20b6487c --- /dev/null +++ b/contrib/brew/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/contrib/brew/README.md b/contrib/brew/README.md new file mode 100644 index 0000000000..d50e66f2a8 --- /dev/null +++ b/contrib/brew/README.md @@ -0,0 +1,78 @@ +# docker-brew + +docker-brew is a command-line tool used to build the docker standard library. + +## Install instructions + +1. Install python if it isn't already available on your OS of choice +1. Install the easy_install tool (`sudo apt-get install python-setuptools` +for Debian) +1. Install the python package manager, `pip` (`easy_install pip`) +1. Run the following command: `pip install -r requirements.txt` +1. You should now be able to use the `docker-brew` script as such. + +## Basics + + ./docker-brew -h + +Display usage and help. + + ./docker-brew + +Default build from the default repo/branch. Images will be created under the +`library/` namespace. Does not perform a remote push. + + ./docker-brew -n mycorp.com -b stable --push git://github.com/mycorp/docker + +Will fetch the library definition files in the `stable` branch of the +`git://github.com/mycorp/docker` repository and create images under the +`mycorp.com` namespace (e.g. `mycorp.com/ubuntu`). Created images will then +be pushed to the official docker repository (pending: support for private +repositories) + +## Library definition files + +The library definition files are plain text files found in the `library/` +subfolder of the docker repository. + +### File names + +The name of a definition file will determine the name of the image(s) it +creates. For example, the `library/ubuntu` file will create images in the +`/ubuntu` repository. If multiple instructions are present in +a single file, all images are expected to be created under a different tag. + +### Instruction format + +Each line represents a build instruction. +There are different formats that `docker-brew` is able to parse. + + + git://github.com/dotcloud/hipache + https://github.com/dotcloud/docker.git + +The simplest format. `docker-brew` will fetch data from the provided git +repository from the `HEAD`of its `master` branch. Generated image will be +tagged as `latest`. Use of this format is discouraged because there is no +way to ensure stability. + + + bleeding-edge git://github.com/dotcloud/docker + unstable https://github.com/dotcloud/docker-redis.git + +A more advanced format. `docker-brew` will fetch data from the provided git +repository from the `HEAD`of its `master` branch. Generated image will be +tagged as ``. Recommended if we always want to provide a snapshot +of the latest development. Again, no way to ensure stability. + + T: + 2.4.0 git://github.com/dotcloud/docker-redis T:2.4.0 + B: + zfs git://github.com/dotcloud/docker B:zfs-support + C: + 2.2.0 https://github.com/dotcloud/docker-redis.git C:a4bf8923ee4ec566d3ddc212 + +The most complete format. `docker-brew` will fetch data from the provided git +repository from the provided reference (if it's a branch, brew will fetch its +`HEAD`). Generated image will be tagged as ``. Recommended whenever +possible. \ No newline at end of file diff --git a/contrib/brew/brew/__init__.py b/contrib/brew/brew/__init__.py new file mode 100644 index 0000000000..a9476ff5a4 --- /dev/null +++ b/contrib/brew/brew/__init__.py @@ -0,0 +1 @@ +from brew import build_library, DEFAULT_REPOSITORY, DEFAULT_BRANCH diff --git a/contrib/brew/brew/brew.py b/contrib/brew/brew/brew.py new file mode 100644 index 0000000000..73d59877fb --- /dev/null +++ b/contrib/brew/brew/brew.py @@ -0,0 +1,156 @@ +import os +import logging +from shutil import rmtree + +import docker + +import git + +DEFAULT_REPOSITORY = 'git://github.com/dotcloud/docker' +DEFAULT_BRANCH = 'library' + +logger = logging.getLogger(__name__) +logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', + level='INFO') +client = docker.Client() +processed = {} + + +def build_library(repository=None, branch=None, namespace=None, push=False, + debug=False, prefill=True, registry=None): + dst_folder = None + summary = Summary() + if repository is None: + repository = DEFAULT_REPOSITORY + if branch is None: + branch = DEFAULT_BRANCH + if debug: + logger.setLevel('DEBUG') + + if not (repository.startswith('https://') or repository.startswith('git://')): + logger.info('Repository provided assumed to be a local path') + dst_folder = repository + + #FIXME: set destination folder and only pull latest changes instead of + # cloning the whole repo everytime + if not dst_folder: + logger.info('Cloning docker repo from {0}, branch: {1}'.format( + repository, branch)) + dst_folder = git.clone_branch(repository, branch) + for buildfile in os.listdir(os.path.join(dst_folder, 'library')): + if buildfile == 'MAINTAINERS': + continue + f = open(os.path.join(dst_folder, 'library', buildfile)) + linecnt = 0 + for line in f: + linecnt = linecnt + 1 + logger.debug('{0} ---> {1}'.format(buildfile, line)) + args = line.split() + try: + if len(args) > 3: + raise RuntimeError('Incorrect line format, ' + 'please refer to the docs') + + url = None + ref = 'refs/heads/master' + tag = None + if len(args) == 1: # Just a URL, simple mode + url = args[0] + elif len(args) == 2 or len(args) == 3: # docker-tag url + url = args[1] + tag = args[0] + + if len(args) == 3: # docker-tag url B:branch or T:tag + ref = None + if args[2].startswith('B:'): + ref = 'refs/heads/' + args[2][2:] + elif args[2].startswith('T:'): + ref = 'refs/tags/' + args[2][2:] + elif args[2].startswith('C:'): + ref = args[2][2:] + else: + raise RuntimeError('Incorrect line format, ' + 'please refer to the docs') + if prefill: + logger.debug('Pulling {0} from official repository (cache ' + 'fill)'.format(buildfile)) + client.pull(buildfile) + img = build_repo(url, ref, buildfile, tag, namespace, push, + registry) + summary.add_success(buildfile, (linecnt, line), img) + processed['{0}@{1}'.format(url, ref)] = img + except Exception as e: + logger.exception(e) + summary.add_exception(buildfile, (linecnt, line), e) + + f.close() + if dst_folder != repository: + rmtree(dst_folder, True) + summary.print_summary(logger) + + +def build_repo(repository, ref, docker_repo, docker_tag, namespace, push, registry): + docker_repo = '{0}/{1}'.format(namespace or 'library', docker_repo) + img_id = None + if '{0}@{1}'.format(repository, ref) not in processed.keys(): + logger.info('Cloning {0} (ref: {1})'.format(repository, ref)) + dst_folder = git.clone(repository, ref) + if not 'Dockerfile' in os.listdir(dst_folder): + raise RuntimeError('Dockerfile not found in cloned repository') + logger.info('Building using dockerfile...') + img_id, logs = client.build(path=dst_folder, quiet=True) + rmtree(dst_folder, True) + else: + img_id = processed['{0}@{1}'.format(repository, ref)] + logger.info('Committing to {0}:{1}'.format(docker_repo, + docker_tag or 'latest')) + client.tag(img_id, docker_repo, docker_tag) + if push: + logger.info('Pushing result to registry {0}'.format( + registry or "default")) + if registry is not None: + docker_repo = '{0}/{1}'.format(registry, docker_repo) + logger.info('Also tagging {0}'.format(docker_repo)) + client.tag(img_id, docker_repo, docker_tag) + client.push(docker_repo) + return img_id + + +class Summary(object): + def __init__(self): + self._summary = {} + self._has_exc = False + + def _add_data(self, image, linestr, data): + if image not in self._summary: + self._summary[image] = { linestr: data } + else: + self._summary[image][linestr] = data + + def add_exception(self, image, line, exc): + lineno, linestr = line + self._add_data(image, linestr, { 'line': lineno, 'exc': str(exc) }) + self._has_exc = True + + def add_success(self, image, line, img_id): + lineno, linestr = line + self._add_data(image, linestr, { 'line': lineno, 'id': img_id }) + + def print_summary(self, logger=None): + linesep = ''.center(61, '-') + '\n' + s = 'BREW BUILD SUMMARY\n' + linesep + success = 'OVERALL SUCCESS: {}\n'.format(not self._has_exc) + details = linesep + for image, lines in self._summary.iteritems(): + details = details + '{}\n{}'.format(image, linesep) + for linestr, data in lines.iteritems(): + details = details + '{0:2} | {1} | {2:50}\n'.format( + data['line'], + 'KO' if 'exc' in data else 'OK', + data['exc'] if 'exc' in data else data['id'] + ) + details = details + linesep + if logger: + logger.info(s + success + details) + else: + print s, success, details \ No newline at end of file diff --git a/contrib/brew/brew/git.py b/contrib/brew/brew/git.py new file mode 100644 index 0000000000..40cae8753b --- /dev/null +++ b/contrib/brew/brew/git.py @@ -0,0 +1,48 @@ +import tempfile +import logging + +from dulwich import index +from dulwich.client import get_transport_and_path +from dulwich.repo import Repo + +logger = logging.getLogger(__name__) + + +def clone_branch(repo_url, branch="master", folder=None): + return clone(repo_url, 'refs/heads/' + branch, folder) + + +def clone_tag(repo_url, tag, folder=None): + return clone(repo_url, 'refs/tags/' + tag, folder) + + +def clone(repo_url, ref=None, folder=None): + is_commit = False + if ref is None: + ref = 'refs/heads/master' + elif not ref.startswith('refs/'): + is_commit = True + logger.debug("clone repo_url={0}, ref={1}".format(repo_url, ref)) + if folder is None: + folder = tempfile.mkdtemp() + logger.debug("folder = {0}".format(folder)) + rep = Repo.init(folder) + client, relative_path = get_transport_and_path(repo_url) + logger.debug("client={0}".format(client)) + + remote_refs = client.fetch(relative_path, rep) + for k, v in remote_refs.iteritems(): + try: + rep.refs.add_if_new(k, v) + except: + pass + + if is_commit: + rep['HEAD'] = rep.commit(ref) + else: + rep['HEAD'] = remote_refs[ref] + indexfile = rep.index_path() + tree = rep["HEAD"].tree + index.build_index_from_tree(rep.path, indexfile, rep.object_store, tree) + logger.debug("done") + return folder diff --git a/contrib/brew/docker-brew b/contrib/brew/docker-brew new file mode 100755 index 0000000000..ead6b65075 --- /dev/null +++ b/contrib/brew/docker-brew @@ -0,0 +1,29 @@ +#!/usr/bin/env python + +import argparse + +import brew + + +if __name__ == '__main__': + parser = argparse.ArgumentParser('Build the docker standard library') + parser.add_argument('--push', action='store_true', default=False, + help='Push generated repositories') + parser.add_argument('--debug', default=False, action='store_true', + help='Enable debugging output') + parser.add_argument('--noprefill', default=True, action='store_false', + dest='prefill', help='Disable cache prefill') + parser.add_argument('-n', metavar='NAMESPACE', default='library', + help='Namespace used for generated repositories.' + ' Default is library') + parser.add_argument('-b', metavar='BRANCH', default=brew.DEFAULT_BRANCH, + help='Branch in the repository where the library definition' + ' files will be fetched. Default is ' + brew.DEFAULT_BRANCH) + parser.add_argument('repository', default=brew.DEFAULT_REPOSITORY, + nargs='?', help='git repository containing the library definition' + ' files. Default is ' + brew.DEFAULT_REPOSITORY) + parser.add_argument('--reg', default=None, help='Registry address to' + ' push build results to. Also sets push to true.') + args = parser.parse_args() + brew.build_library(args.repository, args.b, args.n, + args.push or args.reg is not None, args.debug, args.prefill, args.reg) diff --git a/contrib/brew/requirements.txt b/contrib/brew/requirements.txt new file mode 100644 index 0000000000..78a574953d --- /dev/null +++ b/contrib/brew/requirements.txt @@ -0,0 +1,2 @@ +dulwich==0.9.0 +docker-py==0.1.3 \ No newline at end of file diff --git a/contrib/brew/setup.py b/contrib/brew/setup.py new file mode 100644 index 0000000000..8a099c99ff --- /dev/null +++ b/contrib/brew/setup.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +import os +from setuptools import setup + +ROOT_DIR = os.path.dirname(__file__) +SOURCE_DIR = os.path.join(ROOT_DIR) + +test_requirements = [] +setup( + name="docker-brew", + version='0.0.1', + description="-", + packages=['dockerbrew'], + install_requires=['dulwich', 'docker'] + test_requirements, + zip_safe=False, + classifiers=['Development Status :: 3 - Alpha', + 'Environment :: Other Environment', + 'Intended Audience :: Developers', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Topic :: Utilities'], + ) diff --git a/library/MAINTAINERS b/library/MAINTAINERS new file mode 100644 index 0000000000..b1c52939ef --- /dev/null +++ b/library/MAINTAINERS @@ -0,0 +1 @@ +Joffrey Fuhrer diff --git a/library/busybox b/library/busybox new file mode 100644 index 0000000000..b091054451 --- /dev/null +++ b/library/busybox @@ -0,0 +1 @@ +latest git://github.com/dotcloud/docker-busybox \ No newline at end of file diff --git a/library/hipache b/library/hipache new file mode 100644 index 0000000000..ded0582c63 --- /dev/null +++ b/library/hipache @@ -0,0 +1,2 @@ +latest git://github.com/dotcloud/hipache C:7362ff5b812f93eceafbdbf5e5959f676f731f80 +0.2.4 git://github.com/dotcloud/hipache C:7362ff5b812f93eceafbdbf5e5959f676f731f80 diff --git a/library/ubuntu b/library/ubuntu new file mode 100644 index 0000000000..8a5cdfd445 --- /dev/null +++ b/library/ubuntu @@ -0,0 +1,7 @@ +latest git://github.com/dotcloud/ubuntu-quantal +quantal git://github.com/dotcloud/ubuntu-quantal +12.10 git://github.com/dotcloud/ubuntu-quantal +precise git://github.com/shin-/ubuntu B:precise +12.04 git://github.com/shin-/ubuntu B:precise +raring git://github.com/shin-/ubuntu B:raring +13.04 git://github.com/shin-/ubuntu B:raring