From d47df21a332db1c1d8efedfeb0e36934b09e7a88 Mon Sep 17 00:00:00 2001 From: shin- Date: Thu, 18 Jul 2013 19:22:07 +0200 Subject: [PATCH] Add brew script to the contrib folder --- contrib/brew/.gitignore | 1 + contrib/brew/README.md | 78 +++++++++++++++++++++++++++++++ contrib/brew/brew/__init__.py | 1 + contrib/brew/brew/brew.py | 87 +++++++++++++++++++++++++++++++++++ contrib/brew/brew/git.py | 48 +++++++++++++++++++ contrib/brew/docker-brew | 25 ++++++++++ contrib/brew/requirements.txt | 2 + contrib/brew/setup.py | 22 +++++++++ 8 files changed, 264 insertions(+) create mode 100644 contrib/brew/.gitignore create mode 100644 contrib/brew/README.md create mode 100644 contrib/brew/brew/__init__.py create mode 100644 contrib/brew/brew/brew.py create mode 100644 contrib/brew/brew/git.py create mode 100755 contrib/brew/docker-brew create mode 100644 contrib/brew/requirements.txt create mode 100644 contrib/brew/setup.py diff --git a/contrib/brew/.gitignore b/contrib/brew/.gitignore new file mode 100644 index 0000000000..0d20b6487c --- /dev/null +++ b/contrib/brew/.gitignore @@ -0,0 +1 @@ +*.pyc diff --git a/contrib/brew/README.md b/contrib/brew/README.md new file mode 100644 index 0000000000..d50e66f2a8 --- /dev/null +++ b/contrib/brew/README.md @@ -0,0 +1,78 @@ +# docker-brew + +docker-brew is a command-line tool used to build the docker standard library. + +## Install instructions + +1. Install python if it isn't already available on your OS of choice +1. Install the easy_install tool (`sudo apt-get install python-setuptools` +for Debian) +1. Install the python package manager, `pip` (`easy_install pip`) +1. Run the following command: `pip install -r requirements.txt` +1. You should now be able to use the `docker-brew` script as such. + +## Basics + + ./docker-brew -h + +Display usage and help. + + ./docker-brew + +Default build from the default repo/branch. Images will be created under the +`library/` namespace. Does not perform a remote push. + + ./docker-brew -n mycorp.com -b stable --push git://github.com/mycorp/docker + +Will fetch the library definition files in the `stable` branch of the +`git://github.com/mycorp/docker` repository and create images under the +`mycorp.com` namespace (e.g. `mycorp.com/ubuntu`). Created images will then +be pushed to the official docker repository (pending: support for private +repositories) + +## Library definition files + +The library definition files are plain text files found in the `library/` +subfolder of the docker repository. + +### File names + +The name of a definition file will determine the name of the image(s) it +creates. For example, the `library/ubuntu` file will create images in the +`/ubuntu` repository. If multiple instructions are present in +a single file, all images are expected to be created under a different tag. + +### Instruction format + +Each line represents a build instruction. +There are different formats that `docker-brew` is able to parse. + + + git://github.com/dotcloud/hipache + https://github.com/dotcloud/docker.git + +The simplest format. `docker-brew` will fetch data from the provided git +repository from the `HEAD`of its `master` branch. Generated image will be +tagged as `latest`. Use of this format is discouraged because there is no +way to ensure stability. + + + bleeding-edge git://github.com/dotcloud/docker + unstable https://github.com/dotcloud/docker-redis.git + +A more advanced format. `docker-brew` will fetch data from the provided git +repository from the `HEAD`of its `master` branch. Generated image will be +tagged as ``. Recommended if we always want to provide a snapshot +of the latest development. Again, no way to ensure stability. + + T: + 2.4.0 git://github.com/dotcloud/docker-redis T:2.4.0 + B: + zfs git://github.com/dotcloud/docker B:zfs-support + C: + 2.2.0 https://github.com/dotcloud/docker-redis.git C:a4bf8923ee4ec566d3ddc212 + +The most complete format. `docker-brew` will fetch data from the provided git +repository from the provided reference (if it's a branch, brew will fetch its +`HEAD`). Generated image will be tagged as ``. Recommended whenever +possible. \ No newline at end of file diff --git a/contrib/brew/brew/__init__.py b/contrib/brew/brew/__init__.py new file mode 100644 index 0000000000..f693ad67a4 --- /dev/null +++ b/contrib/brew/brew/__init__.py @@ -0,0 +1 @@ +from brew import build_library diff --git a/contrib/brew/brew/brew.py b/contrib/brew/brew/brew.py new file mode 100644 index 0000000000..69529201d6 --- /dev/null +++ b/contrib/brew/brew/brew.py @@ -0,0 +1,87 @@ +import os +import logging + +import docker + +import git + +DEFAULT_REPOSITORY = 'git://github.com/dotcloud/docker' +DEFAULT_BRANCH = 'library' + +logger = logging.getLogger(__name__) +logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', + level='DEBUG') +client = docker.Client() +processed = {} + + +def build_library(repository=None, branch=None, namespace=None, push=False): + if repository is None: + repository = DEFAULT_REPOSITORY + if branch is None: + branch = DEFAULT_BRANCH + + logger.info('Cloning docker repo from {0}, branch: {1}'.format( + repository, branch)) + #FIXME: set destination folder and only pull latest changes instead of + # cloning the whole repo everytime + dst_folder = git.clone_branch(repository, branch) + for buildfile in os.listdir(os.path.join(dst_folder, 'library')): + if buildfile == 'MAINTAINERS': + continue + f = open(os.path.join(dst_folder, 'library', buildfile)) + for line in f: + logger.debug('{0} ---> {1}'.format(buildfile, line)) + args = line.split() + try: + if len(args) > 3: + raise RuntimeError('Incorrect line format, ' + 'please refer to the docs') + + url = None + ref = 'refs/heads/master' + tag = None + if len(args) == 1: # Just a URL, simple mode + url = args[0] + elif len(args) == 2 or len(args) == 3: # docker-tag url + url = args[1] + tag = args[0] + + if len(args) == 3: # docker-tag url B:branch or T:tag + ref = None + if args[2].startswith('B:'): + ref = 'refs/heads/' + args[2][2:] + elif args[2].startswith('T:'): + ref = 'refs/tags/' + args[2][2:] + elif args[2].startswith('C:'): + ref = args[2][2:] + else: + raise RuntimeError('Incorrect line format, ' + 'please refer to the docs') + img = build_repo(url, ref, buildfile, tag, namespace, push) + processed['{0}@{1}'.format(url, ref)] = img + except Exception as e: + logger.exception(e) + f.close() + + +def build_repo(repository, ref, docker_repo, docker_tag, namespace, push): + docker_repo = '{0}/{1}'.format(namespace or 'library', docker_repo) + img_id = None + if '{0}@{1}'.format(repository, ref) not in processed.keys(): + logger.info('Cloning {0} (ref: {1})'.format(repository, ref)) + dst_folder = git.clone(repository, ref) + if not 'Dockerfile' in os.listdir(dst_folder): + raise RuntimeError('Dockerfile not found in cloned repository') + logger.info('Building using dockerfile...') + img_id, logs = client.build(path=dst_folder) + + if not img_id: + img_id = processed['{0}@{1}'.format(repository, ref)] + logger.info('Committing to {0}:{1}'.format(docker_repo, + docker_tag or 'latest')) + client.tag(img_id, docker_repo, docker_tag) + if push: + logger.info('Pushing result to the main registry') + client.push(docker_repo) + return img_id diff --git a/contrib/brew/brew/git.py b/contrib/brew/brew/git.py new file mode 100644 index 0000000000..40cae8753b --- /dev/null +++ b/contrib/brew/brew/git.py @@ -0,0 +1,48 @@ +import tempfile +import logging + +from dulwich import index +from dulwich.client import get_transport_and_path +from dulwich.repo import Repo + +logger = logging.getLogger(__name__) + + +def clone_branch(repo_url, branch="master", folder=None): + return clone(repo_url, 'refs/heads/' + branch, folder) + + +def clone_tag(repo_url, tag, folder=None): + return clone(repo_url, 'refs/tags/' + tag, folder) + + +def clone(repo_url, ref=None, folder=None): + is_commit = False + if ref is None: + ref = 'refs/heads/master' + elif not ref.startswith('refs/'): + is_commit = True + logger.debug("clone repo_url={0}, ref={1}".format(repo_url, ref)) + if folder is None: + folder = tempfile.mkdtemp() + logger.debug("folder = {0}".format(folder)) + rep = Repo.init(folder) + client, relative_path = get_transport_and_path(repo_url) + logger.debug("client={0}".format(client)) + + remote_refs = client.fetch(relative_path, rep) + for k, v in remote_refs.iteritems(): + try: + rep.refs.add_if_new(k, v) + except: + pass + + if is_commit: + rep['HEAD'] = rep.commit(ref) + else: + rep['HEAD'] = remote_refs[ref] + indexfile = rep.index_path() + tree = rep["HEAD"].tree + index.build_index_from_tree(rep.path, indexfile, rep.object_store, tree) + logger.debug("done") + return folder diff --git a/contrib/brew/docker-brew b/contrib/brew/docker-brew new file mode 100755 index 0000000000..d1843d412b --- /dev/null +++ b/contrib/brew/docker-brew @@ -0,0 +1,25 @@ +#!/usr/bin/env python + +import argparse + +import brew + + +DEFAULT_REPOSITORY = 'git://github.com/dotcloud/docker' +DEFAULT_BRANCH = 'library' + +if __name__ == '__main__': + parser = argparse.ArgumentParser('Build the docker standard library') + parser.add_argument('--push', action='store_true', default=False, + help='push generated repositories to the official registry') + parser.add_argument('-n', metavar='NAMESPACE', default='library', + help='namespace used for generated repositories.' + ' Default is library') + parser.add_argument('-b', metavar='BRANCH', default=DEFAULT_BRANCH, + help='branch in the repository where the library definition' + ' files will be fetched. Default is ' + DEFAULT_BRANCH) + parser.add_argument('repository', default=DEFAULT_REPOSITORY, nargs='?', + help='git repository containing the library definition files.' + ' Default is ' + DEFAULT_REPOSITORY) + args = parser.parse_args() + brew.build_library(args.repository, args.b, args.n, args.push) \ No newline at end of file diff --git a/contrib/brew/requirements.txt b/contrib/brew/requirements.txt new file mode 100644 index 0000000000..5278498791 --- /dev/null +++ b/contrib/brew/requirements.txt @@ -0,0 +1,2 @@ +dulwich==0.9.0 +docker==0.1.0 \ No newline at end of file diff --git a/contrib/brew/setup.py b/contrib/brew/setup.py new file mode 100644 index 0000000000..8a099c99ff --- /dev/null +++ b/contrib/brew/setup.py @@ -0,0 +1,22 @@ +#!/usr/bin/env python +import os +from setuptools import setup + +ROOT_DIR = os.path.dirname(__file__) +SOURCE_DIR = os.path.join(ROOT_DIR) + +test_requirements = [] +setup( + name="docker-brew", + version='0.0.1', + description="-", + packages=['dockerbrew'], + install_requires=['dulwich', 'docker'] + test_requirements, + zip_safe=False, + classifiers=['Development Status :: 3 - Alpha', + 'Environment :: Other Environment', + 'Intended Audience :: Developers', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Topic :: Utilities'], + )