diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000000..f6c83997aa --- /dev/null +++ b/.travis.yml @@ -0,0 +1,23 @@ +# Note: right now we don't use go-specific features of travis. +# Later we might automate "go test" etc. (or do it inside a docker container...?) + +language: go + +go: 1.2 + +# Disable the normal go build. +install: true + +before_script: + - env | sort + - sudo apt-get update -qq + - sudo apt-get install -qq python-yaml + - git remote add upstream git://github.com/dotcloud/docker.git + - git fetch --append --no-tags upstream refs/heads/master:refs/remotes/upstream/master +# sometimes we have upstream master already as origin/master (PRs), but other times we don't, so let's just make sure we have a completely unambiguous way to specify "upstream master" from here out + +script: + - hack/travis/dco.py + - hack/travis/gofmt.py + +# vim:set sw=2 ts=2: diff --git a/AUTHORS b/AUTHORS index f9e9ecdace..1ff9b8db02 100644 --- a/AUTHORS +++ b/AUTHORS @@ -48,7 +48,6 @@ Daniel YC Lin Darren Coxall David Calavera David Sissitka -Dinesh Subhraveti Deni Bertovic Dominik Honnef Don Spaulding diff --git a/CHANGELOG.md b/CHANGELOG.md index 55d8d756a8..eccf52ed00 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,40 @@ # Changelog -## 0.7.3 (2013-01-02) +## 0.7.4 (2014-01-07) + +#### Builder + +- Fix ADD caching issue with . prefixed path +- Fix docker build on devicemapper by reverting sparse file tar option +- Fix issue with file caching and prevent wrong cache hit +* Use same error handling while unmarshalling CMD and ENTRYPOINT + +#### Documentation + +* Simplify and streamline Amazon Quickstart +* Install instructions use unprefixed fedora image +* Update instructions for mtu flag for Docker on GCE ++ Add Ubuntu Saucy to installation +- Fix for wrong version warning on master instead of latest + +#### Runtime + +- Only get the image's rootfs when we need to calculate the image size +- Correctly handle unmapping UDP ports +* Make CopyFileWithTar use a pipe instead of a buffer to save memory on docker build +- Fix login message to say pull instead of push +- Fix "docker load" help by removing "SOURCE" prompt and mentioning STDIN +* Make blank -H option default to the same as no -H was sent +* Extract cgroups utilities to own submodule + +#### Other + ++ Add Travis CI configuration to validate DCO and gofmt requirements ++ Add Developer Certificate of Origin Text +* Upgrade VBox Guest Additions +* Check standalone header when pinging a registry server + +## 0.7.3 (2014-01-02) #### Builder diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f68270fd81..1caf9c8d73 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -105,17 +105,39 @@ name and email address match your git configuration. The AUTHORS file is regenerated occasionally from the git commit history, so a mismatch may result in your changes being overwritten. -### Approval +### Sign your work -Docker maintainers use LGTM (looks good to me) in comments on the code review -to indicate acceptance. +The sign-off is a simple line at the end of the explanation for the +patch, which certifies that you wrote it or otherwise have the right to +pass it on as an open-source patch. The rules are pretty simple: if you +can certify the below: + +``` +Docker Developer Grant and Certificate of Origin 1.0 + +By making a contribution to the Docker Project ("Project"), I represent and warrant that: + +a. The contribution was created in whole or in part by me and I have the right to submit the contribution on my own behalf or on behalf of a third party who has authorized me to submit this contribution to the Project; or + + +b. The contribution is based upon previous work that, to the best of my knowledge, is covered under an appropriate open source license and I have the right and authorization to submit that work with modifications, whether created in whole or in part by me, under the same open source license (unless I am permitted to submit under a different license) that I have identified in the contribution; or + +c. The contribution was provided directly to me by some other person who represented and warranted (a) or (b) and I have not modified it. + +d. I understand and agree that this Project and the contribution are publicly known and that a record of the contribution (including all personal information I submit with it, including my sign-off record) is maintained indefinitely and may be redistributed consistent with this Project or the open source license(s) involved. + +e. I hereby grant to the Project, Docker, Inc and its successors; and recipients of software distributed by the Project a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to reproduce, modify, prepare derivative works of, publicly display, publicly perform, sublicense, and distribute this contribution and such modifications and derivative works consistent with this Project, the open source license indicated in the previous work or other appropriate open source license specified by the Project and approved by the Open Source Initiative(OSI) at http://www.opensource.org. +``` + +then you just add a line saying + + Docker-DCO-1.0-Signed-off-by: Joe Smith (github: github_handle) + +using your real name (sorry, no pseudonyms or anonymous contributions.) + +If you have any questions, please refer to the FAQ in the [docs](http://docs.docker.io) -A change requires LGTMs from an absolute majority of the maintainers of each -component affected. For example, if a change affects docs/ and registry/, it -needs an absolute majority from the maintainers of docs/ AND, separately, an -absolute majority of the maintainers of registry -For more details see [MAINTAINERS.md](hack/MAINTAINERS.md) ### How can I become a maintainer? diff --git a/MAINTAINERS b/MAINTAINERS index ef3aeda493..4a6c0ec22c 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -2,6 +2,7 @@ Solomon Hykes (@shykes) Guillaume Charmes (@creack) Victor Vieux (@vieux) Michael Crosby (@crosbymichael) +.travis.yml: Tianon Gravi (@tianon) api.go: Victor Vieux (@vieux) Dockerfile: Tianon Gravi (@tianon) Makefile: Tianon Gravi (@tianon) diff --git a/REMOTE_TODO.md b/REMOTE_TODO.md new file mode 100644 index 0000000000..8caddc68ba --- /dev/null +++ b/REMOTE_TODO.md @@ -0,0 +1,46 @@ +``` +**GET** + send objects deprecate multi-stream +TODO "/events": getEvents, N +ok "/info": getInfo, 1 +ok "/version": getVersion, 1 +... "/images/json": getImagesJSON, N +TODO "/images/viz": getImagesViz, 0 yes +TODO "/images/search": getImagesSearch, N +#3490 "/images/{name:.*}/get": getImagesGet, 0 +TODO "/images/{name:.*}/history": getImagesHistory, 1 +TODO "/images/{name:.*}/json": getImagesByName, 1 +TODO "/containers/ps": getContainersJSON, N +TODO "/containers/json": getContainersJSON, 1 +ok "/containers/{name:.*}/export": getContainersExport, 0 +TODO "/containers/{name:.*}/changes": getContainersChanges, 1 +TODO "/containers/{name:.*}/json": getContainersByName, 1 +TODO "/containers/{name:.*}/top": getContainersTop, N +TODO "/containers/{name:.*}/attach/ws": wsContainersAttach, 0 yes + +**POST** +TODO "/auth": postAuth, 0 yes +ok "/commit": postCommit, 0 +TODO "/build": postBuild, 0 yes +TODO "/images/create": postImagesCreate, N yes yes (pull) +TODO "/images/{name:.*}/insert": postImagesInsert, N yes yes +TODO "/images/load": postImagesLoad, 1 yes (stdin) +TODO "/images/{name:.*}/push": postImagesPush, N yes +ok "/images/{name:.*}/tag": postImagesTag, 0 +ok "/containers/create": postContainersCreate, 0 +ok "/containers/{name:.*}/kill": postContainersKill, 0 +#3476 "/containers/{name:.*}/restart": postContainersRestart, 0 +ok "/containers/{name:.*}/start": postContainersStart, 0 +ok "/containers/{name:.*}/stop": postContainersStop, 0 +ok "/containers/{name:.*}/wait": postContainersWait, 0 +ok "/containers/{name:.*}/resize": postContainersResize, 0 +TODO "/containers/{name:.*}/attach": postContainersAttach, 0 yes +TODO "/containers/{name:.*}/copy": postContainersCopy, 0 yes + +**DELETE** +#3180 "/containers/{name:.*}": deleteContainers, 0 +TODO "/images/{name:.*}": deleteImages, N + +**OPTIONS** +ok "": optionsHandler, 0 +``` diff --git a/VERSION b/VERSION index f38fc5393f..0a1ffad4b4 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -0.7.3 +0.7.4 diff --git a/Vagrantfile b/Vagrantfile index 54fc783c00..def85e5d72 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -70,7 +70,7 @@ SCRIPT # trigger dkms to build the virtualbox guest module install. $vbox_script = <`_ menu on your AWS Console. - * When picking the source AMI for your instance type, select "Community - AMIs". + * Click the ``Select`` button for a 64Bit Ubuntu image. For example: Ubuntu Server 12.04.3 LTS - * Search for ``amd64 precise``. Pick one of the amd64 Ubuntu images. - - * If you choose a EBS enabled AMI, you'll also be able to launch a + * For testing you can use the default (possibly free) ``t1.micro`` instance (more info on `pricing - `_). ``t1.micro`` instances are - eligible for Amazon's Free Usage Tier. + `_). - * When you click select you'll be taken to the instance setup, and you're one - click away from having your Ubuntu VM up and running. + * Click the ``Next: Configure Instance Details`` button at the bottom right. 2. **Tell CloudInit to install Docker:** diff --git a/docs/sources/installation/fedora.rst b/docs/sources/installation/fedora.rst index de296b4df2..6dd2bf91d9 100644 --- a/docs/sources/installation/fedora.rst +++ b/docs/sources/installation/fedora.rst @@ -67,7 +67,7 @@ Now let's verify that Docker is working. .. code-block:: bash - sudo docker run -i -t mattdm/fedora /bin/bash + sudo docker run -i -t fedora /bin/bash **Done!**, now continue with the :ref:`hello_world` example. diff --git a/docs/sources/installation/google.rst b/docs/sources/installation/google.rst index ff38e1e6e4..38aab019db 100644 --- a/docs/sources/installation/google.rst +++ b/docs/sources/installation/google.rst @@ -57,12 +57,13 @@ docker-playground:~$ curl get.docker.io | bash docker-playground:~$ sudo update-rc.d docker defaults -7. If running in zones: us-central1-a, europe-west1-1, and europe-west1-b, the docker daemon must be started with the `-mtu` flag. Without the flag, you may experience intermittent network pauses. +7. If running in zones: ``us-central1-a``, ``europe-west1-1``, and ``europe-west1-b``, the docker daemon must be started with the ``-mtu`` flag. Without the flag, you may experience intermittent network pauses. `See this issue `_ for more details. .. code-block:: bash - docker -d -mtu 1460 + docker-playground:~$ echo "DOCKER_OPTS="$DOCKER_OPTS -mtu 1460" | sudo tee -a /etc/defaults/docker + docker-playground:~$ sudo service docker restart 8. Start a new container: diff --git a/docs/sources/installation/rhel.rst b/docs/sources/installation/rhel.rst index b928b333f4..9036fb79ea 100644 --- a/docs/sources/installation/rhel.rst +++ b/docs/sources/installation/rhel.rst @@ -65,7 +65,7 @@ Now let's verify that Docker is working. .. code-block:: bash - sudo docker run -i -t mattdm/fedora /bin/bash + sudo docker run -i -t fedora /bin/bash **Done!**, now continue with the :ref:`hello_world` example. diff --git a/docs/sources/installation/ubuntulinux.rst b/docs/sources/installation/ubuntulinux.rst index 00400a94bc..e4432c6710 100644 --- a/docs/sources/installation/ubuntulinux.rst +++ b/docs/sources/installation/ubuntulinux.rst @@ -17,7 +17,7 @@ Ubuntu Docker is supported on the following versions of Ubuntu: - :ref:`ubuntu_precise` -- :ref:`ubuntu_raring` +- :ref:`ubuntu_raring_saucy` Please read :ref:`ufw`, if you plan to use `UFW (Uncomplicated Firewall) `_ @@ -108,10 +108,12 @@ Type ``exit`` to exit **Done!**, now continue with the :ref:`hello_world` example. -.. _ubuntu_raring: +.. _ubuntu_raring_saucy: -Ubuntu Raring 13.04 (64 bit) -^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Ubuntu Raring 13.04 and Saucy 13.10 (64 bit) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +These instructions cover both Ubuntu Raring 13.04 and Saucy 13.10. Dependencies ------------ @@ -169,7 +171,6 @@ Type ``exit`` to exit **Done!**, now continue with the :ref:`hello_world` example. - .. _ufw: Docker and UFW diff --git a/docs/sources/use/basics.rst b/docs/sources/use/basics.rst index 8c896ae793..c5d9871b28 100644 --- a/docs/sources/use/basics.rst +++ b/docs/sources/use/basics.rst @@ -117,6 +117,11 @@ For example: * ``tcp://host:4243`` -> tcp connection on host:4243 * ``unix://path/to/socket`` -> unix socket located at ``path/to/socket`` +``-H``, when empty, will default to the same value as when no ``-H`` was passed in. + +``-H`` also accepts short form for TCP bindings: +``host[:port]`` or ``:port`` + .. code-block:: bash # Run docker in daemon mode diff --git a/docs/sources/use/networking.rst b/docs/sources/use/networking.rst index 0e81440e03..4e75fbc20d 100644 --- a/docs/sources/use/networking.rst +++ b/docs/sources/use/networking.rst @@ -82,7 +82,7 @@ In this scenario: $ sudo ifconfig bridge0 192.168.227.1 netmask 255.255.255.0 # Edit your Docker startup file - $ echo "DOCKER_OPTS=\"-b=bridge0\"" /etc/default/docker + $ echo "DOCKER_OPTS=\"-b=bridge0\"" >> /etc/default/docker # Start Docker $ sudo service docker start diff --git a/docs/theme/docker/static/js/docs.js b/docs/theme/docker/static/js/docs.js index 4151ea5a80..03401909fa 100755 --- a/docs/theme/docker/static/js/docs.js +++ b/docs/theme/docker/static/js/docs.js @@ -92,7 +92,7 @@ $(function(){ $('.version-flyer ul').html('
  • Local
    • '); } - if (doc_version == "master") { + if (doc_version == "latest") { $('.version-flyer .version-note').hide(); } diff --git a/engine/http.go b/engine/http.go index 6391b3ff5a..b115912e2c 100644 --- a/engine/http.go +++ b/engine/http.go @@ -1,8 +1,8 @@ package engine import ( - "path" "net/http" + "path" ) // ServeHTTP executes a job as specified by the http request `r`, and sends the @@ -22,7 +22,7 @@ func (eng *Engine) ServeHTTP(w http.ResponseWriter, r *http.Request) { jobArgs = []string{} } w.Header().Set("Job-Name", jobName) - for _, arg := range(jobArgs) { + for _, arg := range jobArgs { w.Header().Add("Job-Args", arg) } job := eng.Job(jobName, jobArgs...) diff --git a/graph.go b/graph.go index 058ef7e93d..176626d60a 100644 --- a/graph.go +++ b/graph.go @@ -87,17 +87,17 @@ func (graph *Graph) Get(name string) (*Image, error) { if err != nil { return nil, err } - // Check that the filesystem layer exists - rootfs, err := graph.driver.Get(img.ID) - if err != nil { - return nil, fmt.Errorf("Driver %s failed to get image rootfs %s: %s", graph.driver, img.ID, err) - } if img.ID != id { return nil, fmt.Errorf("Image stored at '%s' has wrong id '%s'", id, img.ID) } img.graph = graph if img.Size < 0 { + rootfs, err := graph.driver.Get(img.ID) + if err != nil { + return nil, fmt.Errorf("Driver %s failed to get image rootfs %s: %s", graph.driver, img.ID, err) + } + var size int64 if img.Parent == "" { if size, err = utils.TreeSize(rootfs); err != nil { diff --git a/graphdriver/vfs/driver.go b/graphdriver/vfs/driver.go index fab9d06f83..12230f463a 100644 --- a/graphdriver/vfs/driver.go +++ b/graphdriver/vfs/driver.go @@ -36,9 +36,8 @@ func (d *Driver) Cleanup() error { } func copyDir(src, dst string) error { - cmd := exec.Command("cp", "-aT", "--reflink=auto", src, dst) - if err := cmd.Run(); err != nil { - return err + if output, err := exec.Command("cp", "-aT", "--reflink=auto", src, dst).CombinedOutput(); err != nil { + return fmt.Errorf("Error VFS copying directory: %s (%s)", err, output) } return nil } diff --git a/hack/make.sh b/hack/make.sh index 90683b6ead..fe0f9c175f 100755 --- a/hack/make.sh +++ b/hack/make.sh @@ -17,7 +17,7 @@ set -e # DO NOT CALL THIS SCRIPT DIRECTLY. # - The right way to call this script is to invoke "make" from # your checkout of the Docker repository. -# the Makefile will so a "docker build -t docker ." and then +# the Makefile will do a "docker build -t docker ." and then # "docker run hack/make.sh" in the resulting container image. # diff --git a/hack/travis/dco.py b/hack/travis/dco.py new file mode 100755 index 0000000000..cab26fedd7 --- /dev/null +++ b/hack/travis/dco.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +import re +import subprocess +import yaml + +from env import commit_range + +commit_format = '-%n hash: "%h"%n author: %aN <%aE>%n message: |%n%w(0,2,2)%B' + +gitlog = subprocess.check_output([ + 'git', 'log', '--reverse', + '--format=format:'+commit_format, + '..'.join(commit_range), '--', +]) + +commits = yaml.load(gitlog) +if not commits: + exit(0) # what? how can we have no commits? + +DCO = 'Docker-DCO-1.0-Signed-off-by:' + +p = re.compile(r'^{0} ([^<]+) <([^<>@]+@[^<>]+)> \(github: (\S+)\)$'.format(re.escape(DCO)), re.MULTILINE|re.UNICODE) + +failed_commits = 0 + +for commit in commits: + commit['stat'] = subprocess.check_output([ + 'git', 'log', '--format=format:', '--max-count=1', + '--name-status', commit['hash'], '--', + ]) + if commit['stat'] == '': + print 'Commit {0} has no actual changed content, skipping.'.format(commit['hash']) + continue + + m = p.search(commit['message']) + if not m: + print 'Commit {1} does not have a properly formatted "{0}" marker.'.format(DCO, commit['hash']) + failed_commits += 1 + continue # print ALL the commits that don't have a proper DCO + + (name, email, github) = m.groups() + + # TODO verify that "github" is the person who actually made this commit via the GitHub API + +if failed_commits > 0: + exit(failed_commits) + +print 'All commits have a valid "{0}" marker.'.format(DCO) +exit(0) diff --git a/hack/travis/env.py b/hack/travis/env.py new file mode 100644 index 0000000000..86d90f1567 --- /dev/null +++ b/hack/travis/env.py @@ -0,0 +1,21 @@ +import os +import subprocess + +if 'TRAVIS' not in os.environ: + print 'TRAVIS is not defined; this should run in TRAVIS. Sorry.' + exit(127) + +if os.environ['TRAVIS_PULL_REQUEST'] != 'false': + commit_range = [os.environ['TRAVIS_BRANCH'], 'FETCH_HEAD'] +else: + try: + subprocess.check_call([ + 'git', 'log', '-1', '--format=format:', + os.environ['TRAVIS_COMMIT_RANGE'], '--', + ]) + commit_range = os.environ['TRAVIS_COMMIT_RANGE'].split('...') + if len(commit_range) == 1: # if it didn't split, it must have been separated by '..' instead + commit_range = commit_range[0].split('..') + except subprocess.CalledProcessError: + print 'TRAVIS_COMMIT_RANGE is invalid. This seems to be a force push. We will just assume it must be against upstream master and compare all commits in between.' + commit_range = ['upstream/master', 'HEAD'] diff --git a/hack/travis/gofmt.py b/hack/travis/gofmt.py new file mode 100755 index 0000000000..1bb062e2f6 --- /dev/null +++ b/hack/travis/gofmt.py @@ -0,0 +1,28 @@ +#!/usr/bin/env python +import subprocess + +from env import commit_range + +files = subprocess.check_output([ + 'git', 'diff', '--diff-filter=ACMR', + '--name-only', '...'.join(commit_range), '--', +]) + +exit_status = 0 + +for filename in files.split('\n'): + if filename.endswith('.go'): + try: + out = subprocess.check_output(['gofmt', '-s', '-l', filename]) + if out != '': + print out, + exit_status = 1 + except subprocess.CalledProcessError: + exit_status = 1 + +if exit_status != 0: + print 'Reformat the files listed above with "gofmt -s -w" and try again.' + exit(exit_status) + +print 'All files pass gofmt.' +exit(0) diff --git a/integration/buildfile_test.go b/integration/buildfile_test.go index dccee4a8d3..ef51777390 100644 --- a/integration/buildfile_test.go +++ b/integration/buildfile_test.go @@ -132,6 +132,23 @@ run [ "$(cat /e)" = "blah" ] [][2]string{{"/x", "hello"}, {"/", "blah"}}, }, + // Comments, shebangs, and executability, oh my! + { + ` +FROM {IMAGE} +# This is an ordinary comment. +RUN { echo '#!/bin/sh'; echo 'echo hello world'; } > /hello.sh +RUN [ ! -x /hello.sh ] +RUN chmod +x /hello.sh +RUN [ -x /hello.sh ] +RUN [ "$(cat /hello.sh)" = $'#!/bin/sh\necho hello world' ] +RUN [ "$(/hello.sh)" = "hello world" ] +`, + nil, + nil, + }, + + // Environment variable { ` from {IMAGE} @@ -142,6 +159,19 @@ run [ "$FOO" = "BAR" ] nil, }, + // Environment overwriting + { + ` +from {IMAGE} +env FOO BAR +run [ "$FOO" = "BAR" ] +env FOO BAZ +run [ "$FOO" = "BAZ" ] +`, + nil, + nil, + }, + { ` from {IMAGE} @@ -391,6 +421,8 @@ func TestBuildEntrypoint(t *testing.T) { } if img.Config.Entrypoint[0] != "/bin/echo" { + t.Log(img.Config.Entrypoint[0]) + t.Fail() } } @@ -425,7 +457,7 @@ func TestBuildEntrypointRunCleanup(t *testing.T) { } } -func checkCacheBehavior(t *testing.T, template testContextTemplate, expectHit bool) { +func checkCacheBehavior(t *testing.T, template testContextTemplate, expectHit bool) (imageId string) { eng := NewTestEngine(t) defer nuke(mkRuntimeFromEngine(eng, t)) @@ -434,20 +466,36 @@ func checkCacheBehavior(t *testing.T, template testContextTemplate, expectHit bo t.Fatal(err) } - imageId := img.ID + imageId = img.ID - img = nil img, err = buildImage(template, t, eng, expectHit) if err != nil { t.Fatal(err) } - hit := imageId == img.ID - if hit != expectHit { - t.Logf("Cache misbehavior, got hit=%t, expected hit=%t: (first: %s, second %s)", - hit, expectHit, imageId, img.ID) - t.Fail() + if hit := imageId == img.ID; hit != expectHit { + t.Fatalf("Cache misbehavior, got hit=%t, expected hit=%t: (first: %s, second %s)", hit, expectHit, imageId, img.ID) } + return +} + +func checkCacheBehaviorFromEngime(t *testing.T, template testContextTemplate, expectHit bool, eng *engine.Engine) (imageId string) { + img, err := buildImage(template, t, eng, true) + if err != nil { + t.Fatal(err) + } + + imageId = img.ID + + img, err = buildImage(template, t, eng, expectHit) + if err != nil { + t.Fatal(err) + } + + if hit := imageId == img.ID; hit != expectHit { + t.Fatalf("Cache misbehavior, got hit=%t, expected hit=%t: (first: %s, second %s)", hit, expectHit, imageId, img.ID) + } + return } func TestBuildImageWithCache(t *testing.T) { @@ -474,11 +522,61 @@ func TestBuildADDLocalFileWithCache(t *testing.T) { maintainer dockerio run echo "first" add foo /usr/lib/bla/bar + run [ "$(cat /usr/lib/bla/bar)" = "hello" ] run echo "second" + add . /src/ + run [ "$(cat /src/foo)" = "hello" ] `, - [][2]string{{"foo", "hello"}}, + [][2]string{ + {"foo", "hello"}, + }, nil} - checkCacheBehavior(t, template, true) + eng := NewTestEngine(t) + defer nuke(mkRuntimeFromEngine(eng, t)) + + id1 := checkCacheBehaviorFromEngime(t, template, true, eng) + template.files = append(template.files, [2]string{"bar", "hello2"}) + id2 := checkCacheBehaviorFromEngime(t, template, true, eng) + if id1 == id2 { + t.Fatal("The cache should have been invalided but hasn't.") + } + id3 := checkCacheBehaviorFromEngime(t, template, true, eng) + if id2 != id3 { + t.Fatal("The cache should have been used but hasn't.") + } + template.files[1][1] = "hello3" + id4 := checkCacheBehaviorFromEngime(t, template, true, eng) + if id3 == id4 { + t.Fatal("The cache should have been invalided but hasn't.") + } + template.dockerfile += ` + add ./bar /src2/ + run ls /src2/bar + ` + id5 := checkCacheBehaviorFromEngime(t, template, true, eng) + if id4 == id5 { + t.Fatal("The cache should have been invalided but hasn't.") + } + template.files[1][1] = "hello4" + id6 := checkCacheBehaviorFromEngime(t, template, true, eng) + if id5 == id6 { + t.Fatal("The cache should have been invalided but hasn't.") + } + + template.dockerfile += ` + add bar /src2/bar2 + add /bar /src2/bar3 + run ls /src2/bar2 /src2/bar3 + ` + id7 := checkCacheBehaviorFromEngime(t, template, true, eng) + if id6 == id7 { + t.Fatal("The cache should have been invalided but hasn't.") + } + template.files[1][1] = "hello5" + id8 := checkCacheBehaviorFromEngime(t, template, true, eng) + if id7 == id8 { + t.Fatal("The cache should have been invalided but hasn't.") + } } func TestBuildADDLocalFileWithoutCache(t *testing.T) { diff --git a/integration/iptables_test.go b/integration/iptables_test.go index 060d0fe074..1dd4194350 100644 --- a/integration/iptables_test.go +++ b/integration/iptables_test.go @@ -1,7 +1,7 @@ package docker import ( - "github.com/dotcloud/docker/iptables" + "github.com/dotcloud/docker/pkg/iptables" "os" "testing" ) diff --git a/links.go b/links.go index 2fe255b4c5..55834b92d2 100644 --- a/links.go +++ b/links.go @@ -2,7 +2,7 @@ package docker import ( "fmt" - "github.com/dotcloud/docker/iptables" + "github.com/dotcloud/docker/pkg/iptables" "path" "strings" ) diff --git a/mount/mount.go b/mount/mount.go index 253a4681ef..b087293a9d 100644 --- a/mount/mount.go +++ b/mount/mount.go @@ -4,6 +4,10 @@ import ( "time" ) +func GetMounts() ([]*MountInfo, error) { + return parseMountTable() +} + // Looks at /proc/self/mountinfo to determine of the specified // mountpoint has been mounted func Mounted(mountpoint string) (bool, error) { @@ -14,7 +18,7 @@ func Mounted(mountpoint string) (bool, error) { // Search the table for the mountpoint for _, e := range entries { - if e.mountpoint == mountpoint { + if e.Mountpoint == mountpoint { return true, nil } } diff --git a/mount/mountinfo.go b/mount/mountinfo.go index f4fd24b633..32996f05c8 100644 --- a/mount/mountinfo.go +++ b/mount/mountinfo.go @@ -5,22 +5,35 @@ import ( "fmt" "io" "os" + "strings" ) const ( - // We only parse upto the mountinfo because that is all we - // care about right now - mountinfoFormat = "%d %d %d:%d %s %s %s" + /* 36 35 98:0 /mnt1 /mnt2 rw,noatime master:1 - ext3 /dev/root rw,errors=continue + (1)(2)(3) (4) (5) (6) (7) (8) (9) (10) (11) + + (1) mount ID: unique identifier of the mount (may be reused after umount) + (2) parent ID: ID of parent (or of self for the top of the mount tree) + (3) major:minor: value of st_dev for files on filesystem + (4) root: root of the mount within the filesystem + (5) mount point: mount point relative to the process's root + (6) mount options: per mount options + (7) optional fields: zero or more fields of the form "tag[:value]" + (8) separator: marks the end of the optional fields + (9) filesystem type: name of filesystem of the form "type[.subtype]" + (10) mount source: filesystem specific information or "none" + (11) super options: per super block options*/ + mountinfoFormat = "%d %d %d:%d %s %s %s " ) -// Represents one line from /proc/self/mountinfo -type procEntry struct { - id, parent, major, minor int - source, mountpoint, opts string +type MountInfo struct { + Id, Parent, Major, Minor int + Root, Mountpoint, Opts string + Fstype, Source, VfsOpts string } // Parse /proc/self/mountinfo because comparing Dev and ino does not work from bind mounts -func parseMountTable() ([]*procEntry, error) { +func parseMountTable() ([]*MountInfo, error) { f, err := os.Open("/proc/self/mountinfo") if err != nil { return nil, err @@ -30,10 +43,10 @@ func parseMountTable() ([]*procEntry, error) { return parseInfoFile(f) } -func parseInfoFile(r io.Reader) ([]*procEntry, error) { +func parseInfoFile(r io.Reader) ([]*MountInfo, error) { var ( s = bufio.NewScanner(r) - out = []*procEntry{} + out = []*MountInfo{} ) for s.Scan() { @@ -42,14 +55,24 @@ func parseInfoFile(r io.Reader) ([]*procEntry, error) { } var ( - p = &procEntry{} + p = &MountInfo{} text = s.Text() ) + if _, err := fmt.Sscanf(text, mountinfoFormat, - &p.id, &p.parent, &p.major, &p.minor, - &p.source, &p.mountpoint, &p.opts); err != nil { + &p.Id, &p.Parent, &p.Major, &p.Minor, + &p.Root, &p.Mountpoint, &p.Opts); err != nil { return nil, fmt.Errorf("Scanning '%s' failed: %s", text, err) } + // Safe as mountinfo encodes mountpoints with spaces as \040. + index := strings.Index(text, " - ") + postSeparatorFields := strings.Fields(text[index+3:]) + if len(postSeparatorFields) != 3 { + return nil, fmt.Errorf("Error did not find 3 fields post '-' in '%s'", text) + } + p.Fstype = postSeparatorFields[0] + p.Source = postSeparatorFields[1] + p.VfsOpts = postSeparatorFields[2] out = append(out, p) } return out, nil diff --git a/network.go b/network.go index 5ee5c4bee8..22ea8ba757 100644 --- a/network.go +++ b/network.go @@ -4,7 +4,7 @@ import ( "encoding/binary" "errors" "fmt" - "github.com/dotcloud/docker/iptables" + "github.com/dotcloud/docker/pkg/iptables" "github.com/dotcloud/docker/pkg/netlink" "github.com/dotcloud/docker/proxy" "github.com/dotcloud/docker/utils" @@ -248,12 +248,12 @@ type PortMapper struct { } func (mapper *PortMapper) Map(ip net.IP, port int, backendAddr net.Addr) error { - mapKey := (&net.TCPAddr{Port: port, IP: ip}).String() - if _, exists := mapper.tcpProxies[mapKey]; exists { - return fmt.Errorf("Port %s is already in use", mapKey) - } if _, isTCP := backendAddr.(*net.TCPAddr); isTCP { + mapKey := (&net.TCPAddr{Port: port, IP: ip}).String() + if _, exists := mapper.tcpProxies[mapKey]; exists { + return fmt.Errorf("TCP Port %s is already in use", mapKey) + } backendPort := backendAddr.(*net.TCPAddr).Port backendIP := backendAddr.(*net.TCPAddr).IP if mapper.iptables != nil { @@ -270,6 +270,10 @@ func (mapper *PortMapper) Map(ip net.IP, port int, backendAddr net.Addr) error { mapper.tcpProxies[mapKey] = proxy go proxy.Run() } else { + mapKey := (&net.UDPAddr{Port: port, IP: ip}).String() + if _, exists := mapper.udpProxies[mapKey]; exists { + return fmt.Errorf("UDP: Port %s is already in use", mapKey) + } backendPort := backendAddr.(*net.UDPAddr).Port backendIP := backendAddr.(*net.UDPAddr).IP if mapper.iptables != nil { @@ -290,8 +294,8 @@ func (mapper *PortMapper) Map(ip net.IP, port int, backendAddr net.Addr) error { } func (mapper *PortMapper) Unmap(ip net.IP, port int, proto string) error { - mapKey := (&net.TCPAddr{Port: port, IP: ip}).String() if proto == "tcp" { + mapKey := (&net.TCPAddr{Port: port, IP: ip}).String() backendAddr, ok := mapper.tcpMapping[mapKey] if !ok { return fmt.Errorf("Port tcp/%s is not mapped", mapKey) @@ -307,6 +311,7 @@ func (mapper *PortMapper) Unmap(ip net.IP, port int, proto string) error { } delete(mapper.tcpMapping, mapKey) } else { + mapKey := (&net.UDPAddr{Port: port, IP: ip}).String() backendAddr, ok := mapper.udpMapping[mapKey] if !ok { return fmt.Errorf("Port udp/%s is not mapped", mapKey) diff --git a/network_test.go b/network_test.go index 184b497938..69fcba01a2 100644 --- a/network_test.go +++ b/network_test.go @@ -1,7 +1,7 @@ package docker import ( - "github.com/dotcloud/docker/iptables" + "github.com/dotcloud/docker/pkg/iptables" "github.com/dotcloud/docker/proxy" "net" "testing" @@ -340,6 +340,7 @@ func NewStubProxy(frontendAddr, backendAddr net.Addr) (proxy.Proxy, error) { } func TestPortMapper(t *testing.T) { + // FIXME: is this iptables chain still used anywhere? var chain *iptables.Chain mapper := &PortMapper{ tcpMapping: make(map[string]*net.TCPAddr), diff --git a/pkg/iptables/MAINTAINERS b/pkg/iptables/MAINTAINERS new file mode 100644 index 0000000000..1e998f8ac1 --- /dev/null +++ b/pkg/iptables/MAINTAINERS @@ -0,0 +1 @@ +Michael Crosby (@crosbymichael) diff --git a/iptables/iptables.go b/pkg/iptables/iptables.go similarity index 100% rename from iptables/iptables.go rename to pkg/iptables/iptables.go diff --git a/reflink_copy_darwin.go b/reflink_copy_darwin.go index 3f3147db0f..4f0ea8c4fd 100644 --- a/reflink_copy_darwin.go +++ b/reflink_copy_darwin.go @@ -1,8 +1,8 @@ package docker import ( - "os" "io" + "os" ) func CopyFile(dstFile, srcFile *os.File) error { diff --git a/reflink_copy_linux.go b/reflink_copy_linux.go index 8aae3abf3d..83c7f75413 100644 --- a/reflink_copy_linux.go +++ b/reflink_copy_linux.go @@ -25,8 +25,8 @@ btrfs_reflink(int fd_out, int fd_in) import "C" import ( - "os" "io" + "os" "syscall" ) diff --git a/registry/registry.go b/registry/registry.go index 3c793cf08c..a038fdfb66 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -25,11 +25,11 @@ var ( ErrLoginRequired = errors.New("Authentication is required.") ) -func pingRegistryEndpoint(endpoint string) error { +func pingRegistryEndpoint(endpoint string) (bool, error) { if endpoint == auth.IndexServerAddress() { // Skip the check, we now this one is valid // (and we never want to fallback to http in case of error) - return nil + return false, nil } httpDial := func(proto string, addr string) (net.Conn, error) { // Set the connect timeout to 5 seconds @@ -45,14 +45,26 @@ func pingRegistryEndpoint(endpoint string) error { client := &http.Client{Transport: httpTransport} resp, err := client.Get(endpoint + "_ping") if err != nil { - return err + return false, err } defer resp.Body.Close() if resp.Header.Get("X-Docker-Registry-Version") == "" { - return errors.New("This does not look like a Registry server (\"X-Docker-Registry-Version\" header not found in the response)") + return false, errors.New("This does not look like a Registry server (\"X-Docker-Registry-Version\" header not found in the response)") } - return nil + + standalone := resp.Header.Get("X-Docker-Registry-Standalone") + utils.Debugf("Registry standalone header: '%s'", standalone) + // If the header is absent, we assume true for compatibility with earlier + // versions of the registry + if standalone == "" { + return true, nil + // Accepted values are "true" (case-insensitive) and "1". + } else if strings.EqualFold(standalone, "true") || standalone == "1" { + return true, nil + } + // Otherwise, not standalone + return false, nil } func validateRepositoryName(repositoryName string) error { @@ -122,16 +134,16 @@ func ExpandAndVerifyRegistryUrl(hostname string) (string, error) { // there is no path given. Expand with default path hostname = hostname + "/v1/" } - if err := pingRegistryEndpoint(hostname); err != nil { + if _, err := pingRegistryEndpoint(hostname); err != nil { return "", errors.New("Invalid Registry endpoint: " + err.Error()) } return hostname, nil } endpoint := fmt.Sprintf("https://%s/v1/", hostname) - if err := pingRegistryEndpoint(endpoint); err != nil { + if _, err := pingRegistryEndpoint(endpoint); err != nil { utils.Debugf("Registry %s does not work (%s), falling back to http", endpoint, err) endpoint = fmt.Sprintf("http://%s/v1/", hostname) - if err = pingRegistryEndpoint(endpoint); err != nil { + if _, err = pingRegistryEndpoint(endpoint); err != nil { //TODO: triggering highland build can be done there without "failing" return "", errors.New("Invalid Registry endpoint: " + err.Error()) } @@ -682,12 +694,18 @@ func NewRegistry(authConfig *auth.AuthConfig, factory *utils.HTTPRequestFactory, return nil, err } - // If we're working with a private registry over HTTPS, send Basic Auth headers + // If we're working with a standalone private registry over HTTPS, send Basic Auth headers // alongside our requests. if indexEndpoint != auth.IndexServerAddress() && strings.HasPrefix(indexEndpoint, "https://") { - utils.Debugf("Endpoint %s is eligible for private registry auth. Enabling decorator.", indexEndpoint) - dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) - factory.AddDecorator(dec) + standalone, err := pingRegistryEndpoint(indexEndpoint) + if err != nil { + return nil, err + } + if standalone { + utils.Debugf("Endpoint %s is eligible for private registry auth. Enabling decorator.", indexEndpoint) + dec := utils.NewHTTPAuthDecorator(authConfig.Username, authConfig.Password) + factory.AddDecorator(dec) + } } r.reqFactory = factory diff --git a/registry/registry_test.go b/registry/registry_test.go index 69eb25b247..16bc431e55 100644 --- a/registry/registry_test.go +++ b/registry/registry_test.go @@ -23,10 +23,11 @@ func spawnTestRegistry(t *testing.T) *Registry { } func TestPingRegistryEndpoint(t *testing.T) { - err := pingRegistryEndpoint(makeURL("/v1/")) + standalone, err := pingRegistryEndpoint(makeURL("/v1/")) if err != nil { t.Fatal(err) } + assertEqual(t, standalone, true, "Expected standalone to be true (default)") } func TestGetRemoteHistory(t *testing.T) { diff --git a/runtime.go b/runtime.go index 7c7f722c5d..731e3a8784 100644 --- a/runtime.go +++ b/runtime.go @@ -4,11 +4,12 @@ import ( "container/list" "fmt" "github.com/dotcloud/docker/archive" - "github.com/dotcloud/docker/pkg/graphdb" + "github.com/dotcloud/docker/cgroups" "github.com/dotcloud/docker/graphdriver" "github.com/dotcloud/docker/graphdriver/aufs" _ "github.com/dotcloud/docker/graphdriver/devmapper" _ "github.com/dotcloud/docker/graphdriver/vfs" + "github.com/dotcloud/docker/pkg/graphdb" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -332,7 +333,7 @@ func (runtime *Runtime) restore() error { // FIXME: comment please! func (runtime *Runtime) UpdateCapabilities(quiet bool) { - if cgroupMemoryMountpoint, err := utils.FindCgroupMountpoint("memory"); err != nil { + if cgroupMemoryMountpoint, err := cgroups.FindCgroupMountpoint("memory"); err != nil { if !quiet { log.Printf("WARNING: %s\n", err) } @@ -582,11 +583,6 @@ func (runtime *Runtime) Commit(container *Container, repository, tag, comment, a return img, nil } -// FIXME: this is deprecated by the getFullName *function* -func (runtime *Runtime) getFullName(name string) (string, error) { - return getFullName(name) -} - func getFullName(name string) (string, error) { if name == "" { return "", fmt.Errorf("Container name cannot be empty") @@ -598,7 +594,7 @@ func getFullName(name string) (string, error) { } func (runtime *Runtime) GetByName(name string) (*Container, error) { - fullName, err := runtime.getFullName(name) + fullName, err := getFullName(name) if err != nil { return nil, err } @@ -614,7 +610,7 @@ func (runtime *Runtime) GetByName(name string) (*Container, error) { } func (runtime *Runtime) Children(name string) (map[string]*Container, error) { - name, err := runtime.getFullName(name) + name, err := getFullName(name) if err != nil { return nil, err } diff --git a/server.go b/server.go index a49a34d542..2405380ea3 100644 --- a/server.go +++ b/server.go @@ -6,6 +6,7 @@ import ( "fmt" "github.com/dotcloud/docker/archive" "github.com/dotcloud/docker/auth" + "github.com/dotcloud/docker/cgroups" "github.com/dotcloud/docker/engine" "github.com/dotcloud/docker/pkg/graphdb" "github.com/dotcloud/docker/registry" @@ -702,7 +703,7 @@ func (srv *Server) ContainerTop(name, psArgs string) (*APITop, error) { if !container.State.IsRunning() { return nil, fmt.Errorf("Container %s is not running", name) } - pids, err := utils.GetPidsForContainer(container.ID) + pids, err := cgroups.GetPidsForContainer(container.ID) if err != nil { return nil, err } @@ -1437,7 +1438,7 @@ func (srv *Server) ContainerDestroy(name string, removeVolume, removeLink bool) if container == nil { return fmt.Errorf("No such link: %s", name) } - name, err := srv.runtime.getFullName(name) + name, err := getFullName(name) if err != nil { return err } @@ -1714,12 +1715,8 @@ func (srv *Server) ImageGetCached(imgID string, config *Config) (*Image, error) return nil, nil } -func (srv *Server) RegisterLinks(name string, hostConfig *HostConfig) error { +func (srv *Server) RegisterLinks(container *Container, hostConfig *HostConfig) error { runtime := srv.runtime - container := runtime.Get(name) - if container == nil { - return fmt.Errorf("No such container: %s", name) - } if hostConfig != nil && hostConfig.Links != nil { for _, l := range hostConfig.Links { @@ -1792,8 +1789,7 @@ func (srv *Server) ContainerStart(job *engine.Job) engine.Status { } } // Register any links from the host config before starting the container - // FIXME: we could just pass the container here, no need to lookup by name again. - if err := srv.RegisterLinks(name, &hostConfig); err != nil { + if err := srv.RegisterLinks(container, &hostConfig); err != nil { job.Error(err) return engine.StatusErr } diff --git a/utils.go b/utils.go index 5bcc678184..3eb1eac045 100644 --- a/utils.go +++ b/utils.go @@ -322,7 +322,6 @@ func migratePortMappings(config *Config, hostConfig *HostConfig) error { return nil } - // Links come in the format of // name:alias func parseLink(rawLink string) (map[string]string, error) { diff --git a/utils/utils.go b/utils/utils.go index ffe90ac734..d5dbf35f0a 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -583,28 +583,6 @@ func CompareKernelVersion(a, b *KernelVersionInfo) int { return 0 } -func FindCgroupMountpoint(cgroupType string) (string, error) { - output, err := ioutil.ReadFile("/proc/mounts") - if err != nil { - return "", err - } - - // /proc/mounts has 6 fields per line, one mount per line, e.g. - // cgroup /sys/fs/cgroup/devices cgroup rw,relatime,devices 0 0 - for _, line := range strings.Split(string(output), "\n") { - parts := strings.Split(line, " ") - if len(parts) == 6 && parts[2] == "cgroup" { - for _, opt := range strings.Split(parts[3], ",") { - if opt == cgroupType { - return parts[1], nil - } - } - } - } - - return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType) -} - func GetKernelVersion() (*KernelVersionInfo, error) { var ( err error @@ -803,7 +781,7 @@ func ParseHost(defaultHost string, defaultPort int, defaultUnix, addr string) (s host string port int ) - + addr = strings.TrimSpace(addr) switch { case strings.HasPrefix(addr, "unix://"): proto = "unix" @@ -814,6 +792,9 @@ func ParseHost(defaultHost string, defaultPort int, defaultUnix, addr string) (s case strings.HasPrefix(addr, "tcp://"): proto = "tcp" addr = strings.TrimPrefix(addr, "tcp://") + case addr == "": + proto = "unix" + addr = defaultUnix default: if strings.Contains(addr, "://") { return "", fmt.Errorf("Invalid bind address protocol: %s", addr) @@ -1157,59 +1138,3 @@ func CopyFile(src, dst string) (int64, error) { defer df.Close() return io.Copy(df, sf) } - -// Returns the relative path to the cgroup docker is running in. -func GetThisCgroup(cgroupType string) (string, error) { - output, err := ioutil.ReadFile("/proc/self/cgroup") - if err != nil { - return "", err - } - for _, line := range strings.Split(string(output), "\n") { - parts := strings.Split(line, ":") - // any type used by docker should work - if parts[1] == cgroupType { - return parts[2], nil - } - } - return "", fmt.Errorf("cgroup '%s' not found in /proc/self/cgroup", cgroupType) -} - -// Returns a list of pids for the given container. -func GetPidsForContainer(id string) ([]int, error) { - pids := []int{} - - // memory is chosen randomly, any cgroup used by docker works - cgroupType := "memory" - - cgroupRoot, err := FindCgroupMountpoint(cgroupType) - if err != nil { - return pids, err - } - - cgroupThis, err := GetThisCgroup(cgroupType) - if err != nil { - return pids, err - } - - filename := filepath.Join(cgroupRoot, cgroupThis, id, "tasks") - if _, err := os.Stat(filename); os.IsNotExist(err) { - // With more recent lxc versions use, cgroup will be in lxc/ - filename = filepath.Join(cgroupRoot, cgroupThis, "lxc", id, "tasks") - } - - output, err := ioutil.ReadFile(filename) - if err != nil { - return pids, err - } - for _, p := range strings.Split(string(output), "\n") { - if len(p) == 0 { - continue - } - pid, err := strconv.Atoi(p) - if err != nil { - return pids, fmt.Errorf("Invalid pid '%s': %s", p, err) - } - pids = append(pids, pid) - } - return pids, nil -} diff --git a/utils/utils_test.go b/utils/utils_test.go index b31938d9b5..1f23755d11 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -316,6 +316,9 @@ func TestParseHost(t *testing.T) { if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "tcp://:7777"); err != nil || addr != "tcp://127.0.0.1:7777" { t.Errorf("tcp://:7777 -> expected tcp://127.0.0.1:7777, got %s", addr) } + if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, ""); err != nil || addr != "unix:///var/run/docker.sock" { + t.Errorf("empty argument -> expected unix:///var/run/docker.sock, got %s", addr) + } if addr, err := ParseHost(defaultHttpHost, defaultHttpPort, defaultUnix, "unix:///var/run/docker.sock"); err != nil || addr != "unix:///var/run/docker.sock" { t.Errorf("unix:///var/run/docker.sock -> expected unix:///var/run/docker.sock, got %s", addr) }